def validate_args(args): # if not in API mode, public path cannot be set to root path if not args.api_only and args.public_path == '/': logger.error('Public path cannot be set to root path.') sys.exit(-1) # public path must start with `/` if args.public_path is not None and not args.public_path.startswith('/'): logger.error('Public path should always start with a `/`.') sys.exit(-1)
def retry(ntimes, function, time2sleep, *args, **kwargs): ''' try to execute `function` `ntimes`, if exception catched, the thread will sleep `time2sleep` seconds. ''' for i in range(ntimes): try: return function(*args, **kwargs) except Exception: error_info = '\n'.join(map(str, sys.exc_info())) logger.error("Unexpected error: %s" % error_info) time.sleep(time2sleep)
def get_url(path='', model='', **kwargs): server_url = get_server_url() + '/url/' data = json.dumps({'path': path, 'model': model}) headers = {"Content-Type": "application/json"} res = requests.post(url=server_url, headers=headers, data=data).json() err_code = res.get('code') msg = res.get('msg') if '000000' == err_code: url = msg.get('url') return url else: logger.error(msg) return
def retry(ntimes, function, time2sleep, *args, **kwargs): """ try to execute `function` `ntimes`, if exception catched, the thread will sleep `time2sleep` seconds. """ for i in range(ntimes): try: return function(*args, **kwargs) except Exception: if i < ntimes - 1: error_info = '\n'.join(map(str, sys.exc_info())) logger.error("Unexpected error: %s" % error_info) time.sleep(time2sleep) else: import traceback traceback.print_exc()
def calc_hyper_param_importance(df, hyper_param, target): new_df = df[[hyper_param, target]] no_missing_value_df = new_df.dropna() # Can not calc pearson correlation coefficient when number of samples is less or equal than 2 if len(no_missing_value_df) <= 2: logger.error("Number of samples is less or equal than 2.") return 0 correlation = no_missing_value_df[target].corr( no_missing_value_df[hyper_param]) if np.isnan(correlation): logger.warning("Correlation is nan!") return 0 return abs(correlation)
def get_hparam_list(log_reader): run2tag = get_logs(log_reader, 'hyper_parameters') runs = run2tag['runs'] results = [] records_list = [] for run in runs: run = log_reader.name2tags[run] if run in log_reader.name2tags else run log_reader.load_new_data() records = log_reader.data_manager.get_reservoir( "hyper_parameters").get_items(run, decode_tag('hparam')) records_list.append([records, run]) records_list.sort(key=lambda x: x[0][0].timestamp) for records, run in records_list: hparams = {} for hparamInfo in records[0].hparam.hparamInfos: hparam_type = hparamInfo.WhichOneof("type") if "float_value" == hparam_type: hparams[hparamInfo.name] = hparamInfo.float_value elif "string_value" == hparam_type: hparams[hparamInfo.name] = hparamInfo.string_value elif "int_value" == hparam_type: hparams[hparamInfo.name] = hparamInfo.int_value else: raise TypeError("Invalid hparams param value type `%s`." % hparam_type) metrics = {} for metricInfo in records[0].hparam.metricInfos: try: metrics_data = get_hparam_metric(log_reader, run, metricInfo.name) metrics[metricInfo.name] = metrics_data[-1][-1] except: logger.error( 'Missing data of metrics! Please make sure use add_scalar to log metrics data.' ) metrics[metricInfo.name] = None results.append({'name': run, 'hparams': hparams, 'metrics': metrics}) return results
def upload_to_dev(logdir=None, model=None): if not logdir and not model: logger.error( "Must specify directory to upload via `--logdir` or specify model to upload via `--model`." ) return walks = {} if logdir: walks = get_vdl_log_file(logdir) if not walks: logger.error("There is no valid log file in %s" % logdir) return res = apply_for_token() err_code = res.get('code') msg = res.get('msg') if '000000' == err_code: sts_ak = msg.get('sts_ak') sts_sk = msg.get('sts_sk') sts_token = msg.get('token') bucket_id = msg.get('dir') else: logger.error(msg) return if not sts_ak or not sts_sk or not sts_token: return bos_fs = bfile.BosConfigClient(bos_ak=sts_ak, bos_sk=sts_sk, bos_sts=sts_token) for key, value in walks.items(): filename = bos_fs.join(key, value) bos_fs.upload_object_from_file(path=bucket_id, filename=filename) if model: if os.path.getsize(model) > 1024 * 1024 * 100: logger.error('Size of model must less than 100M.') else: bos_fs.upload_object_from_file(path=bucket_id, filename=model) url = get_url(path=bucket_id, model=model) print("View your visualization results at: `%s`." % url)
def validate_args(args): # if not in API mode, public path cannot be set to root path if not args.api_only and args.public_path == '/': logger.error('Public path cannot be set to root path.') sys.exit(-1) # public path must start with `/` if args.public_path is not None and not args.public_path.startswith('/'): logger.error('Public path should always start with a `/`.') sys.exit(-1) # theme not support if args.theme is not None and args.theme not in support_themes: logger.error('Theme {} is not support.'.format(args.theme)) sys.exit(-1)
def try_call(function, *args, **kwargs): res = lib.retry(error_retry_times, function, error_sleep_time, *args, **kwargs) if not res: logger.error("Internal server error. Retry later.") return res
def get_hparam_indicator(log_reader): run2tag = get_logs(log_reader, 'hyper_parameters') runs = run2tag['runs'] hparams = {} metrics = {} records_list = [] for run in runs: run = log_reader.name2tags[run] if run in log_reader.name2tags else run log_reader.load_new_data() records = log_reader.data_manager.get_reservoir( "hyper_parameters").get_items(run, decode_tag('hparam')) records_list.append([records, run]) records_list.sort(key=lambda x: x[0][0].timestamp) runs = [run for r, run in records_list] for records, run in records_list: for hparamInfo in records[0].hparam.hparamInfos: type = hparamInfo.WhichOneof("type") if "float_value" == type: if hparamInfo.name not in hparams.keys(): hparams[hparamInfo.name] = { 'name': hparamInfo.name, 'type': 'continuous', 'values': [hparamInfo.float_value] } elif hparamInfo.float_value not in hparams[ hparamInfo.name]['values']: hparams[hparamInfo.name]['values'].append( hparamInfo.float_value) elif "string_value" == type: if hparamInfo.name not in hparams.keys(): hparams[hparamInfo.name] = { 'name': hparamInfo.name, 'type': 'string', 'values': [hparamInfo.string_value] } elif hparamInfo.string_value not in hparams[ hparamInfo.name]['values']: hparams[hparamInfo.name]['values'].append( hparamInfo.string_value) elif "int_value" == type: if hparamInfo.name not in hparams.keys(): hparams[hparamInfo.name] = { 'name': hparamInfo.name, 'type': 'numeric', 'values': [hparamInfo.int_value] } elif hparamInfo.int_value not in hparams[ hparamInfo.name]['values']: hparams[hparamInfo.name]['values'].append( hparamInfo.int_value) else: raise TypeError("Invalid hparams param value type `%s`." % type) for metricInfo in records[0].hparam.metricInfos: metrics[metricInfo.name] = { 'name': metricInfo.name, 'type': 'continuous', 'values': [] } for run in runs: try: metrics_data = get_hparam_metric(log_reader, run, metricInfo.name) metrics[metricInfo.name]['values'].append( metrics_data[-1][-1]) break except: logger.error( 'Missing data of metrics! Please make sure use add_scalar to log metrics data.' ) if len(metrics[metricInfo.name]['values']) == 0: metrics.pop(metricInfo.name) else: metrics[metricInfo.name].pop('values') results = { 'hparams': [value for key, value in hparams.items()], 'metrics': [value for key, value in metrics.items()] } return results