def validate_args(args):
# if not in API mode, public path cannot be set to root path
if not args.api_only and args.public_path == '/':
logger.error('Public path cannot be set to root path.')
sys.exit(-1)
# public path must start with `/`
if args.public_path is not None and not args.public_path.startswith('/'):
logger.error('Public path should always start with a `/`.')
sys.exit(-1)
def retry(ntimes, function, time2sleep, *args, **kwargs):
'''
try to execute `function` `ntimes`, if exception catched, the thread will
sleep `time2sleep` seconds.
'''
for i in range(ntimes):
try:
return function(*args, **kwargs)
except Exception:
error_info = '\n'.join(map(str, sys.exc_info()))
logger.error("Unexpected error: %s" % error_info)
time.sleep(time2sleep)
def get_url(path='', model='', **kwargs):
server_url = get_server_url() + '/url/'
data = json.dumps({'path': path, 'model': model})
headers = {"Content-Type": "application/json"}
res = requests.post(url=server_url, headers=headers, data=data).json()
err_code = res.get('code')
msg = res.get('msg')
if '000000' == err_code:
url = msg.get('url')
return url
else:
logger.error(msg)
return
def retry(ntimes, function, time2sleep, *args, **kwargs):
"""
try to execute `function` `ntimes`, if exception catched, the thread will
sleep `time2sleep` seconds.
"""
for i in range(ntimes):
try:
return function(*args, **kwargs)
except Exception:
if i < ntimes - 1:
error_info = '\n'.join(map(str, sys.exc_info()))
logger.error("Unexpected error: %s" % error_info)
time.sleep(time2sleep)
else:
import traceback
traceback.print_exc()
def calc_hyper_param_importance(df, hyper_param, target):
new_df = df[[hyper_param, target]]
no_missing_value_df = new_df.dropna()
# Can not calc pearson correlation coefficient when number of samples is less or equal than 2
if len(no_missing_value_df) <= 2:
logger.error("Number of samples is less or equal than 2.")
return 0
correlation = no_missing_value_df[target].corr(
no_missing_value_df[hyper_param])
if np.isnan(correlation):
logger.warning("Correlation is nan!")
return 0
return abs(correlation)
def get_hparam_list(log_reader):
run2tag = get_logs(log_reader, 'hyper_parameters')
runs = run2tag['runs']
results = []
records_list = []
for run in runs:
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir(
"hyper_parameters").get_items(run, decode_tag('hparam'))
records_list.append([records, run])
records_list.sort(key=lambda x: x[0][0].timestamp)
for records, run in records_list:
hparams = {}
for hparamInfo in records[0].hparam.hparamInfos:
hparam_type = hparamInfo.WhichOneof("type")
if "float_value" == hparam_type:
hparams[hparamInfo.name] = hparamInfo.float_value
elif "string_value" == hparam_type:
hparams[hparamInfo.name] = hparamInfo.string_value
elif "int_value" == hparam_type:
hparams[hparamInfo.name] = hparamInfo.int_value
else:
raise TypeError("Invalid hparams param value type `%s`." %
hparam_type)
metrics = {}
for metricInfo in records[0].hparam.metricInfos:
try:
metrics_data = get_hparam_metric(log_reader, run,
metricInfo.name)
metrics[metricInfo.name] = metrics_data[-1][-1]
except:
logger.error(
'Missing data of metrics! Please make sure use add_scalar to log metrics data.'
)
metrics[metricInfo.name] = None
results.append({'name': run, 'hparams': hparams, 'metrics': metrics})
return results
def upload_to_dev(logdir=None, model=None):
if not logdir and not model:
logger.error(
"Must specify directory to upload via `--logdir` or specify model to upload via `--model`."
)
return
walks = {}
if logdir:
walks = get_vdl_log_file(logdir)
if not walks:
logger.error("There is no valid log file in %s" % logdir)
return
res = apply_for_token()
err_code = res.get('code')
msg = res.get('msg')
if '000000' == err_code:
sts_ak = msg.get('sts_ak')
sts_sk = msg.get('sts_sk')
sts_token = msg.get('token')
bucket_id = msg.get('dir')
else:
logger.error(msg)
return
if not sts_ak or not sts_sk or not sts_token:
return
bos_fs = bfile.BosConfigClient(bos_ak=sts_ak,
bos_sk=sts_sk,
bos_sts=sts_token)
for key, value in walks.items():
filename = bos_fs.join(key, value)
bos_fs.upload_object_from_file(path=bucket_id, filename=filename)
if model:
if os.path.getsize(model) > 1024 * 1024 * 100:
logger.error('Size of model must less than 100M.')
else:
bos_fs.upload_object_from_file(path=bucket_id, filename=model)
url = get_url(path=bucket_id, model=model)
print("View your visualization results at: `%s`." % url)
def validate_args(args):
# if not in API mode, public path cannot be set to root path
if not args.api_only and args.public_path == '/':
logger.error('Public path cannot be set to root path.')
sys.exit(-1)
# public path must start with `/`
if args.public_path is not None and not args.public_path.startswith('/'):
logger.error('Public path should always start with a `/`.')
sys.exit(-1)
# theme not support
if args.theme is not None and args.theme not in support_themes:
logger.error('Theme {} is not support.'.format(args.theme))
sys.exit(-1)
def try_call(function, *args, **kwargs):
res = lib.retry(error_retry_times, function, error_sleep_time, *args,
**kwargs)
if not res:
logger.error("Internal server error. Retry later.")
return res
def get_hparam_indicator(log_reader):
run2tag = get_logs(log_reader, 'hyper_parameters')
runs = run2tag['runs']
hparams = {}
metrics = {}
records_list = []
for run in runs:
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir(
"hyper_parameters").get_items(run, decode_tag('hparam'))
records_list.append([records, run])
records_list.sort(key=lambda x: x[0][0].timestamp)
runs = [run for r, run in records_list]
for records, run in records_list:
for hparamInfo in records[0].hparam.hparamInfos:
type = hparamInfo.WhichOneof("type")
if "float_value" == type:
if hparamInfo.name not in hparams.keys():
hparams[hparamInfo.name] = {
'name': hparamInfo.name,
'type': 'continuous',
'values': [hparamInfo.float_value]
}
elif hparamInfo.float_value not in hparams[
hparamInfo.name]['values']:
hparams[hparamInfo.name]['values'].append(
hparamInfo.float_value)
elif "string_value" == type:
if hparamInfo.name not in hparams.keys():
hparams[hparamInfo.name] = {
'name': hparamInfo.name,
'type': 'string',
'values': [hparamInfo.string_value]
}
elif hparamInfo.string_value not in hparams[
hparamInfo.name]['values']:
hparams[hparamInfo.name]['values'].append(
hparamInfo.string_value)
elif "int_value" == type:
if hparamInfo.name not in hparams.keys():
hparams[hparamInfo.name] = {
'name': hparamInfo.name,
'type': 'numeric',
'values': [hparamInfo.int_value]
}
elif hparamInfo.int_value not in hparams[
hparamInfo.name]['values']:
hparams[hparamInfo.name]['values'].append(
hparamInfo.int_value)
else:
raise TypeError("Invalid hparams param value type `%s`." %
type)
for metricInfo in records[0].hparam.metricInfos:
metrics[metricInfo.name] = {
'name': metricInfo.name,
'type': 'continuous',
'values': []
}
for run in runs:
try:
metrics_data = get_hparam_metric(log_reader, run,
metricInfo.name)
metrics[metricInfo.name]['values'].append(
metrics_data[-1][-1])
break
except:
logger.error(
'Missing data of metrics! Please make sure use add_scalar to log metrics data.'
)
if len(metrics[metricInfo.name]['values']) == 0:
metrics.pop(metricInfo.name)
else:
metrics[metricInfo.name].pop('values')
results = {
'hparams': [value for key, value in hparams.items()],
'metrics': [value for key, value in metrics.items()]
}
return results
