def main():
args = parser.parse_args()
pipeline_config_path = find_config(args.config_path)
https = args.https
ssl_key = args.key
ssl_cert = args.cert
if args.download or args.mode == 'download':
deep_download(pipeline_config_path)
multi_instance = args.multi_instance
stateful = args.stateful
start_epoch_num = args.start_epoch_num
if args.mode == 'train':
train_evaluate_model_from_config(pipeline_config_path, recursive=args.recursive,
start_epoch_num=start_epoch_num)
elif args.mode == 'evaluate':
train_evaluate_model_from_config(pipeline_config_path, to_train=False, to_validate=False,
start_epoch_num=start_epoch_num)
elif args.mode == 'interact':
interact_model(pipeline_config_path)
elif args.mode == 'interactbot':
token = args.token
interact_model_by_telegram(pipeline_config_path, token)
elif args.mode == 'interactmsbot':
ms_id = args.ms_id
ms_secret = args.ms_secret
run_ms_bf_default_agent(model_config=pipeline_config_path,
app_id=ms_id,
app_secret=ms_secret,
multi_instance=multi_instance,
stateful=stateful,
port=args.port)
elif args.mode == 'alexa':
run_alexa_default_agent(model_config=pipeline_config_path,
multi_instance=multi_instance,
stateful=stateful,
port=args.port,
https=https,
ssl_key=ssl_key,
ssl_cert=ssl_cert)
elif args.mode == 'riseapi':
alice = args.api_mode == 'alice'
if alice:
start_alice_server(pipeline_config_path, https, ssl_key, ssl_cert, port=args.port)
else:
start_model_server(pipeline_config_path, https, ssl_key, ssl_cert, port=args.port)
elif args.mode == 'predict':
predict_on_stream(pipeline_config_path, args.batch_size, args.file_path)
elif args.mode == 'install':
install_from_config(pipeline_config_path)
elif args.mode == 'crossval':
if args.folds < 2:
log.error('Minimum number of Folds is 2')
else:
n_folds = args.folds
calc_cv_score(pipeline_config_path, n_folds=n_folds, is_loo=False)
def main():
args = parser.parse_args()
pipeline_config_path = find_config(args.config_path)
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu_id
log.info("use gpu id:" + args.gpu_id)
if args.download or args.mode == 'download':
deep_download(pipeline_config_path)
multi_instance = args.multi_instance
stateful = args.stateful
start_epoch_num = args.start_epoch_num
if args.mode == 'train':
train_evaluate_model_from_config(pipeline_config_path,
recursive=args.recursive,
start_epoch_num=start_epoch_num)
elif args.mode == 'evaluate':
train_evaluate_model_from_config(pipeline_config_path,
to_train=False,
to_validate=False,
start_epoch_num=start_epoch_num)
elif args.mode == 'interact':
interact_model(pipeline_config_path)
elif args.mode == 'interactbot':
token = args.token
interact_model_by_telegram(pipeline_config_path, token)
elif args.mode == 'interactmsbot':
ms_id = args.ms_id
ms_secret = args.ms_secret
run_ms_bf_default_agent(model_config=pipeline_config_path,
app_id=ms_id,
app_secret=ms_secret,
multi_instance=multi_instance,
stateful=stateful)
elif args.mode == 'riseapi':
alice = args.api_mode == 'alice'
https = args.https
ssl_key = args.key
ssl_cert = args.cert
if alice:
start_alice_server(pipeline_config_path, https, ssl_key, ssl_cert)
else:
start_model_server(pipeline_config_path, https, ssl_key, ssl_cert)
elif args.mode == 'predict':
predict_on_stream(pipeline_config_path, args.batch_size,
args.file_path)
elif args.mode == 'install':
install_from_config(pipeline_config_path)
elif args.mode == 'crossval':
if args.folds < 2:
log.error('Minimum number of Folds is 2')
else:
n_folds = args.folds
calc_cv_score(pipeline_config_path, n_folds=n_folds, is_loo=False)
def main():
args = parser.parse_args()
pipeline_config_path = find_config(args.config_path)
https = args.https
ssl_key = args.key
ssl_cert = args.cert
if args.download or args.mode == 'download':
deep_download(pipeline_config_path)
multi_instance = args.multi_instance
stateful = args.stateful
if args.mode == 'train':
train_evaluate_model_from_config(pipeline_config_path,
recursive=args.recursive,
start_epoch_num=args.start_epoch_num)
elif args.mode == 'evaluate':
train_evaluate_model_from_config(pipeline_config_path,
to_train=False,
start_epoch_num=args.start_epoch_num)
elif args.mode == 'interact':
interact_model(pipeline_config_path)
elif args.mode == 'interactbot':
token = args.token
interact_model_by_telegram(
model_config=pipeline_config_path,
token=token,
default_skill_wrap=not args.no_default_skill)
elif args.mode == 'interactmsbot':
ms_id = args.ms_id
ms_secret = args.ms_secret
run_ms_bf_default_agent(model_config=pipeline_config_path,
app_id=ms_id,
app_secret=ms_secret,
multi_instance=multi_instance,
stateful=stateful,
port=args.port,
https=https,
ssl_key=ssl_key,
ssl_cert=ssl_cert,
default_skill_wrap=not args.no_default_skill)
elif args.mode == 'alexa':
run_alexa_default_agent(model_config=pipeline_config_path,
multi_instance=multi_instance,
stateful=stateful,
port=args.port,
https=https,
ssl_key=ssl_key,
ssl_cert=ssl_cert,
default_skill_wrap=not args.no_default_skill)
elif args.mode == 'riseapi':
alice = args.api_mode == 'alice'
if alice:
start_alice_server(pipeline_config_path,
https,
ssl_key,
ssl_cert,
port=args.port)
else:
start_model_server(pipeline_config_path,
https,
ssl_key,
ssl_cert,
port=args.port)
elif args.mode == 'predict':
predict_on_stream(pipeline_config_path, args.batch_size,
args.file_path)
elif args.mode == 'install':
install_from_config(pipeline_config_path)
elif args.mode == 'crossval':
if args.folds < 2:
log.error('Minimum number of Folds is 2')
else:
n_folds = args.folds
calc_cv_score(pipeline_config_path, n_folds=n_folds, is_loo=False)
def main():
params_helper = ParamsSearch()
args = parser.parse_args()
is_loo = False
n_folds = None
if args.folds == 'loo':
is_loo = True
elif args.folds is None:
n_folds = None
elif args.folds.isdigit():
n_folds = int(args.folds)
else:
raise NotImplementedError('Not implemented this type of CV')
# read config
pipeline_config_path = find_config(args.config_path)
config_init = read_json(pipeline_config_path)
config = parse_config(config_init)
data = read_data_by_config(config)
target_metric = parse_config(config_init)['train']['metrics'][0]
if isinstance(target_metric, dict):
target_metric = target_metric['name']
# get all params for search
param_paths = list(params_helper.find_model_path(config, 'search_choice'))
param_values = []
param_names = []
for path in param_paths:
value = params_helper.get_value_from_config(config, path)
param_name = path[-1]
param_value_search = value['search_choice']
param_names.append(param_name)
param_values.append(param_value_search)
# find optimal params
if args.search_type == 'grid':
# generate params combnations for grid search
combinations = list(product(*param_values))
# calculate cv scores
scores = []
for comb in combinations:
config = deepcopy(config_init)
for param_path, param_value in zip(param_paths, comb):
params_helper.insert_value_or_dict_into_config(
config, param_path, param_value)
config = parse_config(config)
if (n_folds is not None) | is_loo:
# CV for model evaluation
score_dict = calc_cv_score(config,
data=data,
n_folds=n_folds,
is_loo=is_loo)
score = score_dict[next(iter(score_dict))]
else:
# train/valid for model evaluation
data_to_evaluate = data.copy()
if len(data_to_evaluate['valid']) == 0:
data_to_evaluate['train'], data_to_evaluate[
'valid'] = train_test_split(data_to_evaluate['train'],
test_size=0.2)
iterator = get_iterator_from_config(config, data_to_evaluate)
score = train_evaluate_model_from_config(
config, iterator=iterator)['valid'][target_metric]
scores.append(score)
# get model with best score
best_params_dict = get_best_params(combinations, scores, param_names,
target_metric)
log.info('Best model params: {}'.format(best_params_dict))
else:
raise NotImplementedError('Not implemented this type of search')
# save config
best_config = config_init
for i, param_name in enumerate(best_params_dict.keys()):
if param_name != target_metric:
params_helper.insert_value_or_dict_into_config(
best_config, param_paths[i], best_params_dict[param_name])
best_model_filename = pipeline_config_path.with_suffix('.cvbest.json')
save_json(best_config, best_model_filename)
log.info('Best model saved in json-file: {}'.format(best_model_filename))
def main():
args = parser.parse_args()
pipeline_config_path = find_config(args.config_path)
if args.download or args.mode == 'download':
deep_download(['-c', pipeline_config_path])
token = args.token or os.getenv('TELEGRAM_TOKEN')
ms_id = args.ms_id or os.getenv('MS_APP_ID')
ms_secret = args.ms_secret or os.getenv('MS_APP_SECRET')
multi_instance = args.multi_instance
stateful = args.stateful
if args.mode == 'train':
train_evaluate_model_from_config(pipeline_config_path)
elif args.mode == 'evaluate':
train_evaluate_model_from_config(pipeline_config_path,
to_train=False,
to_validate=False)
elif args.mode == 'interact':
interact_model(pipeline_config_path)
elif args.mode == 'interactbot':
if not token:
log.error(
'Token required: initiate -t param or TELEGRAM_BOT env var with Telegram bot token'
)
else:
interact_model_by_telegram(pipeline_config_path, token)
elif args.mode == 'interactmsbot':
if not ms_id:
log.error(
'Microsoft Bot Framework app id required: initiate -i param '
'or MS_APP_ID env var with Microsoft app id')
elif not ms_secret:
log.error(
'Microsoft Bot Framework app secret required: initiate -s param '
'or MS_APP_SECRET env var with Microsoft app secret')
else:
run_ms_bf_default_agent(model_config_path=pipeline_config_path,
app_id=ms_id,
app_secret=ms_secret,
multi_instance=multi_instance,
stateful=stateful)
elif args.mode == 'riseapi':
alice = args.api_mode == 'alice'
https = args.https
ssl_key = args.key
ssl_cert = args.cert
start_model_server(pipeline_config_path, alice, https, ssl_key,
ssl_cert)
elif args.mode == 'predict':
predict_on_stream(pipeline_config_path, args.batch_size,
args.file_path)
elif args.mode == 'install':
install_from_config(pipeline_config_path)
elif args.mode == 'crossval':
if args.folds < 2:
log.error('Minimum number of Folds is 2')
else:
n_folds = args.folds
calc_cv_score(pipeline_config_path=pipeline_config_path,
n_folds=n_folds,
is_loo=False)
def main():
args = parser.parse_args()
pipeline_config_path = find_config(args.config_path)
if args.download or args.mode == 'download':
deep_download(pipeline_config_path)
if args.mode == 'train':
train_evaluate_model_from_config(pipeline_config_path,
recursive=args.recursive,
start_epoch_num=args.start_epoch_num)
elif args.mode == 'evaluate':
train_evaluate_model_from_config(pipeline_config_path,
to_train=False,
start_epoch_num=args.start_epoch_num)
elif args.mode == 'interact':
interact_model(pipeline_config_path)
elif args.mode == 'telegram':
interact_model_by_telegram(model_config=pipeline_config_path,
token=args.token)
elif args.mode == 'msbot':
start_ms_bf_server(model_config=pipeline_config_path,
app_id=args.ms_id,
app_secret=args.ms_secret,
port=args.port,
https=args.https,
ssl_key=args.key,
ssl_cert=args.cert)
elif args.mode == 'alexa':
start_alexa_server(model_config=pipeline_config_path,
port=args.port,
https=args.https,
ssl_key=args.key,
ssl_cert=args.cert)
elif args.mode == 'alice':
start_alice_server(model_config=pipeline_config_path,
port=args.port,
https=args.https,
ssl_key=args.key,
ssl_cert=args.cert)
elif args.mode == 'riseapi':
start_model_server(pipeline_config_path,
args.https,
args.key,
args.cert,
port=args.port)
elif args.mode == 'risesocket':
start_socket_server(pipeline_config_path,
args.socket_type,
port=args.port,
socket_file=args.socket_file)
elif args.mode == 'agent-rabbit':
start_rabbit_service(model_config=pipeline_config_path,
service_name=args.service_name,
agent_namespace=args.agent_namespace,
batch_size=args.batch_size,
utterance_lifetime_sec=args.utterance_lifetime,
rabbit_host=args.rabbit_host,
rabbit_port=args.rabbit_port,
rabbit_login=args.rabbit_login,
rabbit_password=args.rabbit_password,
rabbit_virtualhost=args.rabbit_virtualhost)
elif args.mode == 'predict':
predict_on_stream(pipeline_config_path, args.batch_size,
args.file_path)
elif args.mode == 'install':
install_from_config(pipeline_config_path)
elif args.mode == 'crossval':
if args.folds < 2:
log.error('Minimum number of Folds is 2')
else:
calc_cv_score(pipeline_config_path,
n_folds=args.folds,
is_loo=False)
