def train(init_path=None, is_reverse_model=False):
processed_train_corpus_path = get_processed_corpus_path(TRAIN_CORPUS_NAME)
processed_val_corpus_path = get_processed_corpus_path(
CONTEXT_SENSITIVE_VAL_CORPUS_NAME)
index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME)
index_to_condition_path = get_index_to_condition_path(BASE_CORPUS_NAME)
# check the existence of all necessary files before compiling the model
_look_for_saved_files(files_paths=[
processed_train_corpus_path, processed_val_corpus_path,
index_to_token_path
])
index_to_token = load_index_to_item(index_to_token_path)
index_to_condition = load_index_to_item(index_to_condition_path)
w2v_matrix = _get_w2v_embedding_matrix_by_corpus_path(
processed_train_corpus_path, index_to_token)
# get nn_model and train it
nn_model_resolver_factory = S3FileResolver.init_resolver(
bucket_name=S3_MODELS_BUCKET_NAME, remote_dir=S3_NN_MODEL_REMOTE_DIR)
nn_model, _ = get_nn_model(index_to_token,
index_to_condition,
model_init_path=init_path,
w2v_matrix=w2v_matrix,
resolver_factory=nn_model_resolver_factory,
is_reverse_model=is_reverse_model)
train_model(nn_model)
def load_model():
index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME)
index_to_condition_path = get_index_to_condition_path(BASE_CORPUS_NAME)
model_path = get_model_full_path()
index_to_token = load_index_to_item(index_to_token_path)
index_to_condition = load_index_to_item(index_to_condition_path)
nn_model, model_exists = get_nn_model(index_to_token, index_to_condition, nn_model_path=model_path)
if not model_exists:
raise FileNotFoundException('Couldn\'t find model:\n"{}". \nExiting...'.format(model_path))
return nn_model
def load_model(model_path, tokens_index_path=None, conditions_index_path=None):
if tokens_index_path is None:
tokens_index_path = get_index_to_token_path(BASE_CORPUS_NAME)
if conditions_index_path is None:
conditions_index_path = get_index_to_condition_path(BASE_CORPUS_NAME)
index_to_token = load_index_to_item(tokens_index_path)
index_to_condition = load_index_to_item(conditions_index_path)
nn_model, model_exists = get_nn_model(index_to_token, index_to_condition, model_path)
if not model_exists:
raise ValueError('Couldn\'t find model: "{}".'.format(model_path))
return nn_model
def load_model(model_path=None, tokens_index_path=None, conditions_index_path=None):
if model_path is None:
model_path = get_model_full_path()
if tokens_index_path is None:
tokens_index_path = get_index_to_token_path(BASE_CORPUS_NAME)
if conditions_index_path is None:
conditions_index_path = get_index_to_condition_path(BASE_CORPUS_NAME)
index_to_token = load_index_to_item(tokens_index_path)
index_to_condition = load_index_to_item(conditions_index_path)
nn_model, model_exists = get_nn_model(index_to_token, index_to_condition, nn_model_path=model_path)
if not model_exists:
raise ValueError('Couldn\'t find model: "{}".'.format(model_path))
return nn_model
def load_model():
index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME)
index_to_condition_path = get_index_to_condition_path(BASE_CORPUS_NAME)
model_path = get_model_full_path()
index_to_token = load_index_to_item(index_to_token_path)
index_to_condition = load_index_to_item(index_to_condition_path)
nn_model, model_exists = get_nn_model(index_to_token,
index_to_condition,
nn_model_path=model_path)
if not model_exists:
raise FileNotFoundException(
'Couldn\'t find model:\n"{}". \nExiting...'.format(model_path))
return nn_model
def get_trained_model(reverse=False, fetch_from_s3=True):
if fetch_from_s3:
resolver_factory = S3FileResolver.init_resolver(
bucket_name=S3_MODELS_BUCKET_NAME, remote_dir=S3_NN_MODEL_REMOTE_DIR)
else:
resolver_factory = None
nn_model, model_exists = get_nn_model(
_get_index_to_token(fetch_from_s3),
_get_index_to_condition(fetch_from_s3),
resolver_factory=resolver_factory,
is_reverse_model=reverse)
if not model_exists:
raise Exception('Can\'t get the model. '
'Run tools/download_model.py first to get all required files or train it by yourself.')
return nn_model
def get_trained_model(reverse=False, fetch_from_s3=True):
if fetch_from_s3:
resolver_factory = S3FileResolver.init_resolver(
bucket_name=S3_MODELS_BUCKET_NAME,
remote_dir=S3_NN_MODEL_REMOTE_DIR)
else:
resolver_factory = None
nn_model, model_exists = get_nn_model(
index_to_token=_get_index_to_token(fetch_from_s3),
index_to_condition=_get_index_to_condition(fetch_from_s3),
resolver_factory=resolver_factory,
is_reverse_model=reverse)
if not model_exists:
raise FileNotFoundException(
'Can\'t get the pre-trained model. Run tools/download_model.py first '
'to get all required files or train it by yourself.')
return nn_model
def train(is_reverse_model=False):
processed_train_corpus_path = get_processed_corpus_path(TRAIN_CORPUS_NAME)
processed_val_corpus_path = get_processed_corpus_path(CONTEXT_SENSITIVE_VAL_CORPUS_NAME)
index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME)
index_to_condition_path = get_index_to_condition_path(BASE_CORPUS_NAME)
model_path = get_model_full_path(is_reverse_model)
# check the existence of all necessary files before compiling the model
_look_for_saved_files(files_paths=[processed_train_corpus_path, processed_val_corpus_path, index_to_token_path])
_look_for_saved_model(model_path)
index_to_token = load_index_to_item(index_to_token_path)
index_to_condition = load_index_to_item(index_to_condition_path)
w2v_matrix = _get_w2v_embedding_matrix_by_corpus_path(processed_train_corpus_path, index_to_token)
# get nn_model and train it
nn_model, _ = get_nn_model(index_to_token, index_to_condition, w2v_matrix)
train_model(nn_model, is_reverse_model=is_reverse_model)
def predict(model_path,
tokens_index_path=None,
conditions_index_path=None,
default_predictions_path=None,
reverse_model_weights=None,
temperatures=None,
prediction_mode=None):
if not tokens_index_path:
tokens_index_path = get_index_to_token_path(BASE_CORPUS_NAME)
if not conditions_index_path:
conditions_index_path = get_index_to_condition_path(BASE_CORPUS_NAME)
if not temperatures:
temperatures = [DEFAULT_TEMPERATURE]
if not prediction_mode:
prediction_mode = PREDICTION_MODE_FOR_TESTS
# Construct list of parameters values for all possible combinations of passed parameters
prediction_params = [dict()]
if reverse_model_weights:
prediction_params = [
dict(params, mmi_reverse_model_score_weight=w)
for params in prediction_params for w in reverse_model_weights
]
if temperatures:
prediction_params = [
dict(params, temperature=t) for params in prediction_params
for t in temperatures
]
if not is_non_empty_file(tokens_index_path):
_logger.warning(
'Couldn\'t find tokens_index file:\n{}. \nExiting...'.format(
tokens_index_path))
return
index_to_token = load_index_to_item(tokens_index_path)
index_to_condition = load_index_to_item(conditions_index_path)
nn_model, _ = get_nn_model(index_to_token,
index_to_condition,
model_init_path=model_path)
if not default_predictions_path:
default_predictions_path = os.path.join(
DATA_DIR, 'results', 'predictions_' + nn_model.model_name)
# Get path for each combination of parameters
predictions_paths = []
# Add suffix to the filename only for parameters that have a specific value passed as an argument
# If no parameters were specified, no suffix is added
if len(prediction_params) > 1:
for cur_params in prediction_params:
cur_path = '{base_path}_{params_str}.tsv'.format(
base_path=default_predictions_path,
params_str='_'.join(
['{}_{}'.format(k, v) for k, v in cur_params.items()]))
predictions_paths.append(cur_path)
else:
predictions_paths = [default_predictions_path + '.tsv']
_logger.info('Model for prediction:\n{}'.format(nn_model.model_load_path))
_logger.info('Tokens index:\n{}'.format(tokens_index_path))
_logger.info('File with questions:\n{}'.format(QUESTIONS_CORPUS_NAME))
_logger.info('Files to dump responses:\n{}'.format(
'\n'.join(predictions_paths)))
_logger.info('Prediction parameters\n{}'.format('\n'.join(
[str(x) for x in prediction_params])))
processed_test_set = get_tokenized_test_lines(QUESTIONS_CORPUS_NAME,
set(index_to_token.values()))
processed_test_set = list(processed_test_set)
for cur_params, cur_path in zip(prediction_params, predictions_paths):
_logger.info(
'Predicting with the following params: {}'.format(cur_params))
_save_test_results(processed_test_set,
cur_path,
nn_model,
prediction_modes=[prediction_mode])
def predict(model_path=None,
tokens_index_path=None,
conditions_index_path=None,
default_predictions_path=None,
reverse_model_weights=None,
temperatures=None,
prediction_mode=PREDICTION_MODE_FOR_TESTS):
if not model_path:
model_path = get_model_full_path()
if not tokens_index_path:
tokens_index_path = get_index_to_token_path(BASE_CORPUS_NAME)
if not conditions_index_path:
conditions_index_path = get_index_to_condition_path(BASE_CORPUS_NAME)
if not default_predictions_path:
default_predictions_path = os.path.join(DATA_DIR, 'results', 'predictions_' + get_model_full_params_str())
# Construct list of parameters values for all possible combinations of passed parameters
prediction_params = [dict()]
if reverse_model_weights:
prediction_params = [
dict(params, mmi_reverse_model_score_weight=w)
for params in prediction_params
for w in reverse_model_weights
]
if temperatures:
prediction_params = [dict(params, temperature=t) for params in prediction_params for t in temperatures]
# Get path for each combination of parameters
predictions_paths = []
# Add suffix to the filename only for parameters that have a specific value passed as an argument
# If no parameters were specified, no suffix is added
if len(prediction_params) > 1:
for cur_params in prediction_params:
cur_path = '{base_path}_{params_str}.tsv'.format(
base_path=default_predictions_path,
params_str='_'.join(['{}_{}'.format(k, v) for k, v in cur_params.items()]))
predictions_paths.append(cur_path)
else:
predictions_paths = [default_predictions_path + '.tsv']
if not is_non_empty_file(model_path):
_logger.warn('Couldn\'t find model:\n"{}". \nExiting...'.format(model_path))
return
if not is_non_empty_file(tokens_index_path):
_logger.warn('Couldn\'t find tokens_index file:\n"{}". \nExiting...'.format(tokens_index_path))
return
_logger.info('Model for prediction:\n{}'.format(model_path))
_logger.info('Tokens index:\n{}'.format(tokens_index_path))
_logger.info('File with questions:\n{}'.format(QUESTIONS_CORPUS_NAME))
_logger.info('Files to dump responses:\n{}'.format('\n'.join(predictions_paths)))
_logger.info('Prediction parameters\n{}'.format('\n'.join([str(x) for x in prediction_params])))
index_to_token = load_index_to_item(tokens_index_path)
index_to_condition = load_index_to_item(conditions_index_path)
processed_test_set = get_tokenized_test_lines(QUESTIONS_CORPUS_NAME, set(index_to_token.values()))
processed_test_set = list(processed_test_set)
nn_model, _ = get_nn_model(index_to_token, index_to_condition, nn_model_path=model_path)
for cur_params, cur_path in zip(prediction_params, predictions_paths):
_logger.info('Predicting with the following params: {}'.format(cur_params))
_save_test_results(processed_test_set, cur_path, nn_model, prediction_mode, **cur_params)