def get_nn_model(index_to_token,
index_to_condition,
model_init_path=None,
w2v_matrix=None,
resolver_factory=None,
is_reverse_model=False):
model = CakeChatModel(index_to_token,
index_to_condition,
model_init_path=model_init_path,
init_embedding=w2v_matrix,
is_reverse_model=is_reverse_model)
model.print_layer_shapes()
# try to initialise model with pre-trained weights
resolver = resolver_factory(
model.model_load_path) if resolver_factory else DummyFileResolver(
model.model_load_path)
model_exists = resolver.resolve()
if model_exists:
_logger.info('\nLoading weights from file:\n{}\n'.format(
model.model_load_path))
model.load_weights()
elif model_init_path:
raise FileNotFoundException(
'Can\'t initialize model from file:\n{}\n'.format(model_init_path))
else:
_logger.info('\nModel will be built with initial weights.\n')
model.print_matrices_weights()
_logger.info('\nModel is built\n')
return model, model_exists
def _get_index_to_token(fetch_from_s3):
index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME)
if fetch_from_s3:
tokens_idx_resolver = S3FileResolver(index_to_token_path,
S3_MODELS_BUCKET_NAME,
S3_TOKENS_IDX_REMOTE_DIR)
if not tokens_idx_resolver.resolve():
raise FileNotFoundException(
'Can\'t get index_to_token because file does not exist at S3')
else:
if not os.path.exists(index_to_token_path):
raise FileNotFoundException(
'Can\'t get index_to_token because file does not exist. '
'Run tools/download_model.py first to get all required files or construct it by yourself.'
)
return load_index_to_item(index_to_token_path)
def _get_index_to_condition(fetch_from_s3):
index_to_condition_path = get_index_to_condition_path(BASE_CORPUS_NAME)
if fetch_from_s3:
index_to_condition_resolver = S3FileResolver(
index_to_condition_path, S3_MODELS_BUCKET_NAME,
S3_CONDITIONS_IDX_REMOTE_DIR)
if not index_to_condition_resolver.resolve():
raise FileNotFoundException(
'Can\'t get index_to_condition because file does not exist on S3'
)
else:
if not os.path.exists(index_to_condition_path):
raise FileNotFoundException(
'Can\'t get index_to_condition because file does not exist. '
'Run tools/fetch.py first to get all required files or construct '
'it yourself.')
return load_index_to_item(index_to_condition_path)
def _get_index_to_token(fetch_from_s3):
index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME)
file_name = os.path.basename(index_to_token_path)
if fetch_from_s3:
tokens_idx_resolver = S3FileResolver(index_to_token_path,
S3_MODELS_BUCKET_NAME,
S3_TOKENS_IDX_REMOTE_DIR)
if not tokens_idx_resolver.resolve():
raise FileNotFoundException(
'No such file on S3: {}'.format(file_name))
else:
if not os.path.exists(index_to_token_path):
raise FileNotFoundException(
'No such file: {}'.format(file_name) +
'Run "python tools/fetch.py" first to get all necessary files.'
)
return load_index_to_item(index_to_token_path)
def get_trained_model(reverse=False, fetch_from_s3=True):
if fetch_from_s3:
resolver_factory = S3FileResolver.init_resolver(
bucket_name=S3_MODELS_BUCKET_NAME,
remote_dir=S3_NN_MODEL_REMOTE_DIR)
else:
resolver_factory = None
nn_model, model_exists = get_nn_model(
index_to_token=_get_index_to_token(fetch_from_s3),
index_to_condition=_get_index_to_condition(fetch_from_s3),
resolver_factory=resolver_factory,
is_reverse_model=reverse)
if not model_exists:
raise FileNotFoundException(
'Can\'t get the pre-trained model. Run tools/download_model.py first '
'to get all required files or train it by yourself.')
return nn_model
def _look_for_saved_files(files_paths):
for f_path in files_paths:
if not is_non_empty_file(f_path):
raise FileNotFoundException(
'\nCould not find the following file or it\'s empty: {0}'.
format(f_path))