def get_nn_model(index_to_token, index_to_condition, model_init_path=None, w2v_matrix=None, resolver_factory=None, is_reverse_model=False): model = CakeChatModel(index_to_token, index_to_condition, model_init_path=model_init_path, init_embedding=w2v_matrix, is_reverse_model=is_reverse_model) model.print_layer_shapes() # try to initialise model with pre-trained weights resolver = resolver_factory( model.model_load_path) if resolver_factory else DummyFileResolver( model.model_load_path) model_exists = resolver.resolve() if model_exists: _logger.info('\nLoading weights from file:\n{}\n'.format( model.model_load_path)) model.load_weights() elif model_init_path: raise FileNotFoundException( 'Can\'t initialize model from file:\n{}\n'.format(model_init_path)) else: _logger.info('\nModel will be built with initial weights.\n') model.print_matrices_weights() _logger.info('\nModel is built\n') return model, model_exists
def _get_index_to_token(fetch_from_s3): index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME) if fetch_from_s3: tokens_idx_resolver = S3FileResolver(index_to_token_path, S3_MODELS_BUCKET_NAME, S3_TOKENS_IDX_REMOTE_DIR) if not tokens_idx_resolver.resolve(): raise FileNotFoundException( 'Can\'t get index_to_token because file does not exist at S3') else: if not os.path.exists(index_to_token_path): raise FileNotFoundException( 'Can\'t get index_to_token because file does not exist. ' 'Run tools/download_model.py first to get all required files or construct it by yourself.' ) return load_index_to_item(index_to_token_path)
def _get_index_to_condition(fetch_from_s3): index_to_condition_path = get_index_to_condition_path(BASE_CORPUS_NAME) if fetch_from_s3: index_to_condition_resolver = S3FileResolver( index_to_condition_path, S3_MODELS_BUCKET_NAME, S3_CONDITIONS_IDX_REMOTE_DIR) if not index_to_condition_resolver.resolve(): raise FileNotFoundException( 'Can\'t get index_to_condition because file does not exist on S3' ) else: if not os.path.exists(index_to_condition_path): raise FileNotFoundException( 'Can\'t get index_to_condition because file does not exist. ' 'Run tools/fetch.py first to get all required files or construct ' 'it yourself.') return load_index_to_item(index_to_condition_path)
def _get_index_to_token(fetch_from_s3): index_to_token_path = get_index_to_token_path(BASE_CORPUS_NAME) file_name = os.path.basename(index_to_token_path) if fetch_from_s3: tokens_idx_resolver = S3FileResolver(index_to_token_path, S3_MODELS_BUCKET_NAME, S3_TOKENS_IDX_REMOTE_DIR) if not tokens_idx_resolver.resolve(): raise FileNotFoundException( 'No such file on S3: {}'.format(file_name)) else: if not os.path.exists(index_to_token_path): raise FileNotFoundException( 'No such file: {}'.format(file_name) + 'Run "python tools/fetch.py" first to get all necessary files.' ) return load_index_to_item(index_to_token_path)
def get_trained_model(reverse=False, fetch_from_s3=True): if fetch_from_s3: resolver_factory = S3FileResolver.init_resolver( bucket_name=S3_MODELS_BUCKET_NAME, remote_dir=S3_NN_MODEL_REMOTE_DIR) else: resolver_factory = None nn_model, model_exists = get_nn_model( index_to_token=_get_index_to_token(fetch_from_s3), index_to_condition=_get_index_to_condition(fetch_from_s3), resolver_factory=resolver_factory, is_reverse_model=reverse) if not model_exists: raise FileNotFoundException( 'Can\'t get the pre-trained model. Run tools/download_model.py first ' 'to get all required files or train it by yourself.') return nn_model
def _look_for_saved_files(files_paths): for f_path in files_paths: if not is_non_empty_file(f_path): raise FileNotFoundException( '\nCould not find the following file or it\'s empty: {0}'. format(f_path))