def benchmark_reader(): reader_results = [] doc_store = get_document_store("elasticsearch") docs, labels = eval_data_from_file(data_dir / filename) index_to_doc_store(doc_store, docs, None, labels) for reader_name in reader_models: for reader_type in reader_types: try: reader = get_reader(reader_name, reader_type) results = reader.eval(document_store=doc_store, doc_index=doc_index, label_index=label_index, device="cuda") # print(results) results["passages_per_second"] = n_passages / results[ "reader_time"] results["reader"] = reader_name results["error"] = "" reader_results.append(results) except Exception as e: results = { 'EM': 0., 'f1': 0., 'top_n_accuracy': 0., 'top_n': 0, 'reader_time': 0., "passages_per_second": 0., "seconds_per_query": 0., 'reader': reader_name, "error": e } reader_results.append(results) reader_df = pd.DataFrame.from_records(reader_results) reader_df.to_csv("reader_results.csv")
def benchmark_reader(ci=False, update_json=False, save_markdown=False, **kwargs): if ci: reader_models = reader_models_ci max_docs = 100 # heuristic to estimate num of passages for the reduced num of docs n_passages = n_total_passages * (max_docs / n_total_docs) else: reader_models = reader_models_full max_docs = None n_passages = n_total_passages reader_results = [] doc_store = get_document_store("elasticsearch") # download squad data _download_extract_downstream_data(input_file=data_dir / filename) docs, labels = eval_data_from_file(data_dir / filename, max_docs) index_to_doc_store(doc_store, docs, None, labels) for reader_name in reader_models: for reader_type in reader_types: logger.info( f"##### Start reader run - model:{reader_name}, type: {reader_type} ##### " ) try: reader = get_reader(reader_name, reader_type) results = reader.eval(document_store=doc_store, doc_index=doc_index, label_index=label_index, device="cuda") # results = reader.eval_on_file(data_dir, filename, device="cuda") print(results) results["passages_per_second"] = n_passages / results[ "reader_time"] results["reader"] = reader_name results["error"] = "" reader_results.append(results) except Exception as e: results = { 'EM': 0., 'f1': 0., 'top_n_accuracy': 0., 'top_n': 0, 'reader_time': 0., "passages_per_second": 0., "seconds_per_query": 0., 'reader': reader_name, "error": e } reader_results.append(results) reader_df = pd.DataFrame.from_records(reader_results) reader_df.to_csv(results_file) if save_markdown: md_file = results_file.replace(".csv", ".md") with open(md_file, "w") as f: f.write(str(reader_df.to_markdown())) if update_json: populate_reader_json()
def train_and_eval_from_config(common_config, model_config, model_output_dir): # for reproducibility tf.set_random_seed(0) # Load Data from tfrecord # first get the reader based on the config: frame feature reader or # aggregated feature reader reader = get_reader(common_config['feature_names'], common_config['feature_sizes'], common_config['num_classes'], common_config['frame_features']) # get the input data tensors unused_video_id, model_input_raw, labels_batch, num_frames = ( get_input_training_tensors( reader, common_config['training_data_path'], batch_size=common_config['batch_size'], num_readers=common_config['num_readers'], num_epochs=common_config['num_epochs'] )) tf.summary.histogram("model/input_raw", model_input_raw) # create model model_name = next(iter(model_config)) train_model = ModelFactory().get_model(model_name).create_model(model_input_raw, common_config, model_config) # compile the model # Pass the target tensor `labels_batch` to train_model.compile # via the `target_tensors` keyword argument classifier = common_config['classifier'] if classifier == 'multiclass': train_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'], target_tensors=[labels_batch]) elif classifier == 'binary': train_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'], target_tensors=[labels_batch]) train_model.summary() # get the evaluation data tensors video_id_batch, eval_model_input_raw, eval_labels_batch, eval_num_frames = ( get_input_evaluation_tensors( reader, common_config['evaluation_data_path'], batch_size=common_config['batch_size'], num_readers=common_config['num_readers'])) tf.summary.histogram("eval/input_raw", eval_model_input_raw) # create a separate model for evaluation eval_model = ModelFactory().get_model(model_name).create_model(eval_model_input_raw, common_config, model_config) # compile the eval model # Pass the target tensor `eval_labels_batch` to eval_model.compile # via the `target_tensors` keyword argument if classifier == 'multiclass': eval_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'], target_tensors=[eval_labels_batch]) elif classifier == 'binary': eval_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'], target_tensors=[eval_labels_batch]) eval_model.summary() # get the session.. from keras import backend as K sess = K.get_session() sess.run(tf.local_variables_initializer()) sess.run(tf.global_variables_initializer()) # Fit the model using data from the TFRecord data tensors. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: train_model.fit(epochs=common_config['num_epochs'], steps_per_epoch=common_config['training_steps_per_epoch'], callbacks=[EvaluateInputTensor(eval_model, steps=common_config['eval_steps_per_epoch'])], verbose=2) except tf.errors.OutOfRangeError: logging.info("Done training -- epoch limit reached.") # save the model # 1. serialize model to JSON model_json = train_model.to_json() model_file_to_save = os.path.join(model_output_dir, OUTPUT_MODEL_FILENAME) with open(model_file_to_save, "w") as json_file: json_file.write(model_json) # 2. save the model weights weights_file_to_save = os.path.join(model_output_dir, OUTPUT_WEIGHTS_FILENAME) train_model.save_weights(weights_file_to_save) logging.info("Saved model and weights to " + model_output_dir) # Clean up the TF session. coord.request_stop() coord.join(threads) K.clear_session()
def test_from_saved_model(test_data_path, batch_size, num_readers, num_epochs, feature_names, feature_sizes, num_classes, frame_features, model_output_dir, saved_model_file, saved_weights_file): # get the session.. from keras import backend as K sess = K.get_session() # for reproducibility tf.set_random_seed(0) # Load Data from tfrecord # first get the reader based on the config: frame feature reader or # aggregated feature reader reader = utils.get_reader(feature_names, feature_sizes, num_classes, frame_features) # get input test tensors video_id_batch, test_model_input_raw, test_labels_batch, test_num_frames = ( get_input_test_tensors(reader, test_data_path, batch_size=batch_size, num_readers=num_readers)) tf.summary.histogram("test/model_input_raw", test_model_input_raw) feature_dim = len(test_model_input_raw.get_shape()) - 1 # Normalize input features. test_model_input_raw = tf.nn.l2_normalize(test_model_input_raw, feature_dim) # test_model_input_raw = tf.reshape(test_model_input_raw, shape=(batch_size, 1, 300, 128)) # TODO see if we need these 2 lines.. test_labels_batch = tf.cast(test_labels_batch, tf.float32) # load json and create model if model_output_dir.endswith('/'): model_file_to_load = model_output_dir + saved_model_file weights_file_to_load = model_output_dir + saved_weights_file else: model_file_to_load = model_output_dir + '/' + saved_model_file weights_file_to_load = model_output_dir + '/' + saved_weights_file json_file = open(model_file_to_load, 'r') loaded_model_json = json_file.read() json_file.close() test_model = model_from_json(loaded_model_json) # load weights into new model test_model.load_weights(weights_file_to_load) # compile the model test_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() sess.run(tf.local_variables_initializer()) # sess.run(tf.global_variables_initializer()) # evaluate the model using test data from the TFRecord data tensors. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) try: while not coord.should_stop(): loss, acc = test_model.evaluate(test_model_input_raw, test_labels_batch, steps=100, verbose=1) except tf.errors.OutOfRangeError: logging.info("Done training -- limit reached.") # Clean up the TF session. coord.request_stop() coord.join(threads) K.clear_session() print('\nTest accuracy: {0}'.format(acc))