def test_add_and_get_gold_labels_for_category_no_dump(self): num_elements_to_labels = 10 dataset_name = self.test_add_and_get_gold_labels_for_category_no_dump.__name__ target_category = 'Autobots' uris_to_gold_labels_expected = generate_random_uris_and_labels( dataset_name, num_elements_to_labels, [target_category]) non_target_category = 'Decepticons' uris_to_gold_labels = [(uri, dict( labels, **{ non_target_category: Label(labels=LABEL_POSITIVE, metadata={}) })) for uri, labels in uris_to_gold_labels_expected] oracle.add_gold_labels(dataset_name, uris_to_gold_labels) uris_to_retrieve = [ uri for uri, labels_dict in uris_to_gold_labels_expected ] uri_to_gold_labels_found = oracle.get_gold_labels( dataset_name, uris_to_retrieve, target_category) self.assert_uri_labels_equal(uris_to_gold_labels_expected, uri_to_gold_labels_found) loader.clear_gold_labels_file(dataset_name)
def test_add_and_get_gold_labels_from_dump(self): num_elements_to_labels = 10 # generate gold labels for the first dataset dataset_name_first = self.test_add_and_get_gold_labels_from_dump.__name__ + '_a' uris_to_gold_labels_expected_first = generate_random_uris_and_labels( dataset_name_first, num_elements_to_labels, ['Autobots', 'Decepticons']) oracle.add_gold_labels(dataset_name_first, uris_to_gold_labels_expected_first) # generate gold labels for the second dataset (the first dataset is kept in a dump file) dataset_name_second = self.test_add_and_get_gold_labels_from_dump.__name__ + '_b' uris_to_gold_labels_expected_second = generate_random_uris_and_labels( dataset_name_second, num_elements_to_labels, ['Autobots', 'Decepticons']) oracle.add_gold_labels(dataset_name_second, uris_to_gold_labels_expected_second) # switch back to the first dataset and retrieve data from dump file uris_to_retrieve = [ uri for uri, labels_dict in uris_to_gold_labels_expected_first ] uri_to_gold_labels_found = oracle.get_gold_labels( dataset_name_first, uris_to_retrieve) self.assert_uri_labels_equal(uris_to_gold_labels_expected_first, uri_to_gold_labels_found) # clean both dataset dumps loader.clear_gold_labels_file(dataset_name_first) loader.clear_gold_labels_file(dataset_name_second)
def test_get_gold_labels_no_dump_and_no_in_memory(self): dataset_name = self.test_add_and_get_gold_labels_no_dump.__name__ uris_to_retrieve = ['uri1', 'uri2'] uris_to_gold_labels_expected = [] uri_to_gold_labels_found = oracle.get_gold_labels( dataset_name, uris_to_retrieve) self.assert_uri_labels_equal(uris_to_gold_labels_expected, uri_to_gold_labels_found) loader.clear_gold_labels_file(dataset_name)
def train_first_model(self, config: ExperimentParams): if orchestrator_api.workspace_exists(config.workspace_id): orchestrator_api.delete_workspace(config.workspace_id) orchestrator_api.create_workspace( config.workspace_id, config.train_dataset_name, dev_dataset_name=config.dev_dataset_name) orchestrator_api.create_new_category(config.workspace_id, config.category_name, "No description for you") dev_text_elements_uris = orchestrator_api.get_all_text_elements_uris( config.dev_dataset_name) dev_text_elements_and_labels = oracle_data_access_api.get_gold_labels( config.dev_dataset_name, dev_text_elements_uris) if dev_text_elements_and_labels is not None: orchestrator_api.set_labels(config.workspace_id, dev_text_elements_and_labels) random_seed = sum([ord(c) for c in config.workspace_id]) logging.info(str(config)) logging.info(f'random seed: {random_seed}') self.set_first_model_positives(config, random_seed) self.set_first_model_negatives(config, random_seed) # train first model logging.info( f'Starting first model training (model: {config.model.name})\tworkspace: {config.workspace_id}' ) new_model_id = orchestrator_api.train(config.workspace_id, config.category_name, config.model, train_params=config.train_params) if new_model_id is None: raise Exception( f'a new model was not trained\tworkspace: {config.workspace_id}' ) eval_dataset = config.test_dataset_name res_dict = self.evaluate(config, al=self.NO_AL, iteration=0, eval_dataset=eval_dataset) res_dict.update(self.generate_al_batch_dict( config)) # ensures AL-related keys are in the results dictionary logging.info( f'Evaluation on dataset: {eval_dataset}, iteration: 0, first model (id: {new_model_id}) ' f'repeat: {config.repeat_id}, is: {res_dict}\t' f'workspace: {config.workspace_id}') return res_dict
def set_first_model_positives(self, config, random_seed) -> List[TextElement]: """ Choose instances by queries, regardless of their gold label. :param config: experiment config for this run :param random_seed: a seed for the Random being used for sampling :return: a list of TextElements """ general_dataset_name = config.train_dataset_name.split('_train')[0] queries = self.queries_per_dataset[general_dataset_name][ config.category_name] sampled_unlabeled_text_elements = [] for query in queries: sampled_unlabeled_text_elements.extend( self.data_access.sample_unlabeled_text_elements( workspace_id=config.workspace_id, dataset_name=config.train_dataset_name, category_name=config.category_name, sample_size=self.first_model_positives_num, query=query, remove_duplicates=True)['results']) logging.info( f"Positive sampling, after query {query} size is {len(sampled_unlabeled_text_elements)} " ) if len(sampled_unlabeled_text_elements ) > self.first_model_positives_num: random.seed(random_seed) sampled_unlabeled_text_elements = random.sample( sampled_unlabeled_text_elements, self.first_model_positives_num) sampled_uris = [t.uri for t in sampled_unlabeled_text_elements] sampled_uris_and_gold_labels = dict( oracle_data_access_api.get_gold_labels(config.train_dataset_name, sampled_uris)) sampled_uris_and_label = \ [(x.uri, {config.category_name: sampled_uris_and_gold_labels[x.uri][config.category_name]}) for x in sampled_unlabeled_text_elements] orchestrator_api.set_labels(config.workspace_id, sampled_uris_and_label) logging.info( f'Set the label of {len(sampled_uris_and_label)} instances sampled by queries {queries} ' f'using the oracle for category {config.category_name}') logging.info( f"Positive sampling, returned {len(sampled_uris)} elements") return sampled_uris
def generate_performance_metrics_dict(config, evaluation_dataset): all_text_elements = orchestrator_api.get_all_text_elements( evaluation_dataset) all_text_elements_uris = [elem.uri for elem in all_text_elements] predicted_labels = orchestrator_api.infer(config.workspace_id, config.category_name, all_text_elements) uris_and_gold_labels = oracle_data_access_api.get_gold_labels( evaluation_dataset, all_text_elements_uris) gold_of_category = [ x[1][config.category_name].labels for x in uris_and_gold_labels ] performance_metrics = evaluate_predictions(gold_of_category, predicted_labels) return performance_metrics
def test_add_and_get_gold_labels_no_dump(self): num_elements_to_labels = 10 dataset_name = self.test_add_and_get_gold_labels_no_dump.__name__ uris_to_gold_labels_expected = generate_random_uris_and_labels( dataset_name, num_elements_to_labels, ['Autobots', 'Decepticons']) oracle.add_gold_labels(dataset_name, uris_to_gold_labels_expected) uris_to_retrieve = [ uri for uri, labels_dict in uris_to_gold_labels_expected ] uri_to_gold_labels_found = oracle.get_gold_labels( dataset_name, uris_to_retrieve) self.assert_uri_labels_equal(uris_to_gold_labels_expected, uri_to_gold_labels_found) loader.clear_gold_labels_file(dataset_name)
def get_suggested_elements_and_gold_labels(self, config, al): start = time.time() suggested_text_elements_for_labeling = \ orchestrator_api.get_elements_to_label(config.workspace_id, config.category_name, self.active_learning_suggestions_num) end = time.time() logging.info( f'{len(suggested_text_elements_for_labeling)} instances ' f'suggested by active learning strategy: {al.name} ' f'for dataset: {config.train_dataset_name} and category: {config.category_name}.\t' f'runtime: {end - start}\tworkspace: {config.workspace_id}') uris_for_labeling = [ elem.uri for elem in suggested_text_elements_for_labeling ] uris_and_gold_labels = oracle_data_access_api.get_gold_labels( config.train_dataset_name, uris_for_labeling, config.category_name) return suggested_text_elements_for_labeling, uris_and_gold_labels
def set_first_model_positives(self, config, random_seed) -> List[TextElement]: """ Randomly choose instances, regardless of their gold label. :param config: experiment config for this run :param random_seed: a seed for the Random being used for sampling :return: a list of TextElements """ sample_size = self.first_model_positives_num sampled_elements = self.data_access.sample_text_elements(config.train_dataset_name, sample_size, remove_duplicates=True)['results'] sampled_uris = [element.uri for element in sampled_elements] sampled_uris_with_labels = \ oracle_data_access_api.get_gold_labels(config.train_dataset_name, sampled_uris, config.category_name) orchestrator_api.set_labels(config.workspace_id, sampled_uris_with_labels) logging.info(f'set the label of {len(sampled_uris_with_labels)} random instances with their gold label ' f'(can be positive or negative) for category {config.category_name}') return sampled_uris
def test_add_and_get_some_gold_labels_no_dump(self): num_elements_to_labels = 10 sample_ratio = 0.4 dataset_name = self.test_add_and_get_some_gold_labels_no_dump.__name__ uris_to_gold_labels_expected = generate_random_uris_and_labels( dataset_name, num_elements_to_labels, ['Autobots', 'Decepticons']) oracle.add_gold_labels(dataset_name, uris_to_gold_labels_expected) uris_to_retrieve = [ uri for uri, labels_dict in uris_to_gold_labels_expected ] indices = random.sample(range(len(uris_to_retrieve)), floor(num_elements_to_labels * sample_ratio)) uris_to_retrieve = [uris_to_retrieve[i] for i in sorted(indices)] uris_to_gold_labels_expected = [ uri_label for uri_label in uris_to_gold_labels_expected if uri_label[0] in uris_to_retrieve ] uri_to_gold_labels_found = oracle.get_gold_labels( dataset_name, uris_to_retrieve) self.assert_uri_labels_equal(uris_to_gold_labels_expected, uri_to_gold_labels_found) loader.clear_gold_labels_file(dataset_name)
POSITIVE_SAMPLE_SIZE = 10 ** 6 MIN_TERMS_NUM = 3 if __name__ == '__main__': datasets_categories_queries = {'trec': {'LOC': ['Where|countr.*|cit.*']}} data_access = data_access_factory.get_data_access() dataset_part = DatasetPart.DEV dataset_part = dataset_part.name.lower() results_df = pd.DataFrame(columns=['dataset', 'category', 'query', 'hit_count']) for dataset in datasets_categories_queries: dataset_for_query = dataset + '_' + dataset_part uris = data_access.get_all_text_elements_uris(dataset_for_query) gold_labels = oracle_data_access_api.get_gold_labels(dataset_name=dataset_for_query, text_element_uris=uris) gold_labels = dict(gold_labels) for category in datasets_categories_queries[dataset]: for query in datasets_categories_queries[dataset][category]: search_res = data_access.sample_text_elements(dataset_name=dataset_for_query, sample_size=POSITIVE_SAMPLE_SIZE, query=query) query_results = search_res['results'] hit_count = search_res['hit_count'] # # filter too short texts # query_results = [e for e in query_results if len(e.text.split()) >= MIN_TERMS_NUM] logging.info(f'dataset: {dataset_for_query}\tcategory: {category}\t' f'query: {query}\thit_count: {hit_count}') logging.debug('\n\t' + '\n\t'.join([t.text for t in query_results]))