def test_semantic_types(self, data_set, test_sizes): logging.info("Testing semantic types.") rank_score_map = defaultdict(lambda: defaultdict(lambda: 0)) count_map = defaultdict(lambda: defaultdict(lambda: 0)) index_config = {'name': data_set} source_map = self.dataset_map[data_set] double_name_list = list(source_map.values()) * 2 file_write.write("Dataset: " + data_set + "\n") for size in test_sizes: start_time = time.time() for idx, source_name in enumerate(list(source_map.keys())): train_names = [ source.index_name for source in double_name_list[idx + 1:idx + size + 1] ] train_examples_map = searcher.search_types_data( index_config, train_names) source = source_map[source_name] for column in source.column_map.values(): if column.semantic_type: textual_train_map = searcher.search_similar_text_data( index_config, column.value_text, train_names) semantic_types = column.predict_type( train_examples_map, textual_train_map, self.random_forest) logging.debug( " semantic types: {}".format(semantic_types)) for threshold in [0.01]: found = False rank = 1 rank_score = 0 for prediction in semantic_types: if column.semantic_type in prediction[1]: if prediction[0] > threshold and prediction[ 0] != 0: rank_score = 1.0 / (rank) found = True break if prediction[0] != 0: rank += len(prediction[1]) if not found and semantic_types[0][0] < threshold: rank_score = 1 file_write.write(column.name + "\t" + column.semantic_type + "\t" + str(semantic_types) + "\n") file_write.write(str(rank_score) + "\n") rank_score_map[size][threshold] += rank_score count_map[size][threshold] += 1 running_time = time.time() - start_time for threshold in [0.01]: file_write.write("Size: " + str(size) + " F-measure: " + str(rank_score_map[size][threshold] * 1.0 / count_map[size][threshold]) + " Time: " + str(running_time) + " Count: " + str(count_map[size][threshold]) + "\n")
def predict_semantic_type_for_column(self, column): logging.info("Predicting semantic type for column: {}.".format(column)) if self.random_forest is None: logging.error("Prediction not possible. Model not trained.") raise Exception("Prediction not possible. Model not trained.") start_time = time.time() # source_name = "" # if column.source_name: # index_name = re.sub(not_allowed_chars, "", column.source_name) # source_name = column.source_name # index_config = {'name': index_name} # train_examples_map = searcher.search_types_data(index_config, []) # textual_train_map = searcher.search_similar_text_data(index_config, column.value_text, []) # else: # train_examples_map = searcher.search_types_data("", []) # textual_train_map = searcher.search_similar_text_data("", column.value_text, []) # # index_config = {'name': "train_data"} index_config = "" train_examples_map = searcher.search_types_data(index_config, []) textual_train_map = searcher.search_similar_text_data( index_config, column.value_text, []) logging.info("Train examples map size {}".format( len(train_examples_map))) cur_res = { 'source_name': column.source_name, 'column_name': column.name, 'correct_label': column.semantic_type, 'scores': [(1.0, 'fail')] } try: semantic_types = column.predict_type(train_examples_map, textual_train_map, self.random_forest) all_preds = [] for (score, labels) in semantic_types: all_preds += [(score, l) for l in labels] # normalize scores so that they sum up to 1 total = sum([element[0] for element in all_preds]) if total > 0: cur_res['scores'] = [(score / total, l) for score, l in all_preds] else: cur_res['scores'] = [(score, l) for score, l in all_preds] logging.info("Scores normalized") except Exception as e: logging.warning( "Could not get predictions for column {} due to {}".format( column.name, e)) cur_res['scores'] = [(1.0, 'fail')] running_time = time.time() - start_time return { "folder_name": "", "running_time": running_time, "predictions": [cur_res] }
def generate_train_data(self, train_sizes): self.logger.info("generate_train_data") train_data = [] for data_set in self.data_sets: print("data_set: ", data_set) train_data = [] index_config = {'name': data_set} source_map = self.dataset_map[data_set] double_name_list = source_map.values() * 2 for size in train_sizes: for idx, source_name in enumerate(source_map.keys()): train_names = [ source.index_name for source in double_name_list[idx + 1:idx + size + 1] ] train_examples_map = searcher.search_types_data( index_config, train_names) source = source_map[source_name] print("Source: ", source) for column in source.column_map.values(): print("COLUMN: ", column) if column.semantic_type: textual_train_map = searcher.search_similar_text_data( index_config, column.value_text, train_names) feature_vectors = column.generate_candidate_types( train_examples_map, textual_train_map, is_labeled=True) train_data += feature_vectors return train_data
def test_semantic_types(self, data_set, test_sizes): rank_score_map = defaultdict(lambda: defaultdict(lambda: 0)) count_map = defaultdict(lambda: defaultdict(lambda: 0)) index_config = {'name': data_set} source_map = self.dataset_map[data_set] double_name_list = source_map.values() * 2 file_write.write("Dataset: " + data_set + "\n") for size in test_sizes: start_time = time.time() for idx, source_name in enumerate(source_map.keys()): train_names = [source.index_name for source in double_name_list[idx + 1: idx + size + 1]] train_examples_map = searcher.search_types_data(index_config, train_names) source = source_map[source_name] for column in source.column_map.values(): if column.semantic_type: textual_train_map = searcher.search_similar_text_data(index_config, column.value_text, train_names) semantic_types = column.predict_type(train_examples_map, textual_train_map, self.random_forest) for threshold in [0.0]: found = False rank = 1 rank_score = 0 for prediction in semantic_types[:1]: if column.semantic_type in prediction[1]: if prediction[0] > threshold and prediction[0] != 0: rank_score = 1.0 / (rank) found = True break if prediction[0] != 0: rank += len(prediction[1]) if not found and semantic_types[0][0] < threshold: rank_score = 1 # file_write.write( # column.name + "\t" + column.semantic_type + "\t" + str(semantic_types) + "\n") file_write.write(str(rank_score) + "\n") rank_score_map[size][threshold] += rank_score count_map[size][threshold] += 1 running_time = time.time() - start_time for threshold in [0.0]: file_write.write( "Size: " + str(size) + " F-measure: " + str( rank_score_map[size][threshold] * 1.0 / count_map[size][threshold]) + " Time: " + str( running_time) + " Count: " + str(count_map[size][threshold]) + "\n")
def generate_train_data(self, train_sizes): train_data = [] for data_set in self.data_sets: train_data = [] index_config = {'name': data_set} source_map = self.dataset_map[data_set] double_name_list = source_map.values() * 2 for size in train_sizes: for idx, source_name in enumerate(source_map.keys()): train_names = [source.index_name for source in double_name_list[idx + 1: idx + size + 1]] train_examples_map = searcher.search_types_data(index_config, train_names) source = source_map[source_name] for column in source.column_map.values(): if column.semantic_type: textual_train_map = searcher.search_similar_text_data(index_config, column.value_text, train_names) feature_vectors = column.generate_candidate_types(train_examples_map, textual_train_map, is_labeled=True) train_data += feature_vectors return train_data
def test_semantic_types_from_2_sets(self, train_set, test_set): self.read_class_type_from_csv("data/datasets/%s/classes.csv" % test_set) print(self.file_class_map.keys()) rank_score_map = defaultdict(lambda: 0) count_map = defaultdict(lambda: 0) source_result_map = {} train_index_config = {'name': train_set} for idx, source_name in enumerate(self.dataset_map[test_set]): if source_name not in self.file_class_map: continue train_examples_map = searcher.search_types_data( train_index_config, [self.file_class_map[source_name]]) source = self.dataset_map[test_set][source_name] column_result_map = {} for column in source.column_map.values(): if not column.semantic_type or not column.value_list or "ontology" not in column.semantic_type: continue textual_train_map = searcher.search_similar_text_data( train_index_config, column.value_text, [self.file_class_map[source_name]]) semantic_types = column.predict_type(train_examples_map, textual_train_map, self.random_forest) print(column.name) file_write.write(column.name + "\t" + column.semantic_type + "\t" + str(semantic_types) + "\n") for threshold in [0.1, 0.15, 0.2, 0.25, 0.5]: rank = 0 found = False rank_score = 0 for prediction in semantic_types: if column.semantic_type in prediction[1]: if prediction[0][1] >= threshold: rank_score = 1.0 / (rank + 1) found = True if not found and prediction[0][0] != 0: rank += len(prediction[1]) if not found: if semantic_types[0][0][1] < threshold: rank_score = 1 file_write.write(str(rank_score) + "\n") rank_score_map[threshold] += rank_score count_map[threshold] += 1 source_result_map[source_name] = column_result_map for threshold in [0.1, 0.15, 0.2, 0.25, 0.5]: file_write.write(" MRR: " + str(rank_score_map[threshold] * 1.0 / count_map[threshold]) + " Count: " + str(count_map[threshold]) + "\n") return source_result_map
def predict_folder_semantic_types(self, folder_name): """ Predict semantic types for all sources in folder :param folder_name: :return: """ logging.info( "Predicting semantic types for folder: {}.".format(folder_name)) if self.random_forest is None: logging.error("Prediction not possible. Model not trained.") raise Exception("Prediction not possible. Model not trained.") if folder_name not in self.dataset_map: logging.error( "Prediction not possible: folder is not indexed by semantic labeler." ) raise Exception( "Prediction not possible: folder is not indexed by semantic labeler." ) result = [] source_map = self.dataset_map[folder_name] start_time = time.time() for source in source_map.values(): # we need to index the source index_config = {'name': source.index_name} source.save(index_config) for column in source.column_map.values(): cur_res = { 'source_name': source.name, 'column_name': column.name, 'correct_label': column.semantic_type, 'scores': [] } train_examples_map = searcher.search_types_data( index_config, []) textual_train_map = searcher.search_similar_text_data( index_config, column.value_text, []) try: semantic_types = column.predict_type( train_examples_map, textual_train_map, self.random_forest) logging.info( "Column <{}> predicted semantic types {}".format( column.name, semantic_types)) all_preds = [] for (score, labels) in semantic_types: all_preds += [(score, l) for l in labels] # normalize scores so that they sum up to 1 total = sum([element[0] for element in all_preds]) if total > 0: cur_res['scores'] = [(score / total, l) for score, l in all_preds] else: cur_res['scores'] = [(score, l) for score, l in all_preds] logging.info("Scores normalized") except Exception as e: logging.warning( "Could not get predictions for column {} due to {}". format(column.name, e)) cur_res['scores'] = [(1.0, 'fail')] result.append(cur_res) running_time = time.time() - start_time return { "folder_name": folder_name, "running_time": running_time, "predictions": result }
def test_semantic_types_from_2_sets(self, train_set, test_set): # self.read_class_type_from_csv("data/datasets/%s/classes.csv" % test_set) # print self.file_class_map.keys() rank_score_map = defaultdict(lambda: 0) count_map = defaultdict(lambda: 0) source_result_map = {} train_index_config = {'name': train_set} train_names = [source.index_name for source in self.dataset_map[train_set].values()] for idx, source_name in enumerate(self.dataset_map[test_set]): # if source_name not in self.file_class_map: # continue train_examples_map = searcher.search_types_data(train_index_config, train_names) source = self.dataset_map[test_set][source_name] self.logger.info("Test source: %s", source_name) column_result_map = {} for column in source.column_map.values(): # if not column.semantic_type or not column.value_list or "ontology" not in column.semantic_type: # continue if not column.semantic_type or not column.value_list: continue textual_train_map = searcher.search_similar_text_data(train_index_config, column.value_text, train_names) semantic_types = column.predict_type(train_examples_map, textual_train_map, self.random_forest) column_result_map[column.name] = semantic_types self.logger.info(" -> column: %s", column.name) file_write.write( column.name + "\t" + column.semantic_type + "\t" + str(semantic_types) + "\n") for threshold in [0.0, 0.1, 0.15, 0.2, 0.25, 0.5]: found = False rank = 1 rank_score = 0 for prediction in semantic_types[:1]: if column.semantic_type in prediction[1]: if prediction[0] > threshold and prediction[0] != 0: rank_score = 1.0 / rank found = True break if prediction[0] != 0: rank += len(prediction[1]) if not found and semantic_types[0][0] < threshold: rank_score = 1 file_write.write(str(rank_score) + "\n") rank_score_map[threshold] += rank_score count_map[threshold] += 1 source_result_map[source_name] = column_result_map for threshold in [0.0, 0.1, 0.15, 0.2, 0.25, 0.5]: file_write.write( " MRR: " + str( rank_score_map[threshold] * 1.0 / count_map[threshold]) + " Count: " + str( count_map[threshold]) + " threshold=" + str(threshold) + "\n") return source_result_map
def predict_semantic_type_for_column(self, column): train_examples_map = searcher.search_types_data("index_name", []) textual_train_map = searcher.search_similar_text_data("index_name", column.value_text, []) return column.predict_type(train_examples_map, textual_train_map, self.random_forest)
def test_semantic_types_from_2_sets(self, train_set, test_set): # self.read_class_type_from_csv("data/datasets/%s/classes.csv" % test_set) # print self.file_class_map.keys() rank_score_map = defaultdict(lambda: 0) count_map = defaultdict(lambda: 0) source_result_map = {} train_index_config = {'name': train_set} train_names = [ source.index_name for source in self.dataset_map[train_set].values() ] self.logger.info("Train source: %s", train_names) valid = True for idx, source_name in enumerate(self.dataset_map[test_set]): # if source_name not in self.file_class_map: # continue train_examples_map = searcher.search_types_data( train_index_config, train_names) source = self.dataset_map[test_set][source_name] self.logger.info("Test source: %s", source_name) column_result_map = {} for column in source.column_map.values(): # if not column.semantic_type or not column.value_list or "ontology" not in column.semantic_type: # continue if not column.semantic_type or not column.value_list: continue textual_train_map = searcher.search_similar_text_data( train_index_config, column.value_text, train_names) # self.logger.info(textual_train_map) try: semantic_types = column.predict_type( train_examples_map, textual_train_map, self.random_forest) except KeyError: print("KEY ERROR") valid = False break # if(not semantic_types): # self.logger.info("Could not do "+column.name) # continue column_result_map[column.name] = semantic_types self.logger.info(" -> column: %s", column.name) file_write.write(column.name + "\t" + column.semantic_type + "\t" + str(semantic_types) + "\n") for threshold in [0.0, 0.1, 0.15, 0.2, 0.25, 0.5]: found = False rank = 1 rank_score = 0 for prediction in semantic_types[:1]: if column.semantic_type in prediction[1]: if prediction[0] > threshold and prediction[0] != 0: rank_score = 1.0 / rank found = True break if prediction[0] != 0: rank += len(prediction[1]) if not found and semantic_types[0][0] < threshold: rank_score = 1 file_write.write(str(rank_score) + "\n") rank_score_map[threshold] += rank_score count_map[threshold] += 1 source_result_map[source_name] = column_result_map # for threshold in [0.0, 0.1, 0.15, 0.2, 0.25, 0.5]: # file_write.write( # " MRR: " + str( # rank_score_map[threshold] * 1.0 / count_map[threshold]) + " Count: " + str( # count_map[threshold]) + " threshold=" + str(threshold) + "\n") return source_result_map
def predict_semantic_type_for_column(self, column): train_examples_map = searcher.search_types_data("index_name", []) textual_train_map = searcher.search_similar_text_data( "index_name", column.value_text, []) return column.predict_type(train_examples_map, textual_train_map, self.random_forest)