def test_mention_context_batch_sampler_many_last_ids(): mentions_by_page = { 0: [120], 1: [130], 2: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110] } cursor = get_mock_cursor(mentions_by_page) batch_size = 5 page_id_order = [2, 0, 1] mentions_in_page_order = _.mapcat( page_id_order, lambda page_id: mentions_by_page[page_id]) batch_sampler = MentionContextBatchSampler(cursor, page_id_order, batch_size) batches_seen = [] indexes_seen = [] for batch_num, batch_indexes in enumerate(batch_sampler): assert _.is_empty(_.intersection(batch_indexes, indexes_seen)) indexes_seen.extend(batch_indexes) if batch_num == 0: assert _.is_empty(_.difference(batch_indexes, mentions_by_page[2])) assert len(batch_indexes) == batch_size elif batch_num == 1: assert _.is_empty(_.difference(batch_indexes, mentions_by_page[2])) assert len(batch_indexes) == batch_size elif batch_num == 2: assert len(_.intersection(batch_indexes, mentions_by_page[2])) == 2 assert len(_.intersection(batch_indexes, mentions_by_page[0])) == 1 assert len(_.intersection(batch_indexes, mentions_by_page[1])) == 1 assert len(batch_indexes) == 4 batches_seen.append(batch_num) assert _.is_empty(_.difference(mentions_in_page_order, indexes_seen)) assert batches_seen == [0, 1, 2]
def jaccard_similarity(f1, f2): # print(union(f1,f2),"\n") # print(len(union(f1,f2)),"\n\n") div_by = len(union(f1, f2)) if div_by == 0: return 0 return len(intersection(f1, f2)) / div_by
def print_intersection_detail(title, f1, f2): #Retorna nĂºmeros print('\n') print(pad(title, 50, '-')) print('\n', f1, '\n\n', f2, '\n') print(intersection(f1, f2)) print('\n')
def test_mention_context_batch_sampler(): mentions_by_page = { 0: [40, 50, 60, 70, 80, 90], 1: [100], 2: [0, 10, 20, 30] } cursor = get_mock_cursor(mentions_by_page) batch_size = 5 page_id_order = [2, 0, 1] mentions_in_page_order = _.mapcat( page_id_order, lambda page_id: mentions_by_page[page_id]) batch_sampler = MentionContextBatchSampler(cursor, page_id_order, batch_size) batches_seen = [] indexes_seen = [] for batch_num, batch_indexes in enumerate(batch_sampler): assert _.is_empty(_.intersection(batch_indexes, indexes_seen)) indexes_seen.extend(batch_indexes) if batch_num == 0: assert len(set(batch_indexes) - {0, 10, 20, 30}) == 1 assert any([ mention in set(batch_indexes) - {0, 10, 20, 30} for mention in mentions_by_page[0] ]) assert len(batch_indexes) == batch_size elif batch_num == 1: assert _.is_empty(set(batch_indexes) - set(mentions_by_page[0])) assert len(batch_indexes) == batch_size elif batch_num == 2: assert batch_indexes == [100] assert len(batch_indexes) == 1 batches_seen.append(batch_num) assert _.is_empty(_.difference(mentions_in_page_order, indexes_seen)) assert batches_seen == [0, 1, 2]
def overlap_similarity(f1, f2): # print(len(intersection(f1,f2))) # print(min_([len(f1),len(f2)])) # print(len(intersection(f1,f2))/min_([len(f1),len(f2)])) div_by = min_([len(f1), len(f2)]) if div_by == 0: return 0 return len(intersection(f1, f2)) / div_by
def calculate_ship_extension(ship: Ship, candidates: List[Coord]) -> List[Coord]: if len(ship) == 1: return find_adjacent_cells(ship.cells[0], candidates, only_orthogonal=True) increment = ship.cells[1] - ship.cells[0] first, last = ship.cells[0], ship.cells[-1] if increment.j == 0: # vertical ext = [first + Coord((-1, 0)), last + Coord((1, 0))] else: # horizontal ext = [first + Coord((0, -1)), last + Coord((0, 1))] return py_.intersection(ext, candidates)
from pydash import uniq_by, mapcat, intersection import sys import requests wsurl = "https://s3.amazonaws.com/challenge.getcrossbeam.com/public/" rs1 = requests.get(wsurl + sys.argv[1] + ".json") data1 = rs1.json() names1 = uniq_by(data1['companies'], lambda r: r['domain']) n1Ar = mapcat(names1, lambda n: n['domain']) rs2 = requests.get(wsurl + sys.argv[2] + ".json") data2 = rs2.json() names2 = uniq_by(data2['companies'], lambda r: r['domain']) n2Ar = mapcat(names2, lambda n: n['domain']) names3 = intersection(n1Ar, n2Ar) print(repr(len(names1)) + " " + repr(len(names2)) + " " + repr(len(names3)))
def test_mention_context_dataset(): cursor = get_mock_cursor() page_id_order = [3, 1, 2] batch_size = 5 entity_candidates_prior = {'aa': {1: 20}, 'bb': {0: 10, 1: 2}, 'cc': {2: 3}} entity_label_lookup = dict(zip(range(5), range(5))) embedding_dim = 1 embedding_dict = {'<PAD>': torch.tensor([0]), '<UNK>': torch.tensor([2]), '<MENTION_START_HERE>': torch.tensor([-1]), '<MENTION_END_HERE>': torch.tensor([-2])} token_idx_lookup = dict(zip(embedding_dict.keys(), range(len(embedding_dict)))) embedding = nn.Embedding.from_pretrained(torch.stack([embedding_dict[token] for token in token_idx_lookup])) num_entities = 5 num_candidates = 2 dataset = MentionContextDataset(cursor, page_id_order, entity_candidates_prior, entity_label_lookup, embedding, token_idx_lookup, batch_size, num_entities, num_candidates) dataset._mention_infos = {0: {'mention': 'bb', 'offset': 9, 'page_id': 2, 'entity_id': 0, 'mention_id': 0}, 1: {'mention': 'aa', 'offset': 6, 'page_id': 2, 'entity_id': 1, 'mention_id': 1}, 2: {'mention': 'cc', 'offset': 0, 'page_id': 1, 'entity_id': 2, 'mention_id': 2}, 3: {'mention': 'bb', 'offset': 3, 'page_id': 1, 'entity_id': 0, 'mention_id': 3}, 4: {'mention': 'bb', 'offset': 3, 'page_id': 0, 'entity_id': 1, 'mention_id': 4}} dataset._page_content_lookup = {2: 'a b c aa bb', 1: 'cc bb c b a', 0: 'dd bb a b c'} dataset._sentence_spans_lookup = {2: [(0, 5), (6, 11)], 1: [(0, 5), (6, 11)], 0: [(0, 5), (6, 11)]} dataset._embedded_page_content_lookup = {2: [0, 1], 1: [1, 2], 0: [1]} dataset._entity_page_mentions_lookup = {2: [[0]], 1: [[1]], 0: [[1]]} dataset._mentions_per_page_ctr = {2: 2, 1: 2, 0: 1} expected_data = [{'sentence_splits': [['aa', 'bb'], ['bb']], 'label': 0, 'embedded_page_content': [0, 1], 'entity_page_mentions': [[0]], 'candidate_ids': torch.tensor([0, 1]), 'p_prior': torch.tensor([10/12, 2/12])}, {'sentence_splits': [['aa'], ['aa', 'bb']], 'label': 1, 'embedded_page_content': [0, 1], 'entity_page_mentions': [[0]], 'candidate_ids': torch.tensor([1]), 'p_prior': torch.tensor([1.0])}, {'sentence_splits': [['cc'], ['cc', 'bb']], 'label': 2, 'embedded_page_content': [1, 2], 'entity_page_mentions': [[1]], 'candidate_ids': torch.tensor([2]), 'p_prior': torch.tensor([1.0])}, {'sentence_splits': [['cc', 'bb'], ['bb']], 'label': 0, 'embedded_page_content': [1, 2], 'entity_page_mentions': [[1]], 'candidate_ids': torch.tensor([0, 1]), 'p_prior': torch.tensor([10/12, 2/12])}, {'sentence_splits': [['dd', 'bb'], ['bb']], 'label': 1, 'embedded_page_content': [1], 'entity_page_mentions': [[1]], 'candidate_ids': torch.tensor([0, 1]), 'p_prior': torch.tensor([10/12, 2/12])}] iterator = iter(dataset) dataset_values = [next(iterator) for _ in range(len(expected_data))] comparison = {'sentence_splits': _.is_equal, 'label': _.is_equal, 'embedded_page_content': _.is_equal, 'entity_page_mentions': _.is_equal, 'candidate_ids': compare_candidate_ids_tensor, 'p_prior': lambda a, b: len(a) == len(_.intersection(a.tolist(), b.tolist()))} assert coll_compare_keys_by(expected_data, dataset_values, comparison)
def test_intersection(case, expected): assert _.intersection(*case) == expected
'BGBLPGPL': [] } with open('files/gene lists/dbVarCNGsBenignPCGenes.txt', 'r') as dbVarCNGsBenignPCGenes: benignGain = dbVarCNGsBenignPCGenes.read().splitlines() with open('files/gene lists/dbVarCNLsBenignPCGenes.txt', 'r') as dbVarCNLsBenignPCGenes: benignLoss = dbVarCNLsBenignPCGenes.read().splitlines() with open('files/gene lists/dbVarCNGPeaksPathogenicPCGenes.txt', 'r') as dbVarCNGPeaksPathogenicPCGenes: pathoGain = dbVarCNGPeaksPathogenicPCGenes.read().splitlines() with open('files/gene lists/dbVarCNLPeaksPathogenicPCGenes.txt', 'r') as dbVarCNLPeaksPathogenicPCGenes: pathoLoss = dbVarCNLPeaksPathogenicPCGenes.read().splitlines() lists['BGBLPGPL'] = pydash.intersection(benignGain, benignLoss, pathoGain, pathoLoss) _BGBLPG = pydash.intersection(benignGain, benignLoss, pathoGain) lists['BGBLPG'] = pydash.difference(_BGBLPG, pathoLoss) _BGBLPL = pydash.intersection(benignGain, benignLoss, pathoLoss) lists['BGBLPL'] = pydash.difference(_BGBLPL, pathoGain) _BGPGPL = pydash.intersection(benignGain, pathoGain, pathoLoss) lists['BGPGPL'] = pydash.difference(_BGPGPL, benignLoss) _BLPGPL = pydash.intersection(benignLoss, pathoGain, pathoLoss) lists['BLPGPL'] = pydash.difference(_BLPGPL, benignGain) _BGBL = pydash.intersection(benignGain, benignLoss) lists['BGBL'] = pydash.difference(_BGBL, pathoGain, pathoLoss)
def main(): printer = pprint.PrettyPrinter(indent=2) restaurant_columns = [ 'alcohol', 'smoking_area', 'dress_code', 'accessibility', 'price', 'Rambience', 'franchise' ] restaurant_cat_columns = [ 'alcohol', 'smoking_area', 'dress_code', 'accessibility', 'price', 'Rambience', 'franchise' ] user_columns = [ 'smoker', 'drink_level', 'dress_preference', 'ambience', 'transport', 'marital_status', 'hijos', 'birth_year', 'interest', 'personality', 'religion', 'activity', 'weight', 'budget', 'height' ] user_cat_columns = [ 'smoker', 'drink_level', 'dress_preference', 'ambience', 'transport', 'marital_status', 'hijos', 'interest', 'personality', 'religion', 'activity', 'budget' ] features = user_columns + restaurant_columns categorical_features = user_cat_columns + restaurant_cat_columns categorical_feature_indices = [ features.index(feature_name) for feature_name in categorical_features ] restaurant_info = pd.read_csv('./data/geoplaces2.csv', dtype=dict( zip(restaurant_cat_columns, [str] * len(restaurant_cat_columns)))) user_info = pd.read_csv('./data/userprofile.csv', dtype=dict( zip(user_cat_columns, [str] * len(user_cat_columns)))) cuisine_type = pd.read_csv('./data/chefmozcuisine.csv') user_rating = pd.read_csv('./data/rating_final.csv') user_preferences = pd.read_csv('./data/usercuisine.csv') place_ids = _.intersection(user_rating['placeID'].tolist(), restaurant_info['placeID'].tolist()) user_ids = _.intersection(user_rating['userID'].tolist(), user_info['userID'].tolist()) joined_data_dict = {column: [] for column in features + ['rating']} for row in user_rating[['userID', 'placeID', 'rating']].iterrows(): user_id = row[1]['userID'] place_id = row[1]['placeID'] if user_id in user_ids and place_id in place_ids: joined_data_dict['rating'].append(row[1]['rating']) for column in user_columns: joined_data_dict[column].append( user_info[user_info['userID'] == user_id][column].iloc[0]) for column in restaurant_columns: joined_data_dict[column].append(restaurant_info[ restaurant_info['placeID'] == place_id][column].iloc[0]) joined_data = pd.DataFrame(joined_data_dict) data = joined_data.loc[:, joined_data.columns != 'rating'] rating = joined_data['rating'] encoded_data, categorical_names = encode_data(data, features, categorical_features) X = encoded_data y = rating.values == 2 one_hot_encoder = OneHotEncoder( categorical_features=categorical_feature_indices) one_hot_encoder.fit(X) X_train, X_test, y_train, y_test = train_test_split(X, y) model = ensemble.RandomForestClassifier(class_weight='balanced') model.fit(one_hot_encoder.transform(X_train), y_train) predict_fn = lambda X: model.predict(one_hot_encoder.transform(X)) predict_proba_fn = lambda X: model.predict_proba( one_hot_encoder.transform(X)) report(predict_fn, X_test, y_test) explainer = train({ 'X_train': X_train.values, 'y_train': y_train }, { 'feature_names': features, 'categorical_feature_indices': categorical_feature_indices, 'categorical_names': categorical_names }) explanation = inquire( explainer, { 'instance': X_test.iloc[0], 'predict_fn': predict_proba_fn, 'num_features': 5 }) printer.pprint(explanation.as_list())
def len_intersection(f1, f2): return len(intersection(f1, f2))