def fit_cca(self, outfile=''): # fits linear CCA constraint and replaces pretrained name embeddings with CCA transformed embeddings self.load_embeddings() self.extract_pretrained_prototype_embeddings() items, vectors = zip( *[(k, v) for k, v in self.pretrained_prototype_embeddings.items() if k in self.exemplar_to_concept]) concept_embs = Reach(vectors, items) train_vectors = [] for x in items: train_vectors.append(self.train_embeddings[x]) train_vectors = Reach.normalize(train_vectors) cca = CCA(n_components=self.train_embeddings.size, max_iter=10000) cca.fit(train_vectors, concept_embs.norm_vectors) # transform all name embeddings using the CCA mapping all_name_embeddings = deepcopy(self.pretrained_name_embeddings) items = [x for _, x in sorted(all_name_embeddings.indices.items())] projected_name_embeddings = cca.transform( all_name_embeddings.norm_vectors) new_name_embeddings = Reach(projected_name_embeddings, items) self.pretrained_name_embeddings = new_name_embeddings self.load_embeddings() if outfile: with open('{}_cca.p', 'wb') as f: pickle.dump(cca, f)
def synonym_retrieval(self, normalize=True, pretrained=False): rank_util = RankingUtils() # first encode train_names = sorted({name for name, concept in self.train_data}) train_vectors = [ self.pretrained_name_embeddings[x] for x in train_names ] train_embeddings = Reach(train_vectors, train_names) if not pretrained: train_embeddings = self.extract_online_dan_embeddings( provided_names=set(train_embeddings.items.keys()), normalize=normalize) validation_names = sorted( {name for name, concept in self.validation_data}) validation_vectors = [ self.pretrained_name_embeddings[x] for x in validation_names ] validation_embeddings = Reach(validation_vectors, validation_names) if not pretrained: validation_embeddings = self.extract_online_dan_embeddings( provided_names=set(validation_embeddings.items.keys()), normalize=normalize) # then rank training data for each validation_item complete_ranking = [] for reference, concept in self.validation_data: # calculate distances reference_idx = validation_embeddings.items[reference] reference_vector = validation_embeddings.norm_vectors[ reference_idx] scores = train_embeddings.norm_vectors.dot(reference_vector.T) # rank synonym_names = self.training_clusters[concept] synonym_idxs = [ train_embeddings.items[synonym_name] for synonym_name in synonym_names ] ranking = np.argsort(-scores) ranks = [ np.where(ranking == synonym_idx)[0][0] for synonym_idx in synonym_idxs ] ranks, synonyms = zip(*sorted(zip(ranks, synonym_names))) complete_ranking.append((reference, synonyms, ranks)) ranking = [x[-1] for x in complete_ranking] print(rank_util.ranking_accuracy(ranking)) print(rank_util.mrr(ranking)) print(rank_util.mean_average_precision(ranking)) return complete_ranking
def getConceptVectors_choi(self): ''' Wraps a concept vector file from reach, so you do not have to declare this in each feature generation file ''' if self.conceptVectors == None: self.conceptVectors = Reach(cfg.PATH_RESOURCES + 'concepts_choi.txt', header=False) return self.conceptVectors
def getWordVectors(self): ''' Wraps a word vector file from reach, so you do not have to declare this in each feature generation file ''' if self.wordVectors == None: self.wordVectors = Reach(cfg.PATH_RESOURCES+'psych_vectors.txt', header=True) return self.wordVectors
def extract_online_dan_embeddings(self, normalize=True, verbose=False, provided_names=(), preprocess=False): self.model.eval() if provided_names: input_items = provided_names if preprocess: input_items = [self.preprocess(name) for name in input_items] embeddings = self.vectorize.create_reach_object(input_items) else: embeddings = deepcopy(self.pretrained_name_embeddings) input_vectors = embeddings.norm_vectors if normalize else embeddings.vectors input_items = [x for _, x in sorted(embeddings.indices.items())] # batch input items to save up on memory... all_embeddings = [] batch_size = 1000 for i in tqdm(range(0, len(input_items), batch_size), disable=not verbose): input_batch = input_vectors[i:i + batch_size] input_tensor = torch.FloatTensor(input_batch).to(self.device) online_batch = self.model(input_tensor).detach().cpu().numpy() all_embeddings.append(online_batch) all_embeddings = np.concatenate(all_embeddings) online_embeddings = Reach(all_embeddings, input_items) return online_embeddings
def __init__(self): self.client = zulip.Client( site="https://merkalysis.zulipchat.com/api/") self.subscribe_all() self.market = Reach() print("done init") self.subkeys = ["reach"]
def baseline(text, keep_n=10000): """Create a one-hot encoded baseline vector space.""" c = CountVectorizer(text, max_features=keep_n) c.fit(text) words = c.get_feature_names() words = ["UNK"] + words vectors = np.eye(len(words)) return Reach(vectors, list(words), unk_index=0)
def compose(documents, embeddings, window, context_function, use_focus=True, norm=False): """ Map phrases from sentences to vectors. Parameters ========== documents : list of lists A list of lists, where each sublist contains 2 lists of the same length, where the first list contains the tokens of a text, and the second list contains the BIO of the NP chunks for said text. embeddings : Reach A reach instance which contains the embeddings you want to use to vectorize. window : int The window size to use. context_function : function The function which is used to weigh the contexts. Must take a 2D matrix and return a 2D matrix of the same shape. use_focus : bool, optional, default True Whether to vectorize the focus word. norm : bool, optional, default False Whether to use the unit vectors to compose. Returns ======= phrases : Reach A reach instance containing the phrases and their vectors. """ bio_regex = re.compile(r"BI*") phrases, vectors = [], [] for idx, (txt, bio) in enumerate(documents): txt = " ".join(txt).lower().split() bio = "".join([x.split("-")[0] for x in bio]) for t in bio_regex.finditer(bio): b, e = t.span() phrase_string, vector = create_phrase_vector( txt, b, e, window, embeddings, np.mean, np.mean, context_function, use_focus, norm) # Phrase string needs to be augmented with index to make # the dictionary mapping not overwrite itself. phrase_string = "{}-{}".format(phrase_string, len(phrases)) phrases.append(phrase_string) vectors.append(vector) return Reach(vectors, phrases)
def label_chunks(gold_bio, phrase_bio, embeddings): """ Find a label for each phrase chunk based on the gold chunks. Each phrase chunk which does not correctly overlap with a gold chunk is pruned from the embedding space and added as a false positive. Parameters ========== gold_bio : list of string The token-level BIO string for the gold standard data. Must include classes on the B and I labels (e.g. B-Test, I-test). phrase_bio : list of string The token-level BIO string for the phrase data. Does not include any classes on the B and I labels. (e.g. B and I instead of B-test). embeddings : Reach The embedding space for the phrases. Returns ======= pruned_embeddings : Reach The embedding space with any false positive phrases removed. word2label : dict Dictionary mapping from the name of each phrase to a label. chunk_labels : np.array An aligned list from phrases to labels. results : list of tuples An intermediate list of false positives and false negatives constructed during matching the gold and phrase chunks. """ # Create a list of (start, end, label) tuples from BIO. phrase_chunks = bio_to_index(phrase_bio) gold_chunks = bio_to_index(gold_bio) phrase_labels, results = link_chunks_to_gold(phrase_chunks, gold_chunks) # False positives get assigned the label "o", so they need to be removed. allowed = [i for i, v in enumerate(phrase_labels) if v != "o"] if results: t, _ = zip(*results) print("Num false neg: {0}".format(Counter(t))) # We assume alignment between chunks and words. vectors = embeddings.norm_vectors[allowed] chunk_labels = np.array(phrase_labels)[allowed] words = [embeddings.indices[x] for x in allowed] words2label = { embeddings.indices[x]: chunk_labels[idx] for idx, x in enumerate(allowed) } pruned_embeddings = Reach(vectors, words) return pruned_embeddings, words2label, chunk_labels, results
def __init__(self, path, version='0'): g = DiGraph() gaged_reaches = [] db = openFile(path, "r") table = db.getNode('/', 'networks/network' + str(version)) reaches = {} #read data out of file for row in table: if str(row['ComID']) != '-1': reaches[row['ComID']] = Reach(self, row) else: reaches[row['ComID']] = '-1' g.add_edge(Reach(self, row), '-1') if row['MonitoredFlag'] == '1': gaged_reaches.append(row['ComID']) db.close() #make network for comid in reaches.keys(): to_comID = reaches[comid]._ToComID if to_comID != '-1': g.add_edge(reaches[comid], reaches[to_comID]) else: g.add_edge(reaches[comid], -1) self._g_unbroken = g.copy() self._g_unbroken_reverse = self._g_unbroken.reverse() #break upstream of monitored reaches for i in gaged_reaches: if i != '-1': up = g.predecessors(reaches[i]) for j in up: if j != '-1': g.delete_edge(j, reaches[i]) else: g.delete_edge(j, '-1') self._g = g self._g_rev = g.reverse() self._version = str(version) self._path = str(path) self._reaches = reaches db.close()
def create_reach_object(self, strings, normalize=False, outfile=''): vectors = [] for s in tqdm(strings): token_embs = self.vectorize_string(s, norm=normalize) vector = np.average(np.array(token_embs), axis=0) vectors.append(vector) reach_object = Reach(vectors, list(strings)) if outfile: reach_object.save_fast_format(outfile) return reach_object
def create_concepts(concepts, embeddings, include_np=True): """Create concepts by summing over descriptions in embedding spaces.""" # Gold standard labels for concepts: sty = json.load(open("data/concept_label.json")) concept_names = [] vectors = [] concept_labels = [] for name, descriptions in tqdm(list(concepts.items())): try: label = sty[name] except KeyError: continue if not include_np and label == "np": continue concept = [] for idx, desc in enumerate(descriptions): try: desc = desc.lower().split() # desc = [x for x in desc if x not in STOP_WORDS] vec = embeddings.vectorize(desc, remove_oov=True) if not np.any(vec): continue concept.append(np.mean(vec, axis=0)) except ValueError: pass if not concept: continue concept_labels.append(label) name = "{0}_{1}".format(name, "_".join(descriptions[0].split())) concept_names.append(name) vectors.append(np.array(concept).mean(axis=0)) r = Reach(np.array(vectors), concept_names) name2label = dict(zip(concept_names, concept_labels)) return r, name2label
def create_reach_object(self, names, outfile=''): names = sorted(names) vectors = [] for name in tqdm(names): token_embs = self.vectorize_string(name, norm=False) vector = np.average(np.array(token_embs), axis=0) vectors.append(vector) reach_object = Reach(vectors, names) if outfile: reach_object.save_fast_format(outfile) return reach_object
def extract_online_lstm_embeddings(self, prune=False, normalize=True, verbose=False, provided_names=(), preprocess=False): self.model.eval() if provided_names: input_items = provided_names if preprocess: input_items = [self.preprocess(name) for name in input_items] else: embeddings = deepcopy(self.sampling.pretrained_name_embeddings) if prune: names_to_prune = set( self.sampling.exemplar_to_concept.keys()).union( self.sampling.validation_references.keys()) embeddings.prune(names_to_prune) input_items = [x for _, x in sorted(embeddings.indices.items())] # batch input items to save up on memory... all_embeddings = [] batch_size = 500 if self.hidden_size >= 9600 else 1000 for i in tqdm(range(0, len(input_items), batch_size), disable=not verbose): input_batch = input_items[i:i + batch_size] input_vectors = [] for item in input_batch: vector = self.sampling.vectorize_string(item, norm=normalize) input_vectors.append(torch.FloatTensor(vector).to(self.device)) # pass through LSTM network lstm_embeddings = self.forward_lstm(input_vectors) online_batch = lstm_embeddings.detach().cpu().numpy() # add batch all_embeddings.append(online_batch) # convert to embeddings all_embeddings = np.concatenate(all_embeddings) online_embeddings = Reach(all_embeddings, input_items) return online_embeddings
def TraverseReach(self, cur): reachID = len(self.l_Reach) # get a new reach-ID reach = Reach(reachID) # create a new Reach type variable with that ID self.array_done[cur[0], cur[ 1]] = reachID # mark array done for that first non-junction skeleton point by its reach-ID list_nextPoints = self.getNextPoints( cur) # calculate the next non-junction skeleton point while (len(list_nextPoints) == 1): # if only one such point exists cur = list_nextPoints[0] reach.addToStream(cur) # Run addToStream() on that point self.array_done[cur[0], cur[ 1]] = reachID # give this new point the same reach-ID a its previous point list_nextPoints = self.getNextPoints( cur) # proceed to the next point self.l_Reach.append( reach ) # append this new reach ending with the point - cur, to the list - l_Reach return cur, reach
def create_concepts(concepts, embeddings, include_np=True, labels=None): """Create concepts by summing over descriptions in embedding spaces.""" # Gold standard labels for concepts: concept_names = [] vectors = [] for name, descriptions in tqdm(list(concepts.items())): if labels is not None: try: label = sty[name] except KeyError: continue if not include_np and label == "np": continue concept = [] for idx, desc in enumerate(descriptions): try: desc = desc.lower().split() # desc = [x for x in desc if x not in STOP_WORDS] vec = embeddings.vectorize(desc, remove_oov=True) if not np.any(vec): continue concept.append(np.mean(vec, axis=0)) except ValueError: pass if not concept: continue concept_names.append(name) vectors.append(np.array(concept).mean(axis=0)) r = Reach(np.array(vectors), concept_names) return r
def create_cluster_prototypes(self, provided_embeddings=None, total=False, pretrained=True): if provided_embeddings != None: embeddings = provided_embeddings else: if pretrained: embeddings = self.pretrained_name_embeddings else: embeddings = self.extract_online_dan_embeddings(prune=False) clusters = self.clusters if total else self.training_clusters print('Creating cluster prototypes...') cluster_prototypes = {} for label, strings in clusters.items(): strings = set(strings).intersection(self.training_names) cluster_prototypes[label] = self.create_prototype( strings, embeddings) items, vectors = zip(*cluster_prototypes.items()) self.cluster_prototypes = Reach(vectors, items)
def __init__(self, river, len_dang_arcs, fast): # Initialize the variables self.array_skeleton = [] self.row = 0 self.col = 0 self.length_dangling_arcs = len_dang_arcs self.array_Junction = [] self.array_done = [] self.list_Junction = [] self.l_Reach = [] self.flag_fast = fast # Find the skeleton and extract the river boundary # By the end of this constructor function we have a skeleton of the river with no dangling arcs, with junction points and reaches identified # 1. Get the river image in the form of numpy array ------------------- array_Image = river.getRiver() # 2. Find skeleton for the river -------------------------------------- print("Finding the skeleton") self.array_skeleton = morphology.skeletonize(array_Image > 0) self.array_skeleton = npy.array(self.array_skeleton, dtype=npy.uint8) (self.row, self.col) = self.array_skeleton.shape # --------------------------------------------------------------------- # image after taking the skeleton of river plt.imshow(self.array_skeleton) plt.savefig('temp/10_Skeleton.png', format='jpg', dpi=1200) # --------------------------------------------------------------------- # 3. Remove dangling arcs --------------------------------------------- if not fast: print("Removing dangling arcs") self.RemoveDanglingArc() # ----------------------------------------------------------------- # image after removing dangling arcs form the skeleton plt.imshow(self.array_skeleton) plt.savefig('temp/11_RemoveDanglingArc.png', format='jpg', dpi=1200) # ----------------------------------------------------------------- # 4. Find the junction points ----------------------------------------- print("Identifying all the junctions") self.array_done = npy.zeros((self.row, self.col), dtype=npy.int) self.array_Junction = npy.zeros((self.row, self.col), dtype=npy.int) self.MarkJunctions() # --------------------------------------------------------------------- # image after finding the junctions of the river plt.imshow(self.array_Junction) plt.savefig('temp/12_Junctions.png', format='jpg', dpi=1200) # --------------------------------------------------------------------- # 5. Identify all the reaches ----------------------------------------- print("Identifying all the reaches") reach = Reach(0) # create a new Reach type variable with ReachID = 0 self.l_Reach.append(reach) # append it to the list - l_Reach self.IdentifyReach() # self.MarkJunctionsAndNeighbourhood() # self.MarryReachJunction() return
def synonym_retrieval_zeroshot(self, zeroshot_pairs, isolated=False, verbose=False, outfile=''): assert self.train_vectors != None, 'No train vectors are loaded yet!' assert self.test_vectors != None, 'No test vectors are loaded yet!' # new setting: add ALL zeroshot data to train data to cause more confusion train_items = [ x for _, x in sorted(self.train_vectors.indices.items()) ] train_vectors = self.train_vectors.vectors zeroshot_items = set() for concept, reference, synonyms in zeroshot_pairs: zeroshot_items.add(reference) zeroshot_items.update(synonyms) zeroshot_items = sorted(zeroshot_items) zeroshot_vectors = [] for zeroshot_item in zeroshot_items: zeroshot_vectors.append(self.test_vectors[zeroshot_item]) if isolated: fused_vectors = Reach(zeroshot_vectors, zeroshot_items) else: all_items = train_items + zeroshot_items zeroshot_vectors = np.array(zeroshot_vectors) all_vectors = np.concatenate((train_vectors, zeroshot_vectors), axis=0) fused_vectors = Reach(all_vectors, all_items) # now rank complete_ranking = [] for instance in tqdm(zeroshot_pairs, disable=False): concept, reference, synonyms = instance synonym_idxs = [fused_vectors.items[syn] for syn in synonyms] reference_idx = fused_vectors.items[reference] # calculate distances reference_vector = fused_vectors.norm_vectors[reference_idx] scores = fused_vectors.norm_vectors.dot(reference_vector.T) # extract ranking mask = [ 1 if x == reference_idx else 0 for x in range(len(fused_vectors.items)) ] scores = np.ma.array(scores, mask=mask) ranking = np.argsort(-scores) ranks = [ np.where(ranking == synonym_idx)[0][0] for synonym_idx in synonym_idxs ] assert ranks ranks, synonyms = zip(*sorted(zip(ranks, synonyms))) instance = (concept, reference, synonyms) complete_ranking.append((instance, ranks)) if outfile: print('Saving...') with open(outfile, 'w') as f: json.dump(complete_ranking, f) if verbose: instances, rankings = zip(*complete_ranking) print(round(self.mean_average_precision(rankings), 2), '&', round(self.ranking_accuracy(rankings), 2), '&', round(self.mean_reciprocal_rank(rankings), 2), '&') return complete_ranking
if __name__ == "__main__": import logging import time import json # Setup logging.basicConfig(level=logging.INFO) umls = "sample_data/umls_sample.json" msh = "sample_data/abstracts_example.json" path_to_embeddings = "" use_subset = False embeddings = Reach(path_to_embeddings, header=True, verbose=False) logging.info("loaded embeddings.") start = time.time() y = Yarn(embeddings) umls = json.load(open(umls)) msh = json.load(open(msh)) if use_subset: subset = [ u'di', u'tat', u'erp', u'ori', u'crna', u'pep', u'de', u'hip', u'glycoside', u'sterilization', u'ra', u'don', u'ecg', u'cell',
def run(options): logger = get_logger() validation_dataset = get_validation_dataset(options) validation_iterator = get_validation_iterator(options, validation_dataset) word2idx = validation_dataset['word2idx'] embeddings = validation_dataset['embeddings'] idx2word = {v: k for k, v in word2idx.items()} logger.info('Initializing model.') trainer = build_net(options, embeddings, validation_iterator) diora = trainer.net.diora # 1. Get all relevant phrase vectors. dtype = { 'example_ids': 'list', 'labels': 'list', 'positions': 'list', 'sizes': 'list', 'phrases': 'list', 'inside': 'torch', 'outside': 'torch', } batch_recorder = BatchRecorder(dtype=dtype) # Eval mode. trainer.net.eval() batches = validation_iterator.get_iterator(random_seed=options.seed) logger.info('Beginning to embed phrases.') strings = [] with torch.no_grad(): for i, batch_map in enumerate(batches): sentences = batch_map['sentences'] length = sentences.shape[1] # Skips very short examples. if length <= 2: continue strings.extend([ "".join([idx2word[idx] for idx in x]) for x in sentences.numpy() ]) trainer.step(batch_map, train=False, compute_loss=False) batch_result = {} batch_result['inside'] = diora.inside_h[:, -1] batch_result['outside'] = diora.outside_h[:, -1] batch_recorder.record(**batch_result) result = batch_recorder.get_flattened_result() # 2. Build an index of nearest neighbors. vectors = np.concatenate([result['inside'], result['outside']], axis=1) print(len(strings), vectors.shape) r = Reach(vectors, strings) for s in strings: print(s) print(r.most_similar(s))
# stores a list of row numbers and argument strings per verb. import re import numpy as np from reach import Reach import transformargs #pathnames rowspath = './cooccurrence/weighted_sm.rows' embeddingspath = './tulkens-embeddings/160/sonar-160.txt' logpath = './failedwords.txt' exportpath = './verbtrainingindex2' #import data rowsfile = open(rowspath, 'r', encoding='utf-8') r = Reach(embeddingspath, header=True) #holmatrix = np.load(holmatrixpath) #load output file log = open(logpath, 'w', encoding='utf-8') control = np.zeros(160) failedcount = 0 rowcount = 590408 t = transformargs.Transformer() verbarray = np.array( ['', np.array([np.array([0, ''], object)], object)], object ) #will contain line indexes and corresponding argument strings for each verb #dummy first row added to show structure verbindex = 0 oldkey = ''
import sklearn.preprocessing import math import matplotlib.pyplot as plt from tqdm import tqdm_notebook as tqdm import pandas as pd #pathnames indexpath = './verbtrainingindex_withweights.npy' holmatrixpath = './cooccurrence/svd/newmatrix.npy' embeddingspath = './tulkens-embeddings/160/sonar-160.txt' outputpath = './verbmatrices/version3' #import data index = np.load(indexpath) holmatrix = np.load(holmatrixpath) arg_data = Reach(embeddingspath, header=True) #parameters n_dim = 160 s_dim = 200 alpha_value = 50 min_sample_size = 400 # note how in testing, a samplesize of N = 500 was deemed acceptable. here we # do not split in train an test data, so the min sample size can be 80% of the # one used when testing. variance_control = True mean_std = 0.08 matrices = dict() #loop through verbs