def load_simple(self, path): entryset = parsing.run_parser(path) data, size = [], 0 invocab, outvocab = [], [] for i, entry in enumerate(entryset): progress = round(float(i) / len(entryset), 2) print('Progress: {0}'.format(progress), end=' \r') try: # triples greater than 1 if len(entry.modifiedtripleset) > 1: # process source tripleset = [] for i, triple in enumerate(entry.modifiedtripleset): striple = triple.predicate + ' ' + triple.subject + ' ' + triple.object tripleset.append((i, striple)) # given a fixed order by sorting the set of triples automatically (predicate - subject - object) tripleset = sorted(tripleset, key=lambda x: x[1]) triples = [entry.modifiedtripleset[t[0]] for t in tripleset] entitymap = {b:a for a, b in entry.entitymap_to_dict().items()} source, _, entities = load.source(triples, entitymap, {}) invocab.extend(source) targets = [] for lex in entry.lexEntries: # process ordered tripleset _, text, _ = load.snt_source(lex.orderedtripleset, entitymap, entities) text = [w for w in text if w not in ['<SNT>', '</SNT>']] trg_preds = [t[1] for t in utils.split_triples(text)] target = { 'lid': lex.lid, 'comment': lex.comment, 'output': trg_preds } targets.append(target) outvocab.extend(trg_preds) data.append({ 'eid': entry.eid, 'category': entry.category, 'augmented': False, 'size': entry.size, 'source': source, 'targets': targets }) size += len(targets) except: print('Preprocessing error...') invocab.append('unk') outvocab.append('unk') invocab = list(set(invocab)) outvocab = list(set(outvocab)) vocab = { 'input': invocab, 'output': outvocab } print('Path:', path, 'Size: ', size) return data, vocab
def load(self, path): entryset = parsing.run_parser(path) data, size = [], 0 invocab, outvocab, surfacevocab = [], [], [] nerrors = 0 for i, entry in enumerate(entryset): progress = round(i / len(entryset), 2) print('Progress: {0} \t Errors: {1}'.format( progress, round(nerrors / len(entryset), 2)), end='\r') entitymap = {b: a for a, b in entry.entitymap_to_dict().items()} visited = [] for lex in entry.lexEntries: # process ordered tripleset source, delex_source, _ = load.snt_source( lex.orderedtripleset, entitymap, {}) if source not in visited: visited.append(source) invocab.extend(source) targets = [] for lex2 in entry.lexEntries: _, target, entities = load.snt_source( lex2.orderedtripleset, entitymap, {}) if delex_source == target: try: template, vocab = self.extractor.extract( lex2.template) surfacevocab.extend(vocab) for i, word in enumerate(template): if word in entities: template[i] = entities[word] target = { 'lid': lex.lid, 'comment': lex.comment, 'output': template } targets.append(target) outvocab.extend(template) except: nerrors += 1 print('Parsing Error...') data.append({ 'eid': entry.eid, 'category': entry.category, 'size': entry.size, 'source': source, 'targets': targets }) size += len(targets) invocab.append('<unk>') outvocab.append('<unk>') invocab = list(set(invocab)) outvocab = list(set(outvocab)) vocab = {'input': invocab, 'output': outvocab} print('Path:', path, 'Size: ', size) return data, vocab, surfacevocab
def load(self, path, augment=True): entryset = parsing.run_parser(path) data, size = [], 0 invocab, outvocab = [], [] for i, entry in enumerate(entryset): progress = round(float(i) / len(entryset), 2) print('Progress: {0}'.format(progress), end=' \r') try: # process source entitymap = { b: a for a, b in entry.entitymap_to_dict().items() } source, _, entities = load.source(entry.modifiedtripleset, entitymap, {}) invocab.extend(source) targets = [] for lex in entry.lexEntries: # process ordered tripleset text = self.tokenize(text=lex.text) target = { 'lid': lex.lid, 'comment': lex.comment, 'output': text, 'text': lex.text.replace('@', ' ') } targets.append(target) outvocab.extend(text) data.append({ 'eid': entry.eid, 'category': entry.category, 'augmented': False, 'size': entry.size, 'source': source, 'targets': targets }) size += len(targets) # choose the original order and N permutations such as N = len(tripleset)-1 if augment: triplesize = len(entry.modifiedtripleset) perm = list(permutations(entry.modifiedtripleset)) perm = [load.source(src, entitymap, {}) for src in perm] entitylist = [w[2] for w in perm] perm = [w[0] for w in perm] taken = [] # to augment the corpus, pick the minumum between the number of permutations - 1 or 49 X = min(len(perm) - 1, 49) for _ in range(X): found = False while not found and triplesize != 1: pos = randint(0, len(perm) - 1) src, entities = perm[pos], entitylist[pos] if pos not in taken and src != source: taken.append(pos) found = True targets = [] for lex in entry.lexEntries: # process ordered tripleset text = self.tokenize(text=lex.text) target = { 'lid': lex.lid, 'comment': lex.comment, 'output': text, 'text': lex.text.replace('@', ' ') } targets.append(target) outvocab.extend(text) data.append({ 'eid': entry.eid, 'category': entry.category, 'augmented': True, 'size': entry.size, 'source': src, 'targets': targets }) size += len(targets) except: print('Preprocessing error...') invocab.append('unk') outvocab.append('unk') invocab = list(set(invocab)) outvocab = list(set(outvocab)) vocab = {'input': invocab, 'output': outvocab} print('Path:', path, 'Size: ', size) return data, vocab
def load_index(self, path): entryset = parsing.run_parser(path) data, size = [], 0 invocab, outvocab = [], [] for i, entry in enumerate(entryset): progress = round(float(i) / len(entryset), 2) print('Progress: {0}'.format(progress), end=' \r') try: # triples greater than 1 if len(entry.modifiedtripleset) > 1: # process source tripleset = [] for i, triple in enumerate(entry.modifiedtripleset): striple = triple.predicate + ' ' + triple.subject + ' ' + triple.object tripleset.append((i, striple)) # given a fixed order by sorting the set of triples automatically (predicate - subject - object) tripleset = sorted(tripleset, key=lambda x: x[1]) triples = [ entry.modifiedtripleset[t[0]] for t in tripleset ] entitymap = { b: a for a, b in entry.entitymap_to_dict().items() } source, _, entities = load.source(triples, entitymap, {}) invocab.extend(source) targets = [] for lex in entry.lexEntries: # process ordered tripleset trg_idx = [] orderedtripleset = [ item for sublist in lex.orderedtripleset for item in sublist ] for sorted_triple in orderedtripleset: for i, src_triple in enumerate(triples): if sorted_triple.subject == src_triple.subject and \ sorted_triple.predicate == src_triple.predicate and \ sorted_triple.object == src_triple.object and str(i+1) not in trg_idx: trg_idx.append(str(i + 1)) target = { 'lid': lex.lid, 'comment': lex.comment, 'output': trg_idx } targets.append(target) outvocab.extend(trg_idx) data.append({ 'eid': entry.eid, 'category': entry.category, 'augmented': False, 'size': entry.size, 'source': source, 'targets': targets }) size += len(targets) except: print('Preprocessing error...') invocab.append('unk') outvocab.append('unk') invocab = list(set(invocab)) outvocab = list(set(outvocab)) vocab = {'input': invocab, 'output': outvocab} print('Path:', path, 'Size: ', size) return data, vocab
def process(self, entry_path): entryset = parser.run_parser(entry_path) data, size = [], 0 invocab, outvocab = [], [] for i, entry in enumerate(entryset): progress = round(float(i) / len(entryset), 2) print('Progress: {0}'.format(progress), end=' \r') # process source entitymap = entry.entitymap_to_dict() for lex in entry.lexEntries: # process ordered tripleset template = self.temp_extractor.extract(lex.template)[0] template = ' '.join(template).split() refcount = {} for reference in lex.references: tag = reference.tag if tag not in refcount: refcount[tag] = 0 refcount[tag] += 1 entity = '_'.join(reference.entity.split()) if entity != '': refex = self.tokenize(reference.refex) isDigit = entity.replace('.', '').strip().isdigit() regex='([0-9]{4})-([0-9]{2})-([0-9]{2})' isDate = len(re.findall(regex,entity)) > 0 if entity[0] not in ['\'', '\"'] and not isDigit and not isDate: context, pos = [], 0 for i, w in enumerate(template): if w.strip() == tag.strip(): pos += 1 if pos == refcount[tag]: pre_context = copy.copy(context) pos_context = [] for j in range(i+1, len(template)): if template[j].strip() not in entitymap: pos_context.append(template[j].lower()) else: pos_context.append('_'.join(entitymap[template[j]].split())) data.append({ 'entity': entity, 'category': entry.category, 'pre_context': pre_context, 'pos_context': pos_context, 'refex': refex }) size += 1 invocab.extend(pre_context) invocab.extend(pos_context) invocab.append(entity) outvocab.extend(refex) else: if w.strip() not in entitymap: context.append(w.lower()) else: context.append('_'.join(entitymap[w].split())) invocab.append('unk') outvocab.append('unk') invocab.append('eos') outvocab.append('eos') invocab = list(set(invocab)) outvocab = list(set(outvocab)) vocab = { 'input': invocab, 'output': outvocab } print('Path:', entry_path, 'Size: ', size) return data, vocab
def load(self, path): flat = lambda struct: [ w for w in struct if w not in ['<SNT>', '</SNT>'] ] entryset = parsing.run_parser(path) data, size = [], 0 invocab, outvocab = [], [] for entry in entryset: entitymap = {b: a for a, b in entry.entitymap_to_dict().items()} if len(entry.modifiedtripleset) > 1: visited = [] for lex in entry.lexEntries: # process ordered tripleset source, delex_source, _ = load.snt_source( lex.orderedtripleset, entitymap, {}) source, delex_source = flat(source), flat(delex_source) if source not in visited and ' '.join( source).strip() != '': visited.append(source) invocab.extend(source) targets = [] for lex2 in entry.lexEntries: _, text, _ = load.snt_source( lex2.orderedtripleset, entitymap, {}) flatten = flat(text) if delex_source == flatten: trgt_preds = [] for snt in utils.split_struct(text): trgt_preds.append('<SNT>') trgt_preds.extend([t[1] for t in snt]) trgt_preds.append('</SNT>') target = { 'lid': lex2.lid, 'comment': lex2.comment, 'output': trgt_preds } targets.append(target) outvocab.extend(trgt_preds) data.append({ 'eid': entry.eid, 'category': entry.category, 'size': entry.size, 'source': source, 'targets': targets }) size += len(targets) invocab.append('unk') outvocab.append('unk') invocab = list(set(invocab)) outvocab = list(set(outvocab)) vocab = {'input': invocab, 'output': outvocab} print('Path:', path, 'Size: ', size) return data, vocab