def load(self, filename): """Do the exception-prone loading""" # set the allowed tags for later self.allowed_tags = set(['org', 'per', 'loc', 'locorg']) self.mentions = {} for tag in self.allowed_tags: self.mentions[tag] = [] # read the file that should consist of lines like # [TAG] [START_SYMBOL_INDEX] [LENGTH] with safeOpen(filename) as f: r = csv.reader(f, delimiter=' ', quotechar=Config.QUOTECHAR) for index, parts in enumerate(r): # skip the empty lines if len(parts) == 0: continue try: assert(len(parts) == 3) tag = normalize(parts[0]) assert(tag in self.allowed_tags) self.mentions[tag].append(Interval(*parts[1:])) except Exception as e: line_descr = '[{}] [START_SYMBOL_INDEX] [LENGTH]'.format( '/'.join(self.allowed_tags)) raise Exception( 'Error: "{}", line {}.\nExpected: {}\nReceived: {}\nDetails: {}'.format( filename, index, line_descr, ' '.join(parts), str(e)))
def loadSpans(self, filename): """Load the data from a file with the provided name Raw span data should be loaded from one of the system export '.spans' file Expected format: line = <left> SPAN_FILE_SEPARATOR <right> left = <span_id> <tag_name> <start_pos> <nchars> <start_token> <ntokens> right ::= [ <token>]+ [ <token_text>]+ // <ntokens> of each """ self.spans = [] with open(filename, 'r', encoding='utf-8') as f: for index, line in enumerate(f): if len(line) == 0: # skip the empty lines continue parts = line.split(Config.SPAN_FILE_SEPARATOR) if len(parts) != 2: # bad non-empty line raise Exception( 'Expected symbol "{}" missing in line {} of file {}'.format( Config.SPAN_FILE_SEPARATOR, index, filename)) left = parts[0] right = parts[1] filtered_left = [i for i in left.split(Config.DEFAULT_DELIMITER) if len(i) > 0] if len(filtered_left) < 6: raise Exception( 'Missing left parts in line {} of file {}'.format( index, filename)) new_span = Span(*filtered_left) filtered_right = [i for i in right.split(Config.DEFAULT_DELIMITER) if len(i) > 0] if len(filtered_right) != 2*new_span.ntokens: raise Exception( 'Missing right parts in line {} of file {}'.format( index, filename)) token_ids = [x.strip() for x in filtered_right[:new_span.ntokens]] new_span.tokens = sorted([self._token_dict[x] for x in token_ids], key=lambda x: x.start) new_span.text = normalize(' '.join(filtered_right[new_span.ntokens:])) new_span.text = new_span.text.replace('\n', '') self.spans.append(new_span) # fill the span dictionary self._span_dict = dict([(x.id, x) for x in self.spans])
def loadSpans(self, filename): """Load the data from a file with the provided name Raw span data should be loaded from one of the system export '.spans' file Expected format: line = <left> SPAN_FILE_SEPARATOR <right> left = <span_id> <tag_name> <start_pos> <nchars> <start_token> <ntokens> right ::= [ <token>]+ [ <token_text>]+ // <ntokens> of each """ self.spans = [] with open(filename, 'r', encoding='utf-8') as f: for index, line in enumerate(f): if len(line) == 0: # skip the empty lines continue parts = line.split(Config.SPAN_FILE_SEPARATOR) if len(parts) != 2: # bad non-empty line raise Exception( 'Expected symbol "{}" missing in line {} of file {}'.format( Config.SPAN_FILE_SEPARATOR, index, filename)) left = parts[0] right = parts[1] filtered_left = [i for i in left.split(Config.DEFAULT_DELIMITER) if len(i) > 0] if len(filtered_left) < 6: raise Exception( 'Missing left parts in line {} of file {}'.format( index, filename)) new_span = Span(*filtered_left) filtered_right = [i for i in right.split(Config.DEFAULT_DELIMITER) if len(i) > 0] if len(filtered_right) != 2*new_span.ntokens: raise Exception( 'Missing right parts in line {} of file {}'.format( index, filename)) token_ids = [x.strip() for x in filtered_right[:new_span.ntokens]] new_span.tokens = sorted([self._token_dict[x] for x in token_ids], key=lambda x: x.start) new_span.text = normalize(' '.join(filtered_right[new_span.ntokens:])) new_span.text = new_span.text.replace('\n', '') self.spans.append(new_span) # fill the span dictionary self._span_dict = dict([(x.id, x) for x in self.spans])
def __init__(self, id, start, length, text): """Create a new token with the given parameters""" self.id = id self.start = int(start) self.length = int(length) self.end = self.start + self.length - 1 self.text = normalize(text) self.next = None self.prev = None
def fromTest(cls, line): """Load an attribute from the set of lines representing it. This method corresponds to the test format of representation. Returns a new Attribute instance""" parts = line.split(':') assert (len(parts) >= 2) instance = cls() instance.name = parts[0].strip().lower() value = normalize(':'.join(parts[1:])) instance.values.add(value) return instance
def load(self, filename): """Do the exception-prone loading""" self.facts = [] with safeOpen(filename) as f: buffer = '' for raw_line in f: line = normalize(raw_line) if len(line) == 0: if len(buffer) > 0: self.facts.append(Fact.fromTest(buffer)) buffer = '' else: buffer += line + '\n' if len(buffer) > 0: self.facts.append(Fact.fromTest(buffer))
def load(self, filename): """Do the exception-prone loading""" self.facts = [] with safeOpen(filename) as f: buffer = '' for raw_line in f: line = normalize(raw_line) if len(line) == 0: if len(buffer) > 0: self.facts.append(Fact.fromTest(buffer)) buffer = '' else: buffer += line + '\n' if len(buffer) > 0: self.facts.append(Fact.fromTest(buffer))
def fromStandard(cls, lines): """Load an attribute from the set of lines representing it. This method corresponds to the standard format of representation Returns a new Attribute instance""" assert (len(lines) == 1) line = lines[0] parts = line.split(' ') instance = cls() instance.name = parts[0].strip().lower() value = normalize(' '.join(parts[1:])) instance.values.add(value) return instance
def loadText(self, filename): """Load text from the associated text file""" with open(filename, 'r', encoding='utf-8') as f: self.text = normalize(''.join( [line for line in f] ))