示例#1
0
class Sentence(object):
    """Sentence from a document, to be annotated"""
    def __init__(self, text, offset=0, **kwargs):
        self.text = text
        self.sid = kwargs.get("sid")
        self.did = kwargs.get("did")
        self.entities = Entities(sid=self.sid, did=self.did)
        self.offset = offset
        self.pairs = Pairs()
        self.parsetree = None
        self.depparse = None
        self.tokens = []
        self.regex_tokens = re.compile(r'(-|/|\\|\+|\.|\w+)')

    def tokenize_words(self):
        pass

    def process_sentence(self, corenlpserver, doctype="biomedical"):
        corenlpres = corenlpserver.annotate(self.text.encode("utf8"), properties={
            'ssplit.eolonly': True,
            'annotators': 'tokenize,ssplit,pos,ner,lemma',
            #'annotators': 'tokenize,ssplit,pos,parse,ner,lemma,depparse',
            'outputFormat': 'json',
        })
        if isinstance(corenlpres, basestring):
            print corenlpres
            corenlpres = corenlpserver.annotate(self.text.encode("utf8"), properties={
                'ssplit.eolonly': True,
                # 'annotators': 'tokenize,ssplit,pos,depparse,parse',
                'annotators': 'tokenize,ssplit,pos,lemma',
                'outputFormat': 'json',
            })
        if isinstance(corenlpres, basestring):
            print "could not process this sentence:", self.text.encode("utf8")
            print corenlpres
        else:
            self.process_corenlp_output(corenlpres)
        return corenlpres

    def process_corenlp_output(self, corenlpres):

        """
        Process the results obtained with CoreNLP for this sentence
        :param corenlpres:
        :return:
        """
        # self.sentences = []
        if len(corenlpres['sentences']) > 1:
            print self.text
            sys.exit("Number of sentences from CoreNLP is not 1.")
        if len(corenlpres['sentences']) == 0:
            self.tokens = []
            self.create_newtoken("", {})
            logging.debug("no sentences")
            logging.debug(self.text)
            return
        sentence = corenlpres['sentences'][0]
        #logging.debug(str(sentence.keys()))
        #print "sentence", self.text.encode("utf8")
        #print "parse", pp.pprint(sentence["parse"])
        #print "basic", pp.pprint(sentence["basic-dependencies"])
        #print "collapsed", pp.pprint(sentence["collapsed-dependencies"])
        #print "ccprocessed", pp.pprint(sentence["collapsed-ccprocessed-dependencies"])
        self.parsetree = sentence.get('parse')
        self.depparse = sentence.get('basic-dependencies')
        for t in sentence['tokens']:
            if t["word"]:
                # TODO: specific rules for each corpus
                #if ""
                token_seq = self.regex_tokens.split(t["originalText"])#, flags=re.U)
                #token_seq = rext.split(r'(\w+)(/|\\|\+|\.)(\w+)', t[0])
                #token_seq = [t[0]]
                # print t[0], token_seq
                if len(token_seq) > 3: #and t["word"] not in stanford_coding.keys():
                    # logging.info("{}: {}".format(t["word"], "&".join(token_seq)))
                    for its, ts in enumerate(token_seq):
                        if ts.strip() != "":
                            charoffset_begin = int(t["characterOffsetBegin"])
                            if token_seq[:its]: # not the first token
                                charoffset_begin += sum([len(x) for x in token_seq[:its]])
                            # charoffset_begin += its
                            charoffset_end = len(ts) + charoffset_begin
                            #logging.info(str(charoffset_begin) + ":" + str(charoffset_end))
                            ts_props = {"characterOffsetBegin": charoffset_begin,
                                        "characterOffsetEnd": charoffset_end,
                                        "pos": t["pos"],
                                        "ner": t["ner"],
                                        "lemma": t["lemma"][charoffset_begin:charoffset_end]}
                            self.create_newtoken(ts, ts_props)

                else:
                    self.create_newtoken(t["word"], t)

    def create_newtoken(self, text, props):
        newtoken = Token2(text, order=len(self.tokens))
        try:
            newtoken.start = int(props["characterOffsetBegin"])
            newtoken.dstart = newtoken.start + self.offset
            newtoken.end = int(props["characterOffsetEnd"])
            newtoken.dend = newtoken.end + self.offset
            newtoken.pos = props["pos"]
            newtoken.tag = props["ner"]
            newtoken.lemma = props["lemma"]
            # newtoken.stem = porter.stem_word(newtoken.text)
            newtoken.tid = self.sid + ".t" + str(len(self.tokens))
            self.tokens.append(newtoken)
            # print "|{}| <=> |{}|".format(text, self.text[newtoken.start:newtoken.end])
        except KeyError:
            logging.debug("error: text={} props={}".format(text, props))
            return None
        # logging.debug(newtoken.text)
        return newtoken

    def add_relation(self, entity1, entity2, subtype, source="goldstandard", **kwargs):
        if self.pairs.pairs:
            pid = self.sid + ".p" + str(len(self.pairs.pairs))
        else:
            pid = self.sid + ".p0"
        if subtype == "tlink":
            p = TLink(entity1, entity2, original_id=kwargs.get("original_id"),
                                     did=self.did, pid=pid, rtype=subtype)
        else:
            p = Pair((entity1, entity2), subtype, pid=pid, sid=self.sid, did=self.did)
        self.pairs.add_pair(p, source)
        return p

    def exclude_entity(self, start, end, source):
        """
        Exclude all entities matching start-end relative to sentence
        :param start:
        :param end:
        """
        to_delete = []
        for e in self.entities.elist[source]:
            if e.start == start and e.end == end:
                to_delete.append(e)
                for t in e.tokens:
                    tagkeys = t.tags.keys()
                    for tag in tagkeys:
                        if tag.startswith(source):
                            del t.tags[tag]
        for e in to_delete:
            #print "removing {}".format(e)
            self.entities.elist[source].remove(e)
            #print [(ee.start, ee.end) for ee in self.entities.elist[source]]


    def tag_entity(self, start, end, etype, entity=None, source="goldstandard", exclude=None,
                   text=None, **kwargs):
        """Find the tokens that match this entity. start and end are relative to the sentence.
           Totalchars is the offset of the sentence on the document."""
        tlist = []
        # print self.tokens
        nextword = ""
        for t in self.tokens:
            # discard tokens that intersect the entity for now
            # print t.start, t.end, t.text
            if t.start >= start and t.end <= end:
                tlist.append(t)
            elif (t.start == start and t.end > end) or (t.start < start and t.end == end):
                tlist.append(t)
                break
            elif t.start == end+1:
                nextword = t.text
            exclude_list = []
            if exclude is not None:
                for t in tlist:
                    for e in exclude:
                        if t.start >= e[0] and t.end <= e[1]-1:
                            exclude_list.append(t.tid)
            tlist = [t for t in tlist if t.tid not in exclude_list]
        if tlist:
            if exclude is not None:
                newtext = self.text[tlist[0].start:exclude[0][0]]
                #print self.text[exclude[0][0]:exclude[0][1]], exclude
                last_exclude = exclude[0]
                for e in exclude[1:]:
                    if not self.text[e[1]].isspace() and not newtext[-1].isspace():
                        newtext += " "
                    newtext += self.text[last_exclude[1]:e[0]]
                    last_exclude = e
                if not self.text[exclude[-1][1]].isspace() and not newtext[-1].isspace():
                    newtext += " "
                newtext += self.text[exclude[-1][1]:tlist[-1].end]
                # self.text[exclude[1]:tlist[-1].end]
            else:
                newtext = self.text[tlist[0].start:tlist[-1].end]
            if entity:
                entity.text = newtext
            if "text" in kwargs and newtext != kwargs["text"]:
                if newtext not in kwargs["text"] and kwargs["text"] not in newtext:
                    logging.info("diferent text:|system {} {} |{}|=>|{}| {} {} input|{} {}".format(tlist[0].start, tlist[-1].end, newtext, kwargs["text"],
                                                                                                   start, end, self.sid,
                                                                                                   self.text))
                    logging.info("text does not match: {}=>{}".format(newtext, kwargs["text"]))
                    #sys.exit()
                    #return None
                else:
                    logging.info("diferent text:|system {} {} |{}|=>|{}| {} {} input|{} {}".format(tlist[0].start, tlist[-1].end, newtext, kwargs["text"],
                                 start, end, self.sid, self.text))
                    #for t in self.tokens:
                    #    print (t.start, t.end, t.text),
                    #print
                    #return None
                    # print exclude, self.text[tlist[0].start:tlist[-1].end]
            #     print "tokens found:", [t.text for t in tlist]
                    # sys.exit()
            # else:
            # print "found the tokens!", start, end, kwargs["text"], self.sid

            if self.entities.elist.get(source):
                eid = self.sid + ".e" + str(len(self.entities.elist[source]))
            else:
                eid = self.sid + ".e0"
            subtype = kwargs.get("subtype", "all")
            if entity is None:
                if "text" in kwargs:
                    newtext = kwargs["text"]
                kwargs["eid"] = eid
                entity = create_entity(tlist, self.sid, did=self.did, text=newtext, score=kwargs.get("score"),
                                       etype=etype, eid=eid, subtype=kwargs.get("subtype"),
                                       original_id=kwargs.get("original_id"), nextword=nextword)

                entity.normalize()
            self.entities.add_entity(entity, source)
            # print self.entities.elist["goldstandard"]
            self.label_tokens(tlist, source, etype, subtype=subtype)
            #logging.debug("added {} to {}, now with {} entities".format(newtext, self.sid,
            #                                                                 len(self.entities.elist[source])))
            return eid
        else:
            logging.info("no tokens found:")
            logging.info("{} {} {} {}".format(self.sid, start, end, kwargs.get("text")))
            logging.info(str([(t.start, t.end, t.text) for t in self.tokens]))

    def label_tokens(self, tlist, source, etype, subtype="all"):
        if len(tlist) == 1:
            tlist[0].tags[source] = "single"
            tlist[0].tags[source + "_subtype"] = etype
            tlist[0].tags[source + "_" + etype] = "single"
            if subtype != "all":
                #print subtype
                tlist[0].tags[source + "_" + etype + "-" + subtype] = "single"
        else:
            for t in range(len(tlist)):
                if t == 0:
                    tlist[t].tags[source] = "start"
                    tlist[t].tags[source + "_" + etype] = "start"
                    tlist[t].tags[source + "_subtype"] = etype
                    if subtype != "all":
                        tlist[t].tags[source + "_" + etype + "-" + subtype] = "start"
                elif t == len(tlist) - 1:
                    tlist[t].tags[source] = "end"
                    tlist[t].tags[source + "_" + etype] = "end"
                    tlist[t].tags[source + "_subtype"] = etype
                    if subtype != "all":
                        tlist[t].tags[source + "_" + etype + "-" + subtype] = "end"
                else:
                    tlist[t].tags[source] = "middle"
                    tlist[t].tags[source + "_" + etype] = "middle"
                    tlist[t].tags[source + "_subtype"] = etype
                    if subtype != "all":
                        tlist[t].tags[source + "_" + etype + "-" + subtype] = "middle"
        # logging.debug([t.tags for t in tlist])

    def write_bioc_results(self, parent, source):
        bioc_sentence = ET.SubElement(parent, "sentence")
        bioc_sentence_offset = ET.SubElement(bioc_sentence, "offset")
        bioc_sentence_offset.text = str(self.tokens[0].dstart)
        bioc_sentence_text = ET.SubElement(bioc_sentence, "text")
        bioc_sentence_text.text = self.text

        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                bioc_annotation = entity.write_bioc_annotation(bioc_sentence)
        return bioc_sentence

    def get_dic(self, source):
        dic = {}
        dic["id"] = self.sid
        dic["offset"] = str(self.tokens[0].dstart)
        dic["text"] = self.text
        dic["entities"] = []
        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                dic["entities"].append(entity.get_dic())
            dic["entities"] = sorted(dic["entities"], key=lambda k: k['offset'])
            for ei, e in enumerate(dic["entities"]):
                e["eid"] = self.sid + ".e{}".format(ei)
        elif source == "all":
            offsets = Offsets()
            for esource in self.entities.elist:
                for entity in self.entities.elist[esource]:
                    toadd, v, overlapping, to_exclude = offsets.add_offset(Offset(entity.start, entity.end),
                                                                           exclude_this_if=[1, -1, 2, -3],
                                                                           exclude_others_if=[2])
                    if toadd:
                        dic["entities"].append(entity.get_dic())
                dic["entities"] = sorted(dic["entities"], key=lambda k: k['offset'])
                for ei, e in enumerate(dic["entities"]):
                    e["eid"] = self.sid + ".e{}".format(ei)
        dic["pairs"] = self.pairs.get_dic()
        return dic

    def find_tokens(self, text, start, end, count, relativeto="doc"):
        candidates = []
        for t in self.tokens:
            if t.text == text:
                print t.text, text
                candidates.append(t)
        print text, candidates
        if len(candidates) == 0:
            print "could not find tokens!"
        elif len(candidates) == 1:
            return candidates
        elif len(candidates)-1 > count:
            candidates[count]
        """else:
            dist = []
            for c in candidates:
                if relativeto == "doc":
                    d = c.dstart
                else:
                    d = c.start
                dist.append(abs(d-start))
            return [candidates[dist.index(min(dist))]]"""

    def find_tokens_between(self, start, end, relativeto="doc"):
        """Return list of tokens between offsets. Use relativeto to consider doc indexes or
           sentence indexes."""
        foundtokens = []
        for t in self.tokens:
            if relativeto.startswith("doc") and t.dstart >= start and t.dend <= end:
                foundtokens.append(t)
            elif relativeto.startswith("sent") and t.start >= start and t.end <= end:
                foundtokens.append(t)
        return foundtokens

    def test_relations(self, pairs, basemodel, classifiers=[relations.SLK_PRED, relations.SST_PRED],
                       tag="", backup=False, printstd=False):
        #data =  ddi_train_slk.model, ddi_train_sst.model
        tempfiles = []

        if relations.SLK_PRED in classifiers:
            logging.info("**Testing SLK classifier %s ..." % (tag,))
            #testpairdic = ddi_kernels.fromddiDic(testdocs)
            ddi_kernels.generatejSREdata(pairs, self, basemodel, tag + "ddi_test_jsre.txt")
            ddi_kernels.testjSRE(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                 model=tag + "all_ddi_train_slk.model")
            self.pairs.pairs = ddi_kernels.getjSREPredicitons(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                                      self.pairs.pairs)
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_jsre.txt")
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_result.txt")

        if relations.SST_PRED in classifiers:
            logging.info("****Testing SST classifier %s ..." % (tag,))
            self.pairs.pairs = ddi_kernels.testSVMTK(self, self.pairs.pairs, pairs,
                                             model=tag + "all_ddi_train_sst.model", tag=tag)
        for p in self.pairs.pairs:
            for r in self.pairs.pairs[p].recognized_by:
                if self.pairs.pairs[p].recognized_by[r] == 1:
                    p.relation = True
        return tempfiles

    def get_entitites_between(self, entity1, entity2, source):
        if entity1.start > entity2.start:  # entity1 should always be the first entity
            entity1, entity2 = entity2, entity1
        first_between = entity1.end
        last_between = entity2.start
        entities = []
        for entity in self.entities.elist[source]:
            if entity.start >= first_between and entity.end <= last_between:
                entities.append(entity)
        return entities
示例#2
0
class Document(object):
    """A document is constituted by one or more sentences. It should have an ID and
    title. s0, the first sentence, is always the title sentence."""

    def __init__(self, text, process=False, doctype="biomedical", ssplit=False, **kwargs):
        self.text = text
        self.title = kwargs.get("title")
        self.sentences = kwargs.get("sentences", [])
        self.did = kwargs.get("did", "d0")
        self.invalid_sids = []
        self.title_sids = []
        self.source = kwargs.get("source")
        self.pairs = Pairs()
        if ssplit:
            self.sentence_tokenize(doctype)
        if process:
            self.process_document(doctype)

    def sentence_tokenize(self, doctype):
        """
        Split the document text into sentences, add to self.sentences list
        :param doctype: Can be used in the future to choose different methods
        """
        # first sentence should be the title if it exists
        #if self.title:
        #    sid = self.did + ".s0"
        #    self.sentences.append(Sentence(self.title, sid=sid, did=self.did))
        # inputtext = clean_whitespace(self.text)
        inputtext = self.text
        with codecs.open("/tmp/geniainput.txt", 'w', 'utf-8') as geniainput:
            geniainput.write(inputtext)
        current_dir = os.getcwd()
        os.chdir(geniass_path)
        geniaargs = ["./geniass", "/tmp/geniainput.txt", "/tmp/geniaoutput.txt"]
        Popen(geniaargs, stdout=PIPE, stderr=PIPE).communicate()
        os.chdir(current_dir)
        offset = 0
        with codecs.open("/tmp/geniaoutput.txt", 'r', "utf-8") as geniaoutput:
            for l in geniaoutput:
                stext = l.strip()
                if stext == "":
                    offset = self.get_space_between_sentences(offset)
                    continue
                sid = self.did + ".s" + str(len(self.sentences))
                self.sentences.append(Sentence(stext, offset=offset, sid=sid, did=self.did))
                offset += len(stext)
                offset = self.get_space_between_sentences(offset)

    def process_document(self, corenlpserver, doctype="biomedical"):
        """
        Process each sentence in the text (sentence split if there are no sentences) using Stanford CoreNLP
        :param corenlpserver:
        :param doctype:
        :return:
        """
        if len(self.sentences) == 0:
            # use specific sentence splitter
            self.sentence_tokenize(doctype)
        for s in self.sentences:
            #corenlpres = corenlpserver.raw_parse(s.text)
            corenlpres = corenlpserver.annotate(s.text.encode("utf8"), properties={
                'ssplit.eolonly': True,
                #'annotators': 'tokenize,ssplit,pos,ner,lemma',
                'annotators': 'tokenize,ssplit,pos,parse,ner,lemma,depparse',
                'outputFormat': 'json',
            })
            if isinstance(corenlpres, basestring):
                print corenlpres
                corenlpres = corenlpserver.annotate(s.text.encode("utf8"), properties={
                'ssplit.eolonly': True,
                # 'annotators': 'tokenize,ssplit,pos,depparse,parse',
                'annotators': 'tokenize,ssplit,pos,ner,lemma',
                'outputFormat': 'json',
            })
            if isinstance(corenlpres, basestring):
                print "could not process this sentence:", s.text.encode("utf8")
                print corenlpres
                continue
            else:
                s.process_corenlp_output(corenlpres)


    def tag_chemdner_entity(self, start, end, subtype, **kwargs):
        """
        Create an CHEMDNER entity relative to this document.
        :param start: Start index of entity
        :param end: End index of entity
        :param subtype: Subtype of CHEMDNER entity
        :param kwargs: Extra stuff like the text
        :return:
        """
        doct = kwargs.get("doct")
        if doct == "T": # If it's in the title, we already know the sentence (it's the first)
            self.sentences[0].tag_entity(start, end, subtype, **kwargs)
        else: # we have to find the sentence
            found = False
            totalchars = 0
            for s in self.sentences[1:]:
                if totalchars <= start and totalchars + len(s.text) >= end:  # entity is in this sentence
                    s.tag_entity(start-totalchars, end-totalchars, subtype,
                                 totalchars=totalchars, **kwargs)
                    # print "found entity on sentence %s" % s.sid
                    found = True
                    break

                totalchars += len(s.text)
                totalchars = self.get_space_between_sentences(totalchars)
            if not found:
                print "could not find sentence for %s:%s on %s!" % (start,
                                                                       end, self.did)
                # sys.exit()

    def add_relation(self, entity1, entity2, subtype, relation, source="goldstandard", **kwargs):
        if self.pairs.pairs:
            pid = self.did + ".p" + str(len(self.pairs.pairs))
        else:
            pid = self.did + ".p0"
        between_text = self.text[entity1.dend:entity2.start]
        logging.info("adding {}:{}=>{}".format(pid, entity1.text.encode("utf8"), entity2.text.encode("utf8")))
        # print between_text
        if subtype == "tlink":
            pair = TLink(entity1, entity2, relation=relation, original_id=kwargs.get("original_id"),
                                     did=self.did, pid=pid, rtype=subtype, between_text=between_text)
        else:
            pair = Pair((entity1, entity2), subtype, did=self.did, pid=pid, original_id=kwargs.get("original_id"), between_text=between_text)
        self.pairs.add_pair(pair, source)
        return pair

    def get_space_between_sentences(self, totalchars):
        """
        When the sentences are split, the whitespace between each sentence is not preserved, so we need to get it back
        :param totalchars: offset of the end of sentence
        :return: Index where the next sentence starts
        """
        while totalchars < len(self.text) and self.text[totalchars].isspace():
            totalchars += 1
        return totalchars

    def get_unique_results(self, source, ths, rules, mode):
        doc_entities = {}
        for s in self.sentences:
            if s.entities:
                if mode == "ner":
                    sentence_entitites = s.entities.get_unique_entities(source, ths, rules)
                    for e in sentence_entitites:
                        sentence_entitites[e].append(s.text[int(sentence_entitites[e][1]):int(sentence_entitites[e][2])])
                    # print sentence_entitites
                elif mode == "re":
                    sentence_entitites = s.entities.get_unique_relations(source)
            # print doc_entities, sentence_entitites
            doc_entities.update(sentence_entitites)
            # print doc_entities
            # print
        logging.info("{} has {} unique entities".format(self.did, len(doc_entities)))
        return doc_entities

    def write_chemdner_results(self, source, outfile, ths={"chebi":0.0}, rules=[]):
        lines = []
        totalentities = 0
        for s in self.sentences:
            # print "processing", s.sid, "with", len(s.entities.elist[source]), "entities"
            if s.entities:
                res = s.entities.write_chemdner_results(source, outfile, ths, rules, totalentities+1)
                lines += res[0]
                totalentities = res[1]
        return lines

    def write_bioc_results(self, parent, source, ths={}):
        bioc_document = ET.SubElement(parent, "document")
        bioc_id = ET.SubElement(bioc_document, "id")
        bioc_id.text = self.did

        bioc_title_passage = ET.SubElement(bioc_document, "passage")
        bioc_title_info = ET.SubElement(bioc_title_passage, "infon", {"key":"type"})
        bioc_title_info.text = "title"
        bioc_title_offset = ET.SubElement(bioc_title_passage, "offset")
        bioc_title_offset.text = str(0)
        bioc_title = self.sentences[0].write_bioc_results(bioc_title_passage, source)

        bioc_abstract_passage = ET.SubElement(bioc_document, "passage")
        bioc_abstract_info = ET.SubElement(bioc_abstract_passage, "infon", {"key":"type"})
        bioc_abstract_info.text = "abstract"
        bioc_abstract_offset = ET.SubElement(bioc_title_passage, "offset")
        bioc_abstract_offset.text = str(len(self.sentences[0].text) + 1)
        for i, sentence in enumerate(self.sentences[1:]):
            bioc_sentence = sentence.write_bioc_results(bioc_abstract_passage, source)
        return bioc_document

    def get_dic(self, source, ths={}):
        dic = {"title":{}, "abstract":{}}
        dic = {"abstract":{}}
        # dic["title"]["offset"] = "0"
        # dic["title"]["sentences"] = self.sentences[0].get_dic(source)

        dic["abstract"]["offset"] = str(len(self.sentences[0].text) + 1)
        dic["abstract"]["sentences"] = []
        for i, sentence in enumerate(self.sentences[1:]):
            dic["abstract"]["sentences"].append(sentence.get_dic(source))
        return dic

    def get_sentence(self, sid):
        """
        Get the sentence by sentence ID
        :param sid: sentence ID
        :return: the sentence object if it exists
        """
        for s in self.sentences:
            # logging.debug([(t.start, t.end) for t in s.tokens])
            if s.sid == sid:
                # logging.debug("found sid: {}".format(sid))
                return s
        return None

    def find_sentence_containing(self, start, end, chemdner=True):
        """
            Find the sentence between start and end. If chemdner, do not consider the first sentence, which
            is the title.
        """
        if chemdner:
            firstsent = 1
        else:
            firstsent = 0
        for i, s in enumerate(self.sentences[firstsent:]):
            if len(s.tokens) == 0:
                logging.debug("sentence without tokens: {} {}".format(s.sid, s.text))
                continue
            if s.tokens[0].dstart <= start and s.tokens[-1].dend >= end:
                # print "found it!"
                return s
        for s in self.sentences:
            logging.debug("{} {} {} {} {}".format(s.tokens[0].dstart <= start, s.tokens[-1].dend >= end,
                                                s.tokens[0].dstart, s.tokens[-1].dend, s.text))
        return None

    def get_entity_offsets(self, esource, ths, rules):
        offsets = []
        for s in self.sentences:
            if s.entities:
                offsets += s.entities.get_entity_offsets(esource, ths, rules)
        return offsets

    def get_entity(self, eid, source="goldstandard"):
        for sentence in self.sentences:
            for e in sentence.entities.elist[source]:
                if e.eid == eid:
                   return e
        print "no entity found for eid {}".format(eid)
        return None

    def get_entities(self, source):
        entities = []
        for s in self.sentences:
            if source in s.entities.elist:
                for e in s.entities.elist[source]:
                    entities.append(e)
        return entities

    def get_abbreviations(self):
        self.abbreviations = {}
        first_elem = []
        second_elem = []
        open_paren = False
        for sentence in self.sentences:
            # print sentence.text
            for i, t in enumerate(sentence.tokens):
                if t.text == "-LRB-":
                    open_paren = True
                    last_token = sentence.tokens[i-1]
                    while last_token.pos.startswith("NN") or last_token.pos.startswith("JJ"): # use nouns before the parenthesis
                        first_elem.insert(0, last_token)
                        if last_token.order == 0:
                            break
                        else:
                            last_token = sentence.tokens[last_token.order - 1]  # check the token before this one
                    if len(first_elem) > 0:
                        logging.info("starting abbreviation for this text: " + str([tt.text for tt in first_elem]))
                    else:
                        open_paren = False
                elif t.text == "-RRB-" and open_paren == True:
                    first_text = sentence.text[first_elem[0].start:first_elem[-1].end]
                    second_text = sentence.text[second_elem[0].start:second_elem[-1].end]
                    if len(first_text) > len(second_text): #abbreviation is the smallest word
                        second_text, first_text = first_text, second_text
                    # rules
                    if not first_text.islower() and len(first_text) > 1:
                        self.abbreviations[first_text] = second_text
                    open_paren = False
                    first_elem = []
                    second_elem = []
                elif open_paren:
                    second_elem.append(t)
        for abv in self.abbreviations:
            if not any([c.isalpha() for c in abv]):
                print abv, ":", self.abbreviations[abv]
示例#3
0
class Document(object):
    """A document is constituted by one or more sentences. It should have an ID and
    title. s0, the first sentence, is always the title sentence."""
    def __init__(self,
                 text,
                 process=False,
                 doctype="biomedical",
                 ssplit=False,
                 **kwargs):
        self.text = text
        self.title = kwargs.get("title")
        self.sentences = kwargs.get("sentences", [])
        self.did = kwargs.get("did", "d0")
        self.invalid_sids = []
        self.title_sids = []
        self.pairs = Pairs()
        if ssplit:
            self.sentence_tokenize(doctype)
        if process:
            self.process_document(doctype)

    def sentence_tokenize(self, doctype):
        """
        Split the document text into sentences, add to self.sentences list
        :param doctype: Can be used in the future to choose different methods
        """
        # first sentence should be the title if it exists
        if self.title:
            sid = self.did + ".s0"
            self.sentences.append(Sentence(self.title, sid=sid, did=self.did))
        # inputtext = clean_whitespace(self.text)
        inputtext = self.text
        with codecs.open("/tmp/geniainput.txt", 'w', 'utf-8') as geniainput:
            geniainput.write(inputtext)
        current_dir = os.getcwd()
        os.chdir(geniass_path)
        geniaargs = [
            "./geniass", "/tmp/geniainput.txt", "/tmp/geniaoutput.txt"
        ]
        Popen(geniaargs, stdout=PIPE, stderr=PIPE).communicate()
        os.chdir(current_dir)
        offset = 0
        with codecs.open("/tmp/geniaoutput.txt", 'r', "utf-8") as geniaoutput:
            for l in geniaoutput:
                stext = l.strip()
                if stext == "":
                    offset = self.get_space_between_sentences(offset)
                    continue
                sid = self.did + ".s" + str(len(self.sentences))
                self.sentences.append(
                    Sentence(stext, offset=offset, sid=sid, did=self.did))
                offset += len(stext)
                offset = self.get_space_between_sentences(offset)

    def process_document(self, corenlpserver, doctype="biomedical"):
        """
        Process each sentence in the text (sentence split if there are no sentences) using Stanford CoreNLP
        :param corenlpserver:
        :param doctype:
        :return:
        """
        if len(self.sentences) == 0:
            # use specific sentence splitter
            self.sentence_tokenize(doctype)
        for s in self.sentences:
            #corenlpres = corenlpserver.raw_parse(s.text)
            corenlpres = corenlpserver.annotate(
                s.text.encode("utf8"),
                properties={
                    'ssplit.eolonly': True,
                    # 'annotators': 'tokenize,ssplit,pos,depparse,parse',
                    'annotators':
                    'tokenize,ssplit,pos,parse,ner,lemma,depparse',
                    'gazetteer':
                    '/scr/nlp/data/machine-reading/Machine_Reading_P1_Reading_Task_V2.0/data/SportsDomain/NFLScoring_UseCase/NFLgazetteer.txt',
                    'outputFormat': 'json',
                })
            if isinstance(corenlpres, basestring):
                print corenlpres
                corenlpres = corenlpserver.annotate(
                    s.text.encode("utf8"),
                    properties={
                        'ssplit.eolonly': True,
                        # 'annotators': 'tokenize,ssplit,pos,depparse,parse',
                        'nfl.gazetteer':
                        '/scr/nlp/data/machine-reading/Machine_Reading_P1_Reading_Task_V2.0/data/SportsDomain/NFLScoring_UseCase/NFLgazetteer.txt',
                        'annotators': 'tokenize,ssplit,pos,ner,lemma',
                        'outputFormat': 'json',
                    })
            s.process_corenlp_sentence(corenlpres)

    def tag_chemdner_entity(self, start, end, subtype, **kwargs):
        """
        Create an CHEMDNER entity relative to this document.
        :param start: Start index of entity
        :param end: End index of entity
        :param subtype: Subtype of CHEMDNER entity
        :param kwargs: Extra stuff like the text
        :return:
        """
        doct = kwargs.get("doct")
        if doct == "T":  # If it's in the title, we already know the sentence (it's the first)
            self.sentences[0].tag_entity(start, end, subtype, **kwargs)
        else:  # we have to find the sentence
            found = False
            totalchars = 0
            for s in self.sentences[1:]:
                if totalchars <= start and totalchars + len(
                        s.text) >= end:  # entity is in this sentence
                    s.tag_entity(start - totalchars,
                                 end - totalchars,
                                 subtype,
                                 totalchars=totalchars,
                                 **kwargs)
                    # print "found entity on sentence %s" % s.sid
                    found = True
                    break

                totalchars += len(s.text)
                totalchars = self.get_space_between_sentences(totalchars)
            if not found:
                print "could not find sentence for %s:%s on %s!" % (start, end,
                                                                    self.did)
                # sys.exit()

    def add_relation(self,
                     entity1,
                     entity2,
                     subtype,
                     relation,
                     source="goldstandard",
                     **kwargs):
        if self.pairs.pairs:
            pid = self.did + ".p" + str(len(self.pairs.pairs))
        else:
            pid = self.did + ".p0"
        between_text = self.text[entity1.dend:entity2.start]
        logging.info("adding {}:{}=>{}".format(pid,
                                               entity1.text.encode("utf8"),
                                               entity2.text.encode("utf8")))
        # print between_text
        if subtype == "tlink":
            pair = TLink(entity1,
                         entity2,
                         relation=relation,
                         original_id=kwargs.get("original_id"),
                         did=self.did,
                         pid=pid,
                         rtype=subtype,
                         between_text=between_text)
        else:
            pair = Pair((entity1, entity2),
                        subtype,
                        did=self.did,
                        pid=pid,
                        original_id=kwargs.get("original_id"),
                        between_text=between_text)
        self.pairs.add_pair(pair, source)
        return pair

    def get_space_between_sentences(self, totalchars):
        """
        When the sentences are split, the whitespace between each sentence is not preserved, so we need to get it back
        :param totalchars: offset of the end of sentence
        :return: Index where the next sentence starts
        """
        while totalchars < len(self.text) and self.text[totalchars].isspace():
            totalchars += 1
        return totalchars

    def get_unique_results(self, source, ths, rules, mode):
        entries = set()
        for s in self.sentences:
            if s.entities:
                if mode == "ner":
                    sentence_entries = s.entities.get_unique_entities(
                        source, ths, rules)
                elif mode == "re":
                    sentence_entries = s.entities.get_unique_relations(source)
                entries.update(sentence_entries)
        return entries

    def write_chemdner_results(self,
                               source,
                               outfile,
                               ths={"chebi": 0.0},
                               rules=[]):
        lines = []
        totalentities = 0
        for s in self.sentences:
            # print "processing", s.sid, "with", len(s.entities.elist[source]), "entities"
            if s.entities:
                res = s.entities.write_chemdner_results(
                    source, outfile, ths, rules, totalentities + 1)
                lines += res[0]
                totalentities = res[1]
        return lines

    def write_bioc_results(self, parent, source, ths={}):
        bioc_document = ET.SubElement(parent, "document")
        bioc_id = ET.SubElement(bioc_document, "id")
        bioc_id.text = self.did

        bioc_title_passage = ET.SubElement(bioc_document, "passage")
        bioc_title_info = ET.SubElement(bioc_title_passage, "infon",
                                        {"key": "type"})
        bioc_title_info.text = "title"
        bioc_title_offset = ET.SubElement(bioc_title_passage, "offset")
        bioc_title_offset.text = str(0)
        bioc_title = self.sentences[0].write_bioc_results(
            bioc_title_passage, source)

        bioc_abstract_passage = ET.SubElement(bioc_document, "passage")
        bioc_abstract_info = ET.SubElement(bioc_abstract_passage, "infon",
                                           {"key": "type"})
        bioc_abstract_info.text = "abstract"
        bioc_abstract_offset = ET.SubElement(bioc_title_passage, "offset")
        bioc_abstract_offset.text = str(len(self.sentences[0].text) + 1)
        for i, sentence in enumerate(self.sentences[1:]):
            bioc_sentence = sentence.write_bioc_results(
                bioc_abstract_passage, source)
        return bioc_document

    def get_dic(self, source, ths={}):
        dic = {"title": {}, "abstract": {}}
        dic = {"abstract": {}}
        # dic["title"]["offset"] = "0"
        # dic["title"]["sentences"] = self.sentences[0].get_dic(source)

        dic["abstract"]["offset"] = str(len(self.sentences[0].text) + 1)
        dic["abstract"]["sentences"] = []
        for i, sentence in enumerate(self.sentences[1:]):
            dic["abstract"]["sentences"].append(sentence.get_dic(source))
        return dic

    def get_sentence(self, sid):
        """
        Get the sentence by sentence ID
        :param sid: sentence ID
        :return: the sentence object if it exists
        """
        for s in self.sentences:
            # logging.debug([(t.start, t.end) for t in s.tokens])
            if s.sid == sid:
                # logging.debug("found sid: {}".format(sid))
                return s
        return None

    def find_sentence_containing(self, start, end, chemdner=True):
        """
            Find the sentence between start and end. If chemdner, do not consider the first sentence, which
            is the title.
        """
        if chemdner:
            firstsent = 1
        else:
            firstsent = 0
        for i, s in enumerate(self.sentences[firstsent:]):
            if len(s.tokens) == 0:
                logging.debug("sentence without tokens: {} {}".format(
                    s.sid, s.text))
                continue
            if s.tokens[0].dstart <= start and s.tokens[-1].dend >= end:
                # print "found it!"
                return s
        for s in self.sentences:
            print s.tokens[0].dstart <= start, s.tokens[
                -1].dend >= end, s.tokens[0].dstart, s.tokens[-1].dend, s.text
        return None

    def get_offsets(self, esource, ths, rules, off_list=None):
        #print esource

        offsets = []
        for s in self.sentences:
            #print s.text
            offies = gazette.easy_search_terms(s, esource, ths, rules,
                                               off_list)
            if len(offies) == 1:
                offsets += offies  #Check it doesn't affect normal results
            else:
                if s.entities:
                    offsets += s.entities.get_offsets2(esource, ths, rules)
                    offsets += offies

        return list(set(offsets))

    def get_entity(self, eid, source="goldstandard"):
        for sentence in self.sentences:
            for e in sentence.entities.elist[source]:
                if e.eid == eid:
                    return e
        print "no entity found for eid {}".format(eid)
        return None

    def get_entities(self, source):
        entities = []
        for s in self.sentences:
            if source in s.entities.elist:
                for e in s.entities.elist[source]:
                    entities.append(e)
        return entities
示例#4
0
class Sentence(object):
    """Sentence from a document, to be annotated"""
    def __init__(self, text, offset=0, **kwargs):
        self.text = text
        self.sid = kwargs.get("sid")
        self.did = kwargs.get("did")
        self.entities = Entities(sid=self.sid, did=self.did)
        self.offset = offset
        self.pairs = Pairs()
        self.parsetree = None
        self.depparse = None
        self.tokens = []
        self.regex_tokens = re.compile(r'(-|/|\\|\+|\.|\w+)')

    def tokenize_words(self):
        pass

    def process_sentence(self, corenlpserver, doctype="biomedical"):
        corenlpres = corenlpserver.annotate(self.text.encode("utf8"), properties={
            'ssplit.eolonly': True,
            # 'annotators': 'tokenize,ssplit,pos,ner,lemma',
            'annotators': 'tokenize,ssplit,pos,parse,ner,lemma,depparse',
            'outputFormat': 'json',
        })
        if isinstance(corenlpres, basestring):
            print corenlpres
            corenlpres = corenlpserver.annotate(self.text.encode("utf8"), properties={
                'ssplit.eolonly': True,
                # 'annotators': 'tokenize,ssplit,pos,depparse,parse',
                'annotators': 'tokenize,ssplit,pos,ner,lemma',
                'outputFormat': 'json',
            })
        if isinstance(corenlpres, basestring):
            print "could not process this sentence:", self.text.encode("utf8")
            print corenlpres
        else:
            self.process_corenlp_output(corenlpres)
        return corenlpres

    def process_corenlp_output(self, corenlpres):

        """
        Process the results obtained with CoreNLP for this sentence
        :param corenlpres:
        :return:
        """
        # self.sentences = []
        if len(corenlpres['sentences']) > 1:
            print self.text
            sys.exit("Number of sentences from CoreNLP is not 1.")
        if len(corenlpres['sentences']) == 0:
            self.tokens = []
            self.create_newtoken("", {})
            logging.debug("no sentences")
            logging.debug(self.text)
            return
        sentence = corenlpres['sentences'][0]
        #logging.debug(str(sentence.keys()))
        #print "sentence", self.text.encode("utf8")
        #print "parse", pp.pprint(sentence["parse"])
        #print "basic", pp.pprint(sentence["basic-dependencies"])
        #print "collapsed", pp.pprint(sentence["collapsed-dependencies"])
        #print "ccprocessed", pp.pprint(sentence["collapsed-ccprocessed-dependencies"])
        self.parsetree = sentence.get('parse')
        self.depparse = sentence.get('basic-dependencies')
        for t in sentence['tokens']:
            # print t[0]
            if t["word"]:
                # TODO: specific rules for each corpus
                #if ""
                token_seq = self.regex_tokens.split(t["word"])#, flags=re.U)
                #token_seq = rext.split(r'(\w+)(/|\\|\+|\.)(\w+)', t[0])
                #token_seq = [t[0]]
                # print t[0], token_seq
                if len(token_seq) > 3 and t["word"] not in stanford_coding.keys():
                    # logging.info("{}: {}".format(t["word"], "&".join(token_seq)))
                    for its, ts in enumerate(token_seq):
                        if ts.strip() != "":
                            charoffset_begin = int(t["characterOffsetBegin"])
                            if token_seq[:its]: # not the first token
                                charoffset_begin += sum([len(x) for x in token_seq[:its]])
                            # charoffset_begin += its
                            charoffset_end = len(ts) + charoffset_begin
                            #logging.info(str(charoffset_begin) + ":" + str(charoffset_end))
                            ts_props = {"characterOffsetBegin": charoffset_begin,
                                        "characterOffsetEnd": charoffset_end,
                                        "pos": t["pos"],
                                        "ner": t["ner"],
                                        "lemma": t["lemma"][charoffset_begin:charoffset_end]}
                            self.create_newtoken(ts, ts_props)

                else:
                    self.create_newtoken(t["word"], t)

    def create_newtoken(self, text, props):
        newtoken = Token2(text, order=len(self.tokens))
        try:
            newtoken.start = int(props["characterOffsetBegin"])
            newtoken.dstart = newtoken.start + self.offset
            newtoken.end = int(props["characterOffsetEnd"])
            newtoken.dend = newtoken.end + self.offset
            newtoken.pos = props["pos"]
            newtoken.tag = props["ner"]
            newtoken.lemma = props["lemma"]
            # newtoken.stem = porter.stem_word(newtoken.text)
            newtoken.tid = self.sid + ".t" + str(len(self.tokens))
            self.tokens.append(newtoken)
            # print "|{}| <=> |{}|".format(text, self.text[newtoken.start:newtoken.end])
        except KeyError:
            logging.debug("error: text={} props={}".format(text, props))
            return None
        # logging.debug(newtoken.text)
        return newtoken

    def add_relation(self, entity1, entity2, subtype, source="goldstandard", **kwargs):
        if self.pairs.pairs:
            pid = self.sid + ".p" + str(len(self.pairs.pairs))
        else:
            pid = self.sid + ".p0"
        if subtype == "tlink":
            p = TLink(entity1, entity2, original_id=kwargs.get("original_id"),
                                     did=self.did, pid=pid, rtype=subtype)
        else:
            p = Pair((entity1, entity2), subtype, pid=pid, sid=self.sid, did=self.did)
        self.pairs.add_pair(p, source)
        return p

    def exclude_entity(self, start, end, source):
        """
        Exclude all entities matching start-end relative to sentence
        :param start:
        :param end:
        """
        to_delete = []
        for e in self.entities.elist[source]:
            if e.start == start and e.end == end:
                to_delete.append(e)
                for t in e.tokens:
                    tagkeys = t.tags.keys()
                    for tag in tagkeys:
                        if tag.startswith(source):
                            del t.tags[tag]
        for e in to_delete:
            #print "removing {}".format(e)
            self.entities.elist[source].remove(e)
            #print [(ee.start, ee.end) for ee in self.entities.elist[source]]


    def tag_entity(self, start, end, etype, entity=None, source="goldstandard", exclude=None, **kwargs):
        """Find the tokens that match this entity. start and end are relative to the sentence.
           Totalchars is the offset of the sentence on the document."""
        tlist = []
        # print self.tokens
        nextword = ""
        for t in self.tokens:
            # discard tokens that intersect the entity for now
            # print t.start, t.end, t.text
            if t.start >= start and t.end <= end:
                tlist.append(t)
            elif (t.start == start and t.end > end) or (t.start < start and t.end == end):
                tlist.append(t)
                break
            elif t.start == end+1:
                nextword = t.text
            exclude_list = []
            if exclude is not None:
                for t in tlist:
                    for e in exclude:
                        if t.start >= e[0] and t.end <= e[1]-1:
                            exclude_list.append(t.tid)
            tlist = [t for t in tlist if t.tid not in exclude_list]
        if tlist:
            if exclude is not None:
                newtext = self.text[tlist[0].start:exclude[0][0]]
                #print self.text[exclude[0][0]:exclude[0][1]], exclude
                last_exclude = exclude[0]
                for e in exclude[1:]:
                    if not self.text[e[1]].isspace() and not newtext[-1].isspace():
                        newtext += " "
                    newtext += self.text[last_exclude[1]:e[0]]
                    last_exclude = e
                if not self.text[exclude[-1][1]].isspace() and not newtext[-1].isspace():
                    newtext += " "
                newtext += self.text[exclude[-1][1]:tlist[-1].end]
                # self.text[exclude[1]:tlist[-1].end]
            else:
                newtext = self.text[tlist[0].start:tlist[-1].end]
            if entity:
                entity.text = newtext
            if "text" in kwargs and newtext != kwargs["text"]:
                if newtext not in kwargs["text"] and kwargs["text"] not in newtext:
                    logging.info("diferent text:|system {} {} |{}|=>|{}| {} {} input|{} {}".format(tlist[0].start, tlist[-1].end, newtext, kwargs["text"],
                                                                                                   start, end, self.sid,
                                                                                                   self.text))
                    logging.info("text does not match: {}=>{}".format(newtext, kwargs["text"]))
                    #sys.exit()
                    return None
                else:
                    logging.info("diferent text:|system {} {} |{}|=>|{}| {} {} input|{} {}".format(tlist[0].start, tlist[-1].end, newtext, kwargs["text"],
                                 start, end, self.sid, self.text))
                    return None
                    # print exclude, self.text[tlist[0].start:tlist[-1].end]
            #     print "tokens found:", [t.text for t in tlist]
                    # sys.exit()
            # else:
            # print "found the tokens!", start, end, kwargs["text"], self.sid

            if self.entities.elist.get(source):
                eid = self.sid + ".e" + str(len(self.entities.elist[source]))
            else:
                eid = self.sid + ".e0"
            if entity is None:
                if "text" in kwargs:
                    newtext = kwargs["text"]
                entity = create_entity(tlist, self.sid, did=self.did, text=newtext, score=kwargs.get("score"),
                                       etype=etype, eid=eid, subtype=kwargs.get("subtype"),
                                       original_id=kwargs.get("original_id"), nextword=nextword)

                entity.normalize()
            self.entities.add_entity(entity, source)
            self.label_tokens(tlist, source, etype)
            #logging.debug("added {} to {}, now with {} entities".format(newtext, self.sid,
            #                                                                 len(self.entities.elist[source])))
            return eid
        else:
            logging.info("no tokens found:")
            logging.info("{} {} {} {}".format(self.sid, start, end, kwargs.get("text")))
            logging.info(str([(t.start, t.end, t.text) for t in self.tokens]))

    def label_tokens(self, tlist, source, etype):
        if len(tlist) == 1:
            tlist[0].tags[source] = "single"
            tlist[0].tags[source + "_subtype"] = etype
            tlist[0].tags[source + "_" + etype] = "single"
        else:
            for t in range(len(tlist)):
                if t == 0:
                    tlist[t].tags[source] = "start"
                    tlist[t].tags[source + "_" + etype] = "start"
                    tlist[t].tags[source + "_subtype"] = etype
                elif t == len(tlist) - 1:
                    tlist[t].tags[source] = "end"
                    tlist[t].tags[source + "_" + etype] = "end"
                    tlist[t].tags[source + "_subtype"] = etype
                else:
                    tlist[t].tags[source] = "middle"
                    tlist[t].tags[source + "_" + etype] = "middle"
                    tlist[t].tags[source + "_subtype"] = etype
        # logging.debug([t.tags for t in tlist])

    def write_bioc_results(self, parent, source):
        bioc_sentence = ET.SubElement(parent, "sentence")
        bioc_sentence_offset = ET.SubElement(bioc_sentence, "offset")
        bioc_sentence_offset.text = str(self.tokens[0].dstart)
        bioc_sentence_text = ET.SubElement(bioc_sentence, "text")
        bioc_sentence_text.text = self.text

        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                bioc_annotation = entity.write_bioc_annotation(bioc_sentence)
        return bioc_sentence

    def get_dic(self, source):
        dic = {}
        dic["id"] = self.sid
        dic["offset"] = str(self.tokens[0].dstart)
        dic["text"] = self.text
        dic["entities"] = []
        if source in self.entities.elist:
            for entity in self.entities.elist[source]:
                dic["entities"].append(entity.get_dic())
            dic["entities"] = sorted(dic["entities"], key=lambda k: k['offset'])
            for ei, e in enumerate(dic["entities"]):
                e["eid"] = self.sid + ".e{}".format(ei)
        elif source == "all":
            offsets = Offsets()
            for esource in self.entities.elist:
                for entity in self.entities.elist[esource]:
                    toadd, v, overlapping, to_exclude = offsets.add_offset(Offset(entity.start, entity.end),
                                                                           exclude_this_if=[1, -1, 2, -3],
                                                                           exclude_others_if=[2])
                    if toadd:
                        dic["entities"].append(entity.get_dic())
                dic["entities"] = sorted(dic["entities"], key=lambda k: k['offset'])
                for ei, e in enumerate(dic["entities"]):
                    e["eid"] = self.sid + ".e{}".format(ei)
        dic["pairs"] = self.pairs.get_dic()
        return dic

    def find_tokens(self, text, start, end, count, relativeto="doc"):
        candidates = []
        for t in self.tokens:
            if t.text == text:
                print t.text, text
                candidates.append(t)
        print text, candidates
        if len(candidates) == 0:
            print "could not find tokens!"
        elif len(candidates) == 1:
            return candidates
        elif len(candidates)-1 > count:
            candidates[count]
        """else:
            dist = []
            for c in candidates:
                if relativeto == "doc":
                    d = c.dstart
                else:
                    d = c.start
                dist.append(abs(d-start))
            return [candidates[dist.index(min(dist))]]"""

    def find_tokens_between(self, start, end, relativeto="doc"):
        """Return list of tokens between offsets. Use relativeto to consider doc indexes or
           sentence indexes."""
        foundtokens = []
        for t in self.tokens:
            if relativeto.startswith("doc") and t.dstart >= start and t.dend <= end:
                foundtokens.append(t)
            elif relativeto.startswith("sent") and t.start >= start and t.end <= end:
                foundtokens.append(t)
        return foundtokens

    def test_relations(self, pairs, basemodel, classifiers=[relations.SLK_PRED, relations.SST_PRED],
                       tag="", backup=False, printstd=False):
        #data =  ddi_train_slk.model, ddi_train_sst.model
        tempfiles = []

        if relations.SLK_PRED in classifiers:
            logging.info("**Testing SLK classifier %s ..." % (tag,))
            #testpairdic = ddi_kernels.fromddiDic(testdocs)
            ddi_kernels.generatejSREdata(pairs, self, basemodel, tag + "ddi_test_jsre.txt")
            ddi_kernels.testjSRE(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                 model=tag + "all_ddi_train_slk.model")
            self.pairs.pairs = ddi_kernels.getjSREPredicitons(tag + "ddi_test_jsre.txt", tag + "ddi_test_result.txt",
                                                      self.pairs.pairs)
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_jsre.txt")
            tempfiles.append(ddi_kernels.basedir + tag + "ddi_test_result.txt")

        if relations.SST_PRED in classifiers:
            logging.info("****Testing SST classifier %s ..." % (tag,))
            self.pairs.pairs = ddi_kernels.testSVMTK(self, self.pairs.pairs, pairs,
                                             model=tag + "all_ddi_train_sst.model", tag=tag)
        for p in self.pairs.pairs:
            for r in self.pairs.pairs[p].recognized_by:
                if self.pairs.pairs[p].recognized_by[r] == 1:
                    p.relation = True
        return tempfiles
示例#5
0
class Document(object):
    """A document is constituted by one or more sentences. It should have an ID and
    title. s0, the first sentence, is always the title sentence."""
    def __init__(self,
                 text,
                 process=False,
                 doctype="biomedical",
                 ssplit=False,
                 **kwargs):
        self.text = text
        self.title = kwargs.get("title")
        self.sentences = kwargs.get("sentences", [])
        self.did = kwargs.get("did", "d0")
        self.invalid_sids = []
        self.title_sids = []
        self.source = kwargs.get("source")
        self.pairs = Pairs()
        if ssplit:
            self.sentence_tokenize(doctype)
        if process:
            self.process_document(doctype)

    def sentence_tokenize(self, doctype):
        """
        Split the document text into sentences, add to self.sentences list
        :param doctype: Can be used in the future to choose different methods
        """
        # first sentence should be the title if it exists
        #if self.title:
        #    sid = self.did + ".s0"
        #    self.sentences.append(Sentence(self.title, sid=sid, did=self.did))
        # inputtext = clean_whitespace(self.text)
        inputtext = self.text
        with io.open("/tmp/geniainput.txt", 'w',
                     encoding='utf-8') as geniainput:
            geniainput.write(inputtext)
        current_dir = os.getcwd()
        os.chdir(geniass_path)
        geniaargs = [
            "./geniass", "/tmp/geniainput.txt", "/tmp/geniaoutput.txt"
        ]
        Popen(geniaargs, stdout=PIPE, stderr=PIPE).communicate()
        os.chdir(current_dir)
        offset = 0
        with io.open("/tmp/geniaoutput.txt", 'r',
                     encoding="utf-8") as geniaoutput:
            for l in geniaoutput:
                stext = l.strip()
                if stext == "":
                    offset = self.get_space_between_sentences(offset)
                    continue
                sid = self.did + ".s" + str(len(self.sentences))
                self.sentences.append(
                    Sentence(stext, offset=offset, sid=sid, did=self.did))
                offset += len(stext)
                offset = self.get_space_between_sentences(offset)

    def process_document(self, corenlpserver, doctype="biomedical"):
        """
        Process each sentence in the text (sentence split if there are no sentences) using Stanford CoreNLP
        :param corenlpserver:
        :param doctype:
        :return:
        """
        if len(self.sentences) == 0:
            # use specific sentence splitter
            self.sentence_tokenize(doctype)
        for s in self.sentences:
            #corenlpres = corenlpserver.raw_parse(s.text)
            corenlpres = corenlpserver.annotate(
                s.text.encode("utf8"),
                properties={
                    'ssplit.eolonly': True,
                    #'annotators': 'tokenize,ssplit,pos,ner,lemma',
                    'annotators':
                    'tokenize,ssplit,pos,parse,ner,lemma,depparse',
                    'outputFormat': 'json',
                })
            if isinstance(corenlpres, basestring):
                print corenlpres
                corenlpres = corenlpserver.annotate(
                    s.text.encode("utf8"),
                    properties={
                        'ssplit.eolonly': True,
                        # 'annotators': 'tokenize,ssplit,pos,depparse,parse',
                        'annotators': 'tokenize,ssplit,pos,ner,lemma',
                        'outputFormat': 'json',
                    })
            if isinstance(corenlpres, basestring):
                print "could not process this sentence:", s.text.encode("utf8")
                print corenlpres
                continue
            else:
                s.process_corenlp_output(corenlpres)

    def tag_chemdner_entity(self,
                            start,
                            end,
                            subtype,
                            source="goldstandard",
                            **kwargs):
        """
        Create an CHEMDNER entity relative to this document.
        :param start: Start index of entity
        :param end: End index of entity
        :param subtype: Subtype of CHEMDNER entity
        :param kwargs: Extra stuff like the text
        :return:
        """
        doct = kwargs.get("doct")
        title_offset = 0
        if doct == "A":
            title_offset = len(self.title) + 1  # account for extra .
        start, end = start + title_offset, end + title_offset
        sentence = self.find_sentence_containing(start, end, chemdner=False)
        if sentence:
            eid = sentence.tag_entity(start - sentence.offset,
                                      end - sentence.offset,
                                      "chemical",
                                      source=source,
                                      text=kwargs.get("text"),
                                      subtype=subtype,
                                      score=kwargs.get("score"))
            if eid:
                entity = sentence.entities.get_entity(eid, source)
                return entity
        else:
            print "sentence not found between:", start, end
            print "ignored ", kwargs.get("text")
            # print len(self.documents[pmid].title), self.documents[pmid].title
            # for s in self.documents[pmid].sentences:
            #    print s.sid, s.tokens[0].dstart, s.tokens[-1].dend, s.text

    def add_relation(self,
                     entity1,
                     entity2,
                     subtype,
                     relation,
                     source="goldstandard",
                     **kwargs):
        if self.pairs.pairs:
            pid = self.did + ".p" + str(len(self.pairs.pairs))
        else:
            pid = self.did + ".p0"
        between_text = self.text[entity1.dend:entity2.start]
        logging.debug("adding {}:{}=>{}".format(pid,
                                                entity1.text.encode("utf8"),
                                                entity2.text.encode("utf8")))
        # print between_text
        if subtype == "tlink":
            pair = TLink(entity1,
                         entity2,
                         relation=relation,
                         original_id=kwargs.get("original_id"),
                         did=self.did,
                         pid=pid,
                         rtype=subtype,
                         between_text=between_text)
        else:
            pair = Pair((entity1, entity2),
                        subtype,
                        did=self.did,
                        pid=pid,
                        original_id=kwargs.get("original_id"),
                        between_text=between_text)
        self.pairs.add_pair(pair, source)
        return pair

    def get_space_between_sentences(self, totalchars):
        """
        When the sentences are split, the whitespace between each sentence is not preserved, so we need to get it back
        :param totalchars: offset of the end of sentence
        :return: Index where the next sentence starts
        """
        while totalchars < len(self.text) and self.text[totalchars].isspace():
            totalchars += 1
        return totalchars

    def get_unique_results(self, source, ths, rules, mode):
        doc_entities = {}
        for s in self.sentences:
            if s.entities:
                if mode == "ner":
                    sentence_entitites = s.entities.get_unique_entities(
                        source, ths, rules)
                    for e in sentence_entitites:
                        sentence_entitites[e].append(
                            s.text[int(sentence_entitites[e][1]
                                       ):int(sentence_entitites[e][2])])
                    # print sentence_entitites
                elif mode == "re":
                    sentence_entitites = s.entities.get_unique_relations(
                        source)
            # print doc_entities, sentence_entitites
            doc_entities.update(sentence_entitites)
            # print doc_entities
            # print
        logging.info("{} has {} unique entities".format(
            self.did, len(doc_entities)))
        return doc_entities

    def write_chemdner_results(self,
                               source,
                               outfile,
                               ths={"chebi": 0.0},
                               rules=[]):
        lines = []
        totalentities = 0
        for s in self.sentences:
            # print "processing", s.sid, "with", len(s.entities.elist[source]), "entities"
            if s.entities:
                res = s.entities.write_chemdner_results(
                    source, outfile, len(self.sentences[0].text), ths, rules,
                    totalentities + 1)
                lines += res[0]
                totalentities = res[1]
        return lines

    def write_bioc_results(self, parent, source, ths={}):
        bioc_document = ET.SubElement(parent, "document")
        bioc_id = ET.SubElement(bioc_document, "id")
        bioc_id.text = self.did

        bioc_title_passage = ET.SubElement(bioc_document, "passage")
        bioc_title_info = ET.SubElement(bioc_title_passage, "infon",
                                        {"key": "type"})
        bioc_title_info.text = "title"
        bioc_title_offset = ET.SubElement(bioc_title_passage, "offset")
        bioc_title_offset.text = str(0)
        bioc_title = self.sentences[0].write_bioc_results(
            bioc_title_passage, source)

        bioc_abstract_passage = ET.SubElement(bioc_document, "passage")
        bioc_abstract_info = ET.SubElement(bioc_abstract_passage, "infon",
                                           {"key": "type"})
        bioc_abstract_info.text = "abstract"
        bioc_abstract_offset = ET.SubElement(bioc_title_passage, "offset")
        bioc_abstract_offset.text = str(len(self.sentences[0].text) + 1)
        for i, sentence in enumerate(self.sentences[1:]):
            bioc_sentence = sentence.write_bioc_results(
                bioc_abstract_passage, source)
        return bioc_document

    def get_dic(self, source, ths={}):
        dic = {"title": {}, "abstract": {}}
        dic = {"abstract": {}}
        # dic["title"]["offset"] = "0"
        # dic["title"]["sentences"] = self.sentences[0].get_dic(source)

        dic["abstract"]["offset"] = str(len(self.sentences[0].text) + 1)
        dic["abstract"]["sentences"] = []
        for i, sentence in enumerate(self.sentences[1:]):
            dic["abstract"]["sentences"].append(sentence.get_dic(source))
        return dic

    def get_sentence(self, sid):
        """
        Get the sentence by sentence ID
        :param sid: sentence ID
        :return: the sentence object if it exists
        """
        for s in self.sentences:
            # logging.debug([(t.start, t.end) for t in s.tokens])
            if s.sid == sid:
                # logging.debug("found sid: {}".format(sid))
                return s
        return None

    def find_sentence_containing(self, start, end, chemdner=True):
        """
            Find the sentence between start and end. If chemdner, do not consider the first sentence, which
            is the title.
        """
        if chemdner:
            firstsent = 1
        else:
            firstsent = 0
        for i, s in enumerate(self.sentences[firstsent:]):
            if len(s.tokens) == 0:
                #logging.debug("sentence without tokens: {} {}".format(s.sid, s.text.encoding("utf-8")))
                continue
            if s.tokens[0].dstart <= start and s.tokens[-1].dend >= end:
                # print "found it!"
                return s
        for s in self.sentences:
            logging.debug("sentence not found: {}-{}".format(start, end))
            if len(s.tokens) > 0:
                logging.debug("{} {} {} {} {}".format(
                    s.tokens[0].dstart <= start, s.tokens[-1].dend >= end,
                    s.tokens[0].dstart, s.tokens[-1].dend,
                    s.text.encode("utf-8")))
        return None

    def get_entity_offsets(self, esource, ths, rules):
        offsets = []
        for s in self.sentences:
            if s.entities:
                offsets += s.entities.get_entity_offsets(
                    esource, ths, rules, s.tokens)
        return offsets

    def get_entity(self, eid, source="goldstandard"):
        for sentence in self.sentences:
            for e in sentence.entities.elist[source]:
                if e.eid == eid:
                    return e
        print "no entity found for eid {}".format(eid)
        return None

    def get_entities(self, source):
        entities = []
        for s in self.sentences:
            if source in s.entities.elist:
                for e in s.entities.elist[source]:
                    entities.append(e)
        return entities

    def get_abbreviations(self):
        self.abbreviations = {}
        first_elem = []
        second_elem = []
        open_paren = False
        for sentence in self.sentences:
            # print sentence.text
            for i, t in enumerate(sentence.tokens):
                if t.text == "-LRB-":
                    open_paren = True
                    last_token = sentence.tokens[i - 1]
                    while last_token.pos.startswith(
                            "NN") or last_token.pos.startswith(
                                "JJ"):  # use nouns before the parenthesis
                        first_elem.insert(0, last_token)
                        if last_token.order == 0:
                            break
                        else:
                            last_token = sentence.tokens[
                                last_token.order -
                                1]  # check the token before this one
                    if len(first_elem) > 0:
                        logging.info("starting abbreviation for this text: " +
                                     str([tt.text for tt in first_elem]))
                    else:
                        open_paren = False
                elif t.text == "-RRB-" and open_paren == True:
                    first_text = sentence.text[first_elem[0].
                                               start:first_elem[-1].end]
                    second_text = sentence.text[second_elem[0].
                                                start:second_elem[-1].end]
                    if len(first_text) > len(
                            second_text):  #abbreviation is the smallest word
                        second_text, first_text = first_text, second_text
                    # rules
                    if not first_text.islower() and len(first_text) > 1:
                        self.abbreviations[first_text] = second_text
                    open_paren = False
                    first_elem = []
                    second_elem = []
                elif open_paren:
                    second_elem.append(t)
        for abv in self.abbreviations:
            if not any([c.isalpha() for c in abv]):
                print abv, ":", self.abbreviations[abv]