Esempio n. 1
0
 def load_models(self):
     for i, t in enumerate(self.types):
         # model = StanfordNERModel(self.basepath + "_" + t, t, subtypes=self.basemodel.subtypes)
         model = CrfSuiteModel(self.basepath + "_" + t,
                               t,
                               subtypes=self.basemodel.subtypes)
         model.load_tagger(self.baseport + i)
         self.models[t] = model
Esempio n. 2
0
 def train_types(self):
     """
     Train models for each subtype of entity, and a general model.
     :param types: subtypes of entities to train individual models, as well as a general model
     """
     self.basemodel.load_data(self.corpus, feature_extractors.keys())
     for t in self.types:
         typepath = self.basepath + "_" + t
         # model = StanfordNERModel(typepath, etype=t)
         model = CrfSuiteModel(typepath, etype=t)
         model.copy_data(self.basemodel, t)
         logging.info("training subtype %s" % t)
         model.train()
         self.models[t] = model
Esempio n. 3
0
 def load_models(self):
     # Run load_tagger method of all models
     for i, a in enumerate(self.entity_annotators.keys()):
         self.create_annotationset(a[0])
         if a[1] == "stanfordner":
             model = StanfordNERModel("annotators/{}/{}".format(a[2], a[0]), a[2])
             model.load_tagger(self.baseport + i)
             self.entity_annotators[a] = model
         elif a[1] == "crfsuite":
             model = CrfSuiteModel("annotators/{}/{}".format(a[2], a[0]), a[2])
             model.load_tagger(self.baseport + i)
             self.entity_annotators[a] = model
         elif a[1] == "banner":
             model = BANNERModel("annotators/{}/{}".format(a[2], a[0]), a[2])
             # model.load_tagger(self.baseport + i)
             self.entity_annotators[a] = model
     for i, a in enumerate(self.relation_annotators.keys()):
         self.create_annotationset(a[0])
         if a[1] == "jsre":
             model = JSREKernel(None, a[2], train=False, modelname="annotators/{}/{}.model".format(a[2], a[0]), ner="all")
             model.load_classifier()
             self.relation_annotators[a] = model
         elif a[1] == "smil":
             model = MILClassifier(None, a[2], relations=[], modelname="{}.model".format(a[0]),
                                   ner="all", generate=False, test=True)
             model.basedir = "annotators/{}".format(a[2])
             model.load_kb("corpora/transmir/transmir_relations.txt")
             model.load_classifier()
             self.relation_annotators[a] = model
Esempio n. 4
0
 def __init__(self, basepath, baseport=9191, **kwargs):
     self.models = {}
     self.basepath = basepath
     self.corpus = kwargs.get("corpus")
     submodels = []
     self.baseport = baseport
     self.types = []
     if basepath.split("/")[-1].startswith("chemdner+ddi"):
         self.types = self.DDI_TYPES + self.CHEMDNER_TYPES + [
             "chemdner", "ddi"
         ]
     elif basepath.split("/")[-1].startswith("ddi"):
         self.types = self.DDI_TYPES + ["all"]
     elif basepath.split("/")[-1].startswith("chemdner") or basepath.split(
             "/")[-1].startswith("cemp"):
         self.types = ["all"] + self.CHEMDNER_TYPES
     elif basepath.split("/")[-1].startswith("gpro"):
         self.types = self.GPRO_TYPES + ["all"]
     else:
         self.types = kwargs.get("subtypes")
     print "training:", self.types
     # self.basemodel = StanfordNERModel(self.basepath, "all")
     self.basemodel = CrfSuiteModel(self.basepath, "all")
Esempio n. 5
0
 def train_types(self):
     """
     Train models for each subtype of entity, and a general model.
     :param types: subtypes of entities to train individual models, as well as a general model
     """
     self.basemodel.load_data(self.corpus, feature_extractors.keys())
     for t in self.types:
         typepath = self.basepath + "_" + t
         # model = StanfordNERModel(typepath, etype=t)
         model = CrfSuiteModel(typepath, etype=t)
         model.copy_data(self.basemodel, t)
         logging.info("training subtype %s" % t)
         model.train()
         self.models[t] = model
Esempio n. 6
0
 def __init__(self, basepath, baseport = 9191, **kwargs):
     self.models = {}
     self.basepath = basepath
     self.corpus = kwargs.get("corpus")
     submodels = []
     self.baseport = baseport
     self.types = []
     if basepath.split("/")[-1].startswith("chemdner+ddi"):
         self.types = self.DDI_TYPES + self.CHEMDNER_TYPES + ["chemdner", "ddi"]
     elif basepath.split("/")[-1].startswith("ddi"):
         self.types = self.DDI_TYPES + ["all"]
     elif basepath.split("/")[-1].startswith("chemdner") or basepath.split("/")[-1].startswith("cemp"):
         self.types = ["all"] + self.CHEMDNER_TYPES
     elif basepath.split("/")[-1].startswith("gpro"):
         self.types = self.GPRO_TYPES + ["all"]
     else:
         self.types = kwargs.get("subtypes")
     print "training:", self.types
     # self.basemodel = StanfordNERModel(self.basepath, "all")
     self.basemodel = CrfSuiteModel(self.basepath, "all")
Esempio n. 7
0
class TaggerCollection(object):
    """
    Collection of tagger classifiers used to train and test specific subtype models
    """
    CHEMDNER_TYPES = [
        "IDENTIFIER", "MULTIPLE", "FAMILY", "FORMULA", "SYSTEMATIC",
        "ABBREVIATION", "TRIVIAL"
    ]
    GPRO_TYPES = ["NESTED", "IDENTIFIER", "FULL_NAME", "ABBREVIATION"]
    DDI_TYPES = ["drug", "group", "brand", "drug_n"]

    def __init__(self, basepath, baseport=9191, **kwargs):
        self.models = {}
        self.basepath = basepath
        self.corpus = kwargs.get("corpus")
        submodels = []
        self.baseport = baseport
        self.types = []
        if basepath.split("/")[-1].startswith("chemdner+ddi"):
            self.types = self.DDI_TYPES + self.CHEMDNER_TYPES + [
                "chemdner", "ddi"
            ]
        elif basepath.split("/")[-1].startswith("ddi"):
            self.types = self.DDI_TYPES + ["all"]
        elif basepath.split("/")[-1].startswith("chemdner") or basepath.split(
                "/")[-1].startswith("cemp"):
            self.types = ["all"] + self.CHEMDNER_TYPES
        elif basepath.split("/")[-1].startswith("gpro"):
            self.types = self.GPRO_TYPES + ["all"]
        else:
            self.types = kwargs.get("subtypes")
        print "training:", self.types
        # self.basemodel = StanfordNERModel(self.basepath, "all")
        self.basemodel = CrfSuiteModel(self.basepath, "all")

    def train_types(self):
        """
        Train models for each subtype of entity, and a general model.
        :param types: subtypes of entities to train individual models, as well as a general model
        """
        self.basemodel.load_data(self.corpus, feature_extractors.keys())
        for t in self.types:
            typepath = self.basepath + "_" + t
            # model = StanfordNERModel(typepath, etype=t)
            model = CrfSuiteModel(typepath, etype=t)
            model.copy_data(self.basemodel, t)
            logging.info("training subtype %s" % t)
            model.train()
            self.models[t] = model

    def load_models(self):
        for i, t in enumerate(self.types):
            # model = StanfordNERModel(self.basepath + "_" + t, t, subtypes=self.basemodel.subtypes)
            model = CrfSuiteModel(self.basepath + "_" + t,
                                  t,
                                  subtypes=self.basemodel.subtypes)
            model.load_tagger(self.baseport + i)
            self.models[t] = model

    def process_type(self, modelst, t, corpus, basemodel, basepath, port):
        # load data only for one model since this takes at least 5 minutes each time
        logging.debug("{}: copying data...".format(t))
        modelst.copy_data(basemodel)
        #logging.debug("pre test %s" % model)
        logging.debug("{}: testing...".format(t))
        res = modelst.test(corpus, port)
        logging.info("{}:done...".format(t))
        return res

    def test_types(self, corpus):
        """
        Classify the corpus with multiple classifiers from different subtypes
        :return ResultSetNER object with the results obtained for the models
        """
        # TODO: parallelize
        results = ResultSetNER(corpus, self.basepath)
        self.basemodel.load_data(corpus, feature_extractors.keys())
        all_results = []
        tasks = [(self.models[t], t, corpus, self.basemodel, self.basepath,
                  self.baseport + i) for i, t in enumerate(self.types)]

        all_results = []
        for t in tasks:
            r = self.process_type(*t)
            all_results.append(r)
        logging.info("adding results...")
        for res, i in enumerate(all_results):
            #logging.debug("adding these results: {}".format(self.types[i]))
            results.add_results(res)
        return results
Esempio n. 8
0
def main():
    start_time = time.time()
    parser = argparse.ArgumentParser(description='')
    parser.add_argument("actions",
                        default="classify",
                        help="Actions to be performed.",
                        choices=[
                            "load_corpus", "annotate", "classify",
                            "write_results", "write_goldstandard", "train",
                            "test", "train_multiple", "test_multiple",
                            "train_matcher", "test_matcher", "crossvalidation",
                            "train_relations", "test_relations"
                        ])
    parser.add_argument(
        "--goldstd",
        default="",
        dest="goldstd",
        nargs="+",
        help="Gold standard to be used. Will override corpus, annotations",
        choices=config.paths.keys())
    parser.add_argument("--submodels",
                        default="",
                        nargs='+',
                        help="sub types of classifiers"),
    parser.add_argument(
        "-i",
        "--input",
        dest="input",
        action="store",
        default='''Administration of a higher dose of indinavir should be \\
considered when coadministering with megestrol acetate.''',
        help="Text to classify.")
    parser.add_argument(
        "--corpus",
        dest="corpus",
        nargs=2,
        default=[
            "chemdner",
            "CHEMDNER/CHEMDNER_SAMPLE_JUNE25/chemdner_sample_abstracts.txt"
        ],
        help="format path")
    parser.add_argument("--annotations", dest="annotations")
    parser.add_argument("--tag",
                        dest="tag",
                        default="0",
                        help="Tag to identify the text.")
    parser.add_argument("--models",
                        dest="models",
                        help="model destination path, without extension")
    parser.add_argument("--entitytype",
                        dest="etype",
                        help="type of entities to be considered",
                        default="all")
    parser.add_argument("--pairtype",
                        dest="ptype",
                        help="type of pairs to be considered",
                        default="all")
    parser.add_argument("--doctype",
                        dest="doctype",
                        help="type of document to be considered",
                        default="all")
    parser.add_argument("--annotated",
                        action="store_true",
                        default=False,
                        dest="annotated",
                        help="True if the input has <entity> tags.")
    parser.add_argument(
        "-o",
        "--output",
        "--format",
        dest="output",
        nargs=2,
        help="format path; output formats: xml, html, tsv, text, chemdner.")
    parser.add_argument("--crf",
                        dest="crf",
                        help="CRF implementation",
                        default="stanford",
                        choices=["stanford", "crfsuite"])
    parser.add_argument("--log",
                        action="store",
                        dest="loglevel",
                        default="WARNING",
                        help="Log level")
    parser.add_argument("--kernel",
                        action="store",
                        dest="kernel",
                        default="svmtk",
                        help="Kernel for relation extraction")
    options = parser.parse_args()

    # set logger
    numeric_level = getattr(logging, options.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % options.loglevel)
    while len(logging.root.handlers) > 0:
        logging.root.removeHandler(logging.root.handlers[-1])
    logging_format = '%(asctime)s %(levelname)s %(filename)s:%(lineno)s:%(funcName)s %(message)s'
    logging.basicConfig(level=numeric_level,
                        format=logging_format,
                        filename="debug.log")
    logging.getLogger().setLevel(numeric_level)
    logging.getLogger("requests.packages").setLevel(30)
    logging.info("Processing action {0} on {1}".format(options.actions,
                                                       options.goldstd))

    # set configuration variables based on the goldstd option if the corpus has a gold standard,
    # or on corpus and annotation options
    # pre-processing options
    if options.actions == "load_corpus":
        if len(options.goldstd) > 1:
            print("load only one corpus each time")
            sys.exit()
        options.goldstd = options.goldstd[0]
        corpus_format = config.paths[options.goldstd]["format"]
        corpus_path = config.paths[options.goldstd]["text"]
        corpus_ann = config.paths[options.goldstd]["annotations"]

        corenlp_client = StanfordCoreNLP('http://localhost:9000')
        corpus = load_corpus(options.goldstd, corpus_path, corpus_format,
                             corenlp_client)
        corpus.save(config.paths[options.goldstd]["corpus"])
        if corpus_ann:  #add annotation if it is not a test set
            corpus.load_annotations(corpus_ann, options.etype, options.ptype)
            corpus.save(config.paths[options.goldstd]["corpus"])

    elif options.actions == "annotate":  # rext-add annotation to corpus
        if len(options.goldstd) > 1:
            print("load only one corpus each time")
            sys.exit()
        options.goldstd = options.goldstd[0]
        corpus_path = config.paths[options.goldstd]["corpus"]
        corpus_ann = config.paths[options.goldstd]["annotations"]
        logging.info("loading corpus %s" % corpus_path)
        corpus = pickle.load(open(corpus_path, 'rb'))
        logging.debug("loading annotations...")
        corpus.clear_annotations(options.etype)
        corpus.load_annotations(corpus_ann, options.etype, options.ptype)
        # corpus.get_invalid_sentences()
        corpus.save(config.paths[options.goldstd]["corpus"])
    else:
        corpus = Corpus("corpus/" + "&".join(options.goldstd))
        for g in options.goldstd:
            corpus_path = config.paths[g]["corpus"]
            logging.info("loading corpus %s" % corpus_path)
            this_corpus = pickle.load(open(corpus_path, 'rb'))
            corpus.documents.update(this_corpus.documents)
        if options.actions == "write_goldstandard":
            model = BiasModel(options.output[1])
            model.load_data(corpus, [])
            results = model.test()
            #results = ResultsNER(options.output[1])
            #results.get_ner_results(corpus, model)
            results.save(options.output[1] + ".pickle")
            #logging.info("saved gold standard results to " + options.output[1] + ".txt")

        # training
        elif options.actions == "train":
            if options.crf == "stanford":
                model = StanfordNERModel(options.models, options.etype)
            elif options.crf == "crfsuite":
                model = CrfSuiteModel(options.models, options.etype)
            model.load_data(corpus, feature_extractors.keys(), options.etype)
            model.train()
        elif options.actions == "train_matcher":  # Train a simple classifier based on string matching
            model = MatcherModel(options.models)
            model.train(corpus)
            # TODO: term list option
            #model.train("TermList.txt")
        elif options.actions == "train_multiple":  # Train one classifier for each type of entity in this corpus
            # logging.info(corpus.subtypes)
            models = TaggerCollection(basepath=options.models,
                                      corpus=corpus,
                                      subtypes=corpus.subtypes)
            models.train_types()
        elif options.actions == "train_relations":
            if options.kernel == "jsre":
                model = JSREKernel(corpus, options.ptype)
            elif options.kernel == "svmtk":
                model = SVMTKernel(corpus, options.ptype)
            elif options.kernel == "stanfordre":
                model = StanfordRE(corpus, options.ptype)
            elif options.kernel == "multir":
                model = MultiR(corpus, options.ptype)
            elif options.kernel == "scikit":
                model = ScikitRE(corpus, options.ptype)
            elif options.kernel == "crf":
                model = CrfSuiteRE(corpus, options.ptype)
            model.train()
        # testing
        elif options.actions == "test":
            base_port = 9191
            if len(options.submodels) > 1:
                allresults = ResultSetNER(corpus, options.output[1])
                for i, submodel in enumerate(options.submodels):
                    model = StanfordNERModel(options.models + "_" + submodel)
                    model.load_tagger(base_port + i)
                    # load data into the model format
                    model.load_data(corpus,
                                    feature_extractors.keys(),
                                    mode="test")
                    # run the classifier on the data
                    results = model.test(corpus, port=base_port + i)
                    allresults.add_results(results)
                    model.kill_process()
                # save the results to an object that can be read again, and log files to debug
                final_results = allresults.combine_results()
            else:
                if options.crf == "stanford":
                    model = StanfordNERModel(options.models, options.etype)
                elif options.crf == "crfsuite":
                    model = CrfSuiteModel(options.models, options.etype)
                model.load_tagger()
                model.load_data(corpus, feature_extractors.keys(), mode="test")
                final_results = model.test(corpus)
            #with codecs.open(options.output[1] + ".txt", 'w', 'utf-8') as outfile:
            #    lines = final_results.corpus.write_chemdner_results(options.models, outfile)
            #final_results.lines = lines
            final_results.save(options.output[1] + ".pickle")
        elif options.actions == "test_matcher":
            if "mirna" in options.models:
                model = MirnaMatcher(options.models)
            else:
                model = MatcherModel(options.models)
            results = ResultsNER(options.models)
            results.corpus, results.entities = model.test(corpus)
            allentities = set()
            for e in results.entities:
                allentities.add(results.entities[e].text)
            with codecs.open(options.output[1] + ".txt", 'w',
                             'utf-8') as outfile:
                outfile.write('\n'.join(allentities))

            results.save(options.output[1] + ".pickle")
        elif options.actions == "test_multiple":
            logging.info("testing with multiple classifiers... {}".format(
                ' '.join(options.submodels)))
            allresults = ResultSetNER(corpus, options.output[1])
            if len(options.submodels) < 2:
                models = TaggerCollection(basepath=options.models)
                models.load_models()
                results = models.test_types(corpus)
                final_results = results.combine_results()
            else:
                base_port = 9191
                for submodel in options.submodels:
                    models = TaggerCollection(basepath=options.models + "_" +
                                              submodel,
                                              baseport=base_port)
                    models.load_models()
                    results = models.test_types(corpus)
                    logging.info("combining results...")
                    submodel_results = results.combine_results()
                    allresults.add_results(submodel_results)
                    base_port += len(models.models)
                final_results = allresults.combine_results()
            logging.info("saving results...")
            final_results.save(options.output[1] + ".pickle")
        elif options.actions == "test_relations":
            if options.kernel == "jsre":
                model = JSREKernel(corpus, options.ptype, train=False)
            elif options.kernel == "svmtk":
                model = SVMTKernel(corpus, options.ptype)
            elif options.kernel == "rules":
                model = RuleClassifier(corpus, options.ptype)
            elif options.kernel == "stanfordre":
                model = StanfordRE(corpus, options.ptype)
            elif options.kernel == "scikit":
                model = ScikitRE(corpus, options.ptype)
            elif options.kernel == "crf":
                model = CrfSuiteRE(corpus, options.ptype, test=True)
            model.load_classifier()
            model.test()
            results = model.get_predictions(corpus)
            results.save(options.output[1] + ".pickle")

    total_time = time.time() - start_time
    logging.info("Total time: %ss" % total_time)
Esempio n. 9
0
def run_crossvalidation(goldstd_list,
                        corpus,
                        model,
                        cv,
                        crf="stanford",
                        entity_type="all",
                        cvlog="cv.log"):
    logfile = open(cvlog, 'w')
    doclist = corpus.documents.keys()
    random.shuffle(doclist)
    size = int(len(doclist) / cv)
    sublists = chunks(doclist, size)
    logging.debug("Chunks:")
    logging.debug(sublists)
    p, r = [], []
    all_results = ResultsNER(model)
    all_results.path = model + "_results"
    for nlist in range(cv):
        testids, trainids = None, None
        testids = sublists[nlist]
        trainids = list(itertools.chain.from_iterable(sublists[:nlist]))
        trainids += list(itertools.chain.from_iterable(sublists[nlist + 1:]))
        train_corpus, test_corpus = None, None
        print 'CV{} - test set: {}; train set: {}'.format(
            nlist, len(testids), len(trainids))
        train_corpus = Corpus(
            corpus.path + "_train",
            documents={did: corpus.documents[did]
                       for did in trainids})
        test_corpus = Corpus(
            corpus.path + "_test",
            documents={did: corpus.documents[did]
                       for did in testids})
        # logging.debug("train corpus docs: {}".format("\n".join(train_corpus.documents.keys())))
        #test_entities = len(test_corpus.get_all_entities("goldstandard"))
        #train_entities = len(train_corpus.get_all_entities("goldstandard"))
        #logging.info("test set entities: {}; train set entities: {}".format(test_entities, train_entities))
        basemodel = model + "_cv{}".format(nlist)
        logging.debug('CV{} - test set: {}; train set: {}'.format(
            nlist, len(test_corpus.documents), len(train_corpus.documents)))
        '''for d in train_corpus.documents:
            for s in train_corpus.documents[d].sentences:
                print len([t.tags.get("goldstandard") for t in s.tokens if t.tags.get("goldstandard") != "other"])
        sys.exit()'''
        # train
        logging.info('CV{} - TRAIN'.format(nlist))
        # train_model = StanfordNERModel(basemodel)
        train_model = None
        if crf == "stanford":
            train_model = StanfordNERModel(basemodel, entity_type)
        elif crf == "crfsuite":
            train_model = CrfSuiteModel(basemodel, entity_type)
        train_model.load_data(train_corpus, feature_extractors.keys())
        train_model.train()

        # test
        logging.info('CV{} - TEST'.format(nlist))
        test_model = None
        if crf == "stanford":
            test_model = StanfordNERModel(basemodel, entity_type)
        elif crf == "crfsuite":
            test_model = CrfSuiteModel(basemodel, entity_type)
        test_model.load_tagger(port=9191 + nlist)
        test_model.load_data(test_corpus,
                             feature_extractors.keys(),
                             mode="test")
        final_results = None
        final_results = test_model.test(test_corpus, port=9191 + nlist)
        if crf == "stanford":
            test_model.kill_process()
        final_results.basepath = basemodel + "_results"
        final_results.path = basemodel

        all_results.entities.update(final_results.entities)
        all_results.corpus.documents.update(final_results.corpus.documents)
        # validate
        """if config.use_chebi:
            logging.info('CV{} - VALIDATE'.format(nlist))
            final_results = add_chebi_mappings(final_results, basemodel)
            final_results = add_ssm_score(final_results, basemodel)
            final_results.combine_results(basemodel, basemodel)"""

        # evaluate
        logging.info('CV{} - EVALUATE'.format(nlist))
        test_goldset = set()
        for gs in goldstd_list:
            goldset = get_gold_ann_set(config.paths[gs]["format"],
                                       config.paths[gs]["annotations"],
                                       entity_type, "pairtype",
                                       config.paths[gs]["text"])
            for g in goldset[0]:
                if g[0] in testids:
                    test_goldset.add(g)
        precision, recall = get_results(final_results, basemodel, test_goldset,
                                        {}, [])
        # evaluation = run_chemdner_evaluation(config.paths[goldstd]["cem"], basemodel + "_results.txt", "-t")
        # values = evaluation.split("\n")[1].split('\t')
        p.append(precision)
        r.append(recall)
        # logging.info("precision: {} recall:{}".format(str(values[13]), str(values[14])))
    pavg = sum(p) / cv
    ravg = sum(r) / cv
    print "precision: average={} all={}".format(
        str(pavg), '|'.join([str(pp) for pp in p]))
    print "recall: average={}  all={}".format(str(ravg),
                                              '|'.join([str(rr) for rr in r]))
    all_goldset = set()
    for gs in goldstd_list:
        goldset = get_gold_ann_set(config.paths[gs]["format"],
                                   config.paths[gs]["annotations"],
                                   entity_type, config.paths[gs]["text"])
        for g in goldset:
            all_goldset.add(g)
    get_results(all_results, model, all_goldset, {}, [])
Esempio n. 10
0
 def load_models(self):
     for i, t in enumerate(self.types):
         # model = StanfordNERModel(self.basepath + "_" + t, t, subtypes=self.basemodel.subtypes)
         model = CrfSuiteModel(self.basepath + "_" + t, t, subtypes=self.basemodel.subtypes)
         model.load_tagger(self.baseport + i)
         self.models[t] = model
Esempio n. 11
0
class TaggerCollection(object):
    """
    Collection of tagger classifiers used to train and test specific subtype models
    """
    CHEMDNER_TYPES =  ["IDENTIFIER", "MULTIPLE", "FAMILY", "FORMULA", "SYSTEMATIC", "ABBREVIATION", "TRIVIAL"]
    GPRO_TYPES = ["NESTED", "IDENTIFIER", "FULL_NAME", "ABBREVIATION"]
    DDI_TYPES = ["drug", "group", "brand", "drug_n"]

    def __init__(self, basepath, baseport = 9191, **kwargs):
        self.models = {}
        self.basepath = basepath
        self.corpus = kwargs.get("corpus")
        submodels = []
        self.baseport = baseport
        self.types = []
        if basepath.split("/")[-1].startswith("chemdner+ddi"):
            self.types = self.DDI_TYPES + self.CHEMDNER_TYPES + ["chemdner", "ddi"]
        elif basepath.split("/")[-1].startswith("ddi"):
            self.types = self.DDI_TYPES + ["all"]
        elif basepath.split("/")[-1].startswith("chemdner") or basepath.split("/")[-1].startswith("cemp"):
            self.types = ["all"] + self.CHEMDNER_TYPES
        elif basepath.split("/")[-1].startswith("gpro"):
            self.types = self.GPRO_TYPES + ["all"]
        else:
            self.types = kwargs.get("subtypes")
        print "training:", self.types
        # self.basemodel = StanfordNERModel(self.basepath, "all")
        self.basemodel = CrfSuiteModel(self.basepath, "all")

    def train_types(self):
        """
        Train models for each subtype of entity, and a general model.
        :param types: subtypes of entities to train individual models, as well as a general model
        """
        self.basemodel.load_data(self.corpus, feature_extractors.keys())
        for t in self.types:
            typepath = self.basepath + "_" + t
            # model = StanfordNERModel(typepath, etype=t)
            model = CrfSuiteModel(typepath, etype=t)
            model.copy_data(self.basemodel, t)
            logging.info("training subtype %s" % t)
            model.train()
            self.models[t] = model

    def load_models(self):
        for i, t in enumerate(self.types):
            # model = StanfordNERModel(self.basepath + "_" + t, t, subtypes=self.basemodel.subtypes)
            model = CrfSuiteModel(self.basepath + "_" + t, t, subtypes=self.basemodel.subtypes)
            model.load_tagger(self.baseport + i)
            self.models[t] = model

    def process_type(self, modelst, t, corpus, basemodel, basepath, port):
        # load data only for one model since this takes at least 5 minutes each time
        logging.debug("{}: copying data...".format(t))
        modelst.copy_data(basemodel)
        #logging.debug("pre test %s" % model)
        logging.debug("{}: testing...".format(t))
        res = modelst.test(corpus, port)
        logging.info("{}:done...".format(t))
        return res

    def test_types(self, corpus):
        """
        Classify the corpus with multiple classifiers from different subtypes
        :return ResultSetNER object with the results obtained for the models
        """
        # TODO: parallelize
        results = ResultSetNER(corpus, self.basepath)
        self.basemodel.load_data(corpus, feature_extractors.keys())
        all_results = []
        tasks = [(self.models[t], t, corpus, self.basemodel, self.basepath, self.baseport + i) for i, t in enumerate(self.types)]

        all_results = []
        for t in tasks:
            r = self.process_type(*t)
            all_results.append(r)
        logging.info("adding results...")
        for res, i in enumerate(all_results):
            #logging.debug("adding these results: {}".format(self.types[i]))
            results.add_results(res)
        return results