コード例 #1
0
    def tokenizer(self):
        """
        Get the tokenizer. Set it if it is null.

        :return:    The tokenizer
        """
        config = cache_ops.get_redis()
        if self._tokenizer is None:
            cfg_str = config.get('topic_tiler_config')
            if cfg_str:
                cfg = dict(json.loads(cfg_str))
                width = int(cfg.get('w', 20))
                k_size = int(cfg.get('k', 10))
                stop_words = cfg.get('stopwords', 'english')
                stop_words = stopwords.words(stop_words)
                cutoff_policy = cfg.get('cutoff_policy', 'HC')
            else:
                width = 20
                k_size = 10
                stop_words = 'english'
                stop_words = stopwords.words(stop_words)
                cutoff_policy = 'HC'
            self._tokenizer = TopicTokenizer(cutoff_policy, stop_words, width,
                                             k_size)
        return self._tokenizer
コード例 #2
0
 def tokenizer(self):
     """
     Tokenizer getter
     :return:    The sentence tokenizer
     """
     config = cache_ops.get_redis()
     if self._tokenizer is None:
         cfg_str = config.get('sent_tokenizer_config')
         if cfg_str:
             cfg = dict(json.loads(cfg_str))
             tok_path = cfg.get('sent_tokenizer_path')
             self._tokenizer = SentenceTokenizer(tok_path)
         else:
             tok_path = "nltk:tokenizers/punkt/english.pickle"
             self._tokenizer = SentenceTokenizer(tok_path)
     return self._tokenizer
コード例 #3
0
ファイル: ner_task.py プロジェクト: asevans48/NLPServer
    def ner(self):
        """
        Get the tokenizer. Set it if it is null.

        :return:    The tokenizer
        """
        client = cache_ops.get_redis()
        if self._ner is None:
            config_str = client.get('ner_config')
            if config_str:
                config = dict(json.loads(config_str))
                load_gpu = config.get('use_gpu', False)
                model_type = config.get('ner_model', 'en_core_web_sm')
                self._ner = NERModel(model_type, load_gpu)
            else:
                self._ner = NERModel('en_core_web_sm', False)
        return self._ner
コード例 #4
0
  -h --help  Show this help screen

"""

from docopt import docopt
import logging

from nlp_server.cache import cache_ops
from nlp_server.nlp_celery.celery_app import setup_app, set_config
from nlp_server.nlp_celery.tasks.ner_task import NERTask
from nlp_server.nlp_celery.tasks.sent_tokenizer_task import SentTokenizerTask
from nlp_server.nlp_celery.tasks.topic_tiler_task import TopicTilerTask

if __name__ == "__main__":
    logging.info("Setting up Application")
    APP = setup_app()

    DOC = docopt(__doc__, version='NLP Server 0.1')
    CLIENT = cache_ops.get_redis()
    logging.info('Setting Config')
    set_config(DOC, CLIENT)
    logging.info("Registering Tasks")
    print(APP.tasks)
    logging.info("Sending to Celery")
    APP.send_task(
        'NERTask',
        args=[
            'My name is slim Shady and all you other slim shadys can.',
            ['PERSON']
        ])