示例#1
0
def OnTimer(timerid):
    Logger.LogDebug("Py OnTimer:%d" % timerid)

    from common import Timer
    Timer.onTimer(timerid)

    return 0, "success"
示例#2
0
def _eval(mymodel,
          args,
          data_loader,
          data_set=None,
          RELOAD_MODEL=None,
          use_cuda=False):
    old_model_path = os.path.join(args.result_dir, RELOAD_MODEL)
    if RELOAD_MODEL is not None and os.path.exists(old_model_path):
        mymodel.load_model(old_model_path)
        LOGGER.info("Reload model successfully~")
    else:
        LOGGER.info(
            f'There is no such file in {old_model_path}, End evaluation')
        return

    LOGGER.info(f'reload model {RELOAD_MODEL}')
    hyper_param = {
        'batch_size': 100,
        'issave': True,
        'result_dir': args.result_dir
    }

    timer = Timer()
    timer.set(args.time_budget)
    with timer.time_limit('eval'):
        mymodel.eval_model(data_loader, data_set, hyper_param, rebuild=True)
示例#3
0
 def __init__(self):
     rospy.init_node('Gateway', log_level=rospy.DEBUG)
     server_config_file = rospy.get_param("~server_config_file")
     self.config = Config(server_config_file)
     self.pf = ProtocolFactory(self.config)
     self.run_id = rospy.get_param("run_id")
     print("runid = ", self.run_id)
     self.node_list = rosnode.get_node_names()
     self.timer = Timer()
     self.monitor = Monitor(self.node_list, self.timer)
     self._server_request = {}  # stores server_request
     self._event_bus = Queue()
     self._heartbeat_timeout_job = None
     self._tele_report_job = None
     self._report_car_job = None
     self._report_task_job = None
     self._service = DrivingTaskService(self._event_bus)
     self.__client = Connector(self._event_bus, self.config.host,
                               self.config.port)
     self._handler_map = {}
     self._event_handler_map = {}
     self._add_command_handler()
     self._add_event_handler()
     self._web_server = WebServer(self.monitor, self._service, self.config)
     self._tele_control_service = TeleControlService()
示例#4
0
def main():
    import sys
    from common import Timer

    try:
        img_fn = sys.argv[1]
    except:
        img_fn = 'data/baboon.jpg'

    img = cv.imread(cv.samples.findFile(img_fn))
    if img is None:
        print('Failed to load image file:', img_fn)
        sys.exit(1)

    filters = build_filters()

    with Timer('running single-threaded'):
        res1 = process(img, filters)
    with Timer('running multi-threaded'):
        res2 = process_threaded(img, filters)

    print('res1 == res2: ', (res1 == res2).all())
    cv.imshow('img', img)
    cv.imshow('result', res2)
    cv.waitKey()
    print('Done')
示例#5
0
def main():
    import sys
    from common import Timer

    try:
        img_fn = sys.argv[1]
    except:
        img_fn = 'PythonData/baboon.jpg'

    img = cv.imread(cv.samples.findFile(img_fn))
    if img is None:
        print('Failed to load image file:', img_fn)
        sys.exit(1)

    filters = build_filters()

    with Timer('running single-threaded'):
        res1 = process(img, filters)
    with Timer('running multi-threaded'):
        res2 = process_threaded(img, filters)

    print('res1 == res2: ', (res1 == res2).all())

    both = np.empty((img.shape[0], img.shape[1]*2, 3), img.dtype)
    both = cv.hconcat([img, res2])
    cv.imshow("Original and filtered image", both)
    cv.waitKey()
示例#6
0
def OnClientProxy(sockfd, type, data, serverobj):
    """收到客户端的socket请求入口"""
    print("OnClientProxy:%d,%d,%s" % (sockfd, type, data))
    if type == FD_TYPE_ACCEPT:
        logger.debug('OnClientProxy,type=FD_TYPE_ACCEPT,sock=%d' % sockfd)
        try:
            serverobj.add_client_rpc_channel(sockfd)
        except Exception as e:
            print("Exception:", e)
    elif type == FD_TYPE_CLIENT:
        logger.debug('OnClientProxy,type=FD_TYPE_READ,sock=%s' % sockfd)
        try:
            serverobj.handle_client_rpc_channel(sockfd, data)
        except Exception as e:
            print("Exception:", e)
    elif type == FD_TYPE_CLOSE:
        logger.debug('OnClientProxy,type=FD_TYPE_CLOSE,sock=%s' % sockfd)
        try:
            serverobj.del_client_rpc_channel(sockfd)
        except Exception as e:
            print("Exception:", e)
    elif type == FD_TYPE_TIMER:
        logger.debug('OnClientProxy,type=FD_TYPE_TIMER,sock=%s' % sockfd)
        from common import Timer
        Timer.onTimer(sockfd)  # timerId
示例#7
0
def _train(mymodel, args, data_loader, train_dataset=None, eval_dataset=None, RELOAD_MODEL=None, use_cuda=False):
    if RELOAD_MODEL is None:
        LOGGER.info(f'Rebuild model')
    else:
        old_model_path = os.path.join(args.result_dir, RELOAD_MODEL)
        if os.path.exists(old_model_path):
            mymodel.load_model(old_model_path)
            LOGGER.info("Reload model successfully~")
        else:
            LOGGER.info(f'There is no such file in {old_model_path}, Rebuild model')

    ##TODO:
    BATCH_SIZE = 8 if data_loader.sentence_max_len > 200 else 16
    if use_cuda:
        train_param = {
            'EPOCH': 30,         #45  TODO:15
            'batch_size': BATCH_SIZE,    #512   TODO:64
            'learning_rate_bert': 5e-5,  
            'learning_rate_upper': 1e-3,  #TODO:1e-3
            'bert_finetune': True,
            'visualize_length': 20, #10
            'isshuffle': True,
            'result_dir': args.result_dir,
            'model_name':'model_test.p'
        }

    else:
        train_param = {
            'EPOCH': CPU_EPOCH,         #45
            'batch_size': CPU_BATCHSIZE,    #512
            'learning_rate_bert': 5e-5,
            'learning_rate_upper': 1e-3,
            'bert_finetune': False,
            'visualize_length': CPU_VISUAL, #10
            'isshuffle': True,
            'result_dir': args.result_dir,
            'model_name':'model_test.p'
        }        

    timer = Timer()
    timer.set(args.time_budget)
    loss_record = None
    with timer.time_limit('training'):
        loss_record, score_record = mymodel.train_model(data_loader, hyper_param=train_param, train_dataset=train_dataset, eval_dataset=eval_dataset)

    loss_record = np.array(loss_record)
    loss_save_path = os.path.join(args.result_dir, 'loss_train.txt')
    loss_img_path = os.path.join(args.result_dir, 'loss.png')
    np.savetxt(loss_save_path, loss_record)

    score_record = np.array(score_record)
    score_save_path = os.path.join(args.result_dir, 'score_train.txt')
    score_img_path = os.path.join(args.result_dir, 'score.png')
    np.savetxt(score_save_path, score_record)  ### (epochs, 3)

    loss = np.loadtxt(loss_save_path).reshape(1, -1)
    score = np.loadtxt(score_save_path).T
    plot_img(loss, loss_img_path)
    plot_img(score, score_img_path)
示例#8
0
def main():
    print('Running....')

    with Timer('Single threaded'):
        singlethreaded()

    with Timer('Multiple threads'):
        multithreaded()
def main():
    print('Running....')

    with Timer('Single process'):
        singleprocessed()

    with Timer('Multiple processes'):
        multiprocessed()
class AnimationController(object):
    def __init__(self, model):
        self.model = model
        self.timer = Timer()
    
    def start(self):
        try:
            emitter = AnimationController.animation_timer
        except:
            print 'Animation Controller -- No Instance of Timer Running in Class'
            print 'Please run AnimationController.init()'
            emitter = None
            
        if emitter is not None:
            self.timer.start(emitter, interval = self.model.interval)
            self.timer.on('tick', self.next)
        
        
    def stop(self):
        self.timer.stop()
        
    def resume(self):
        self.timer.resume()
        
    def next(self):
        print "next frame now"
        self.model.next_frame()
        
    @classmethod
    def init(cls, emitter, message_type='tick'):
        if not guarded(lambda:cls.animation_timer):
            cls.animation_timer = Timer()
            cls.animation_timer.start(emitter, message_type=message_type)
        return cls.animation_timer
        
示例#11
0
    def __init__(self):
        self.home = Path("/opt/stanford-corenlp-full-2018-10-05/")
        with Timer() as self.model_load_time:
            import jpype

            jpype.startJVM(
                jpype.getDefaultJVMPath(),
                "-ea",
                "-Djava.class.path=" +
                str(self.home / "stanford-corenlp-3.9.2.jar"),
            )

            ling = jpype.JPackage("edu").stanford.nlp.ling
            nndep = jpype.JPackage("edu").stanford.nlp.parser.nndep

            self.parser = nndep.DependencyParser.loadFromModelFile(
                str(self.home / "stanford-german-corenlp-2018-10-05-models" /
                    "edu/stanford/nlp/models/parser/nndep/UD_German.gz"))

            def myprocessor(myinput):
                results = list()
                for sent in string2doc(myinput, hide_fields=HIDDEN_FIELDS):
                    sent_arr = jpype.java.util.ArrayList()
                    for tok in sent:
                        sent_arr.add(ling.TaggedWord(tok.word, tok.xpos))
                    results.append(self.parser.predict(sent_arr))
                return results

            self.processor = myprocessor
示例#12
0
    def _ClearDrawMode(self):
        with Timer() as t:
            drawModeAttr = self._modelAPI.GetModelDrawModeAttr()
            if drawModeAttr:
                sessionSpec = _GetPropertySpecInSessionLayer(drawModeAttr)
                if sessionSpec:
                    self._primViewItem.drawModeWidget = None
                    self._primViewItem.treeWidget().closePersistentEditor(
                        self._primViewItem, PrimViewColumnIndex.DRAWMODE)

                    sessionSpec.ClearDefaultValue()
                    sessionSpec.layer.ScheduleRemoveIfInert(sessionSpec)
                    self._refreshFunc(self._primViewItem)
                else:
                    PrintWarning(
                        self._modelAPI.GetPath(), "Failed to get "
                        "session layer spec for the model:drawMode attribute")
                    return
            else:
                PrintWarning(self._modelAPI.GetPath(), "Failed to get "
                             "model:drawMode attribute")
                return
        if self._printTiming:
            t.PrintTime(
                "clear model:drawMode on <%s> to %s" %
                (self._modelAPI.GetPath(), self._comboBox.currentText()))
示例#13
0
def main():
    args = get_args()
    client = Client('127.0.0.1:8786')
    ncores = sum(client.ncores().values())
    pd.set_option('display.large_repr', 'truncate')
    pd.set_option('display.max_columns',
                  0)  # noqa pd.set_option('display.max_rows', 1000)  # noqa
    cann_group_df = make_cann_group_df(num_products=100)
    df = read_df(args, cann_group_df['productKey'])
    logger.info('Setting index')
    df = df.set_index('customerKey', drop=True)
    logger.info('Repartitioning')
    df = df.repartition(npartitions=ncores)
    logger.info('Mapping Cann Group')
    df['cannGroupKey'] = df['productKey'].map(cann_group_df['cannGroupKey'])
    logger.info('Persisting')
    df = client.persist(df)
    logger.info('Cann Groups')
    for cann_group_key in cann_group_df['cannGroupKey'].unique().tolist():
        print('Filtering Cann Group %s' % cann_group_key)
        cann_df = df[df['cannGroupKey'] == cann_group_key]
        print('This df: %s' % (len(cann_df), ))
        with Timer('%s' % (cann_group_key, )):
            calculate_switching(cann_df)
        return
示例#14
0
    def stage(self, value):
        """Sets the current Usd.Stage object, and emits a signal if it is
        different from the previous stage.
        """

        validStage = (value is None) or isinstance(value, Usd.Stage)
        if not validStage:
            raise ValueError("Expected USD Stage, got: {}".format(repr(value)))

        if value is not self._stage:

            if self._pcListener:
                self._pcListener.Revoke()
                self._pcListener = None

            if value is None:
                with Timer() as t:
                    self._stage = None
                if self._printTiming:
                    t.PrintTime('close stage')
            else:
                self._stage = value

            if self._stage:
                from pxr import Tf
                self._pcListener = \
                    Tf.Notice.Register(Usd.Notice.ObjectsChanged,
                                       self.__OnPrimsChanged, self._stage)

            self.signalStageReplaced.emit()
示例#15
0
    def __init__(self):
        with Timer() as self.model_load_time:
            from iwnlp.iwnlp_wrapper import IWNLPWrapper
            from stts2upos import conv_table
            data_loc = "/opt/iwnlp/IWNLP.Lemmatizer_20181001.json"
            self.lemmatizer = IWNLPWrapper(lemmatizer_path=data_loc)

            def myprocessor(myinput):
                mydoc = string2doc(myinput)
                for sent in mydoc:
                    for tok in sent:
                        try:
                            matching_lemmas = self.lemmatizer.lemmatize(
                                tok.word, conv_table.get(tok.xpos))
                            if matching_lemmas is None:
                                tok.lemma = "_"
                                # elif len(matching_lemmas) > 1:
                                #     print("lots o lemmas!", matching_lemmas)
                            else:
                                # unclear how to select best alternative
                                # just use first item in list
                                tok.lemma = matching_lemmas[0]
                        except ValueError:
                            tok.lemma = "_"
                        # don't repeat gold pos in output
                        tok.hide_fields(HIDDEN_FIELDS)
                return mydoc

            self.processor = myprocessor
示例#16
0
    def __init__(self):
        with Timer() as self.model_load_time:
            import spacy

            nlp = spacy.load("de", disable=["tagger", "parser", "ner"])
            nlp.add_pipe(nlp.create_pipe("sentencizer"))
            self.processor = nlp
示例#17
0
    def __init__(self):
        self.name = "stanfordnlp"
        with Timer() as self.model_load_time:
            from stanfordnlp import Pipeline, Document
            from stanfordnlp.models.common.conll import CoNLLFile

            self.pipeline = Pipeline(
                lang="de",
                tokenize_pretokenized=True,
                processors="depparse",
                # lower batch size so our GPU can cope
                depparse_batch_size=1000,
            )

            def myprocessor(myinput):
                # run input through converter to hide fields, etc.
                self.input_doc = common.Document(myinput,
                                                 hidden_fields=HIDDEN_FIELDS)
                modified_input = doc2string(self.input_doc)
                self.snlp_doc = Document("")
                self.snlp_doc.conll_file = CoNLLFile(input_str=modified_input)
                self.snlp_doc.load_annotations()
                return self.pipeline(self.snlp_doc)

            self.processor = myprocessor
示例#18
0
    def Run(self):
        '''
        The main entry point to launch a process using UsdView.
        '''

        parser = argparse.ArgumentParser(prog=sys.argv[0],
                                         description=self.GetHelpDescription())

        traceCollector = None

        with Timer() as totalTimer:
            self.RegisterPositionals(parser)
            self.RegisterOptions(parser)
            arg_parse_result = self.ParseOptions(parser)
            self.ValidateOptions(arg_parse_result)

            if arg_parse_result.traceToFile:
                from pxr import Trace
                traceCollector = Trace.Collector()
                traceCollector.pythonTracingEnabled = True
                traceCollector.enabled = True

            self.__LaunchProcess(arg_parse_result)

        if traceCollector:
            traceCollector.enabled = False

        if arg_parse_result.timing and arg_parse_result.quitAfterStartup:
            totalTimer.PrintTime('open and close usdview')

        if traceCollector:
            Trace.Reporter.globalReporter.ReportChromeTracingToFile(
                arg_parse_result.traceToFile)
示例#19
0
def read_df(args, product_keys):
    d_cli = get_data_client(args)
    with Timer('Read'):
        df = d_cli.read_df()
        df = df[df['productKey'].isin(product_keys)]
        df = df.persist()
        logger.info('Read data: %s rows, %s mb' %
                    (len(df), df_mem_in_mb(df).compute()))
    return df
示例#20
0
    def __init__(self):
        with Timer() as self.model_load_time:
            from stanfordnlp import Pipeline

            self.processor = Pipeline(
                lang="de",
                # 'mwt' processor would expand things
                # like 'am' to 'an dem'
                processors="tokenize",
            )
示例#21
0
    def __init__(self):
        self.name = "stanfordnlp"
        with Timer() as self.model_load_time:
            from stanfordnlp import Pipeline

            self.processor = Pipeline(
                lang="de",
                tokenize_pretokenized=True,
                processors="tokenize,mwt,pos,lemma",
            )
示例#22
0
 def updateVariantSelection(self, index, printTiming):
     variantSet = self.prim.GetVariantSet(self.variantSetName)
     currentVariantSelection = variantSet.GetVariantSelection()
     newVariantSelection = str(self.currentText())
     if currentVariantSelection != newVariantSelection:
         with Timer() as t:
             variantSet.SetVariantSelection(newVariantSelection)
         if printTiming:
             t.PrintTime("change variantSet %s to %s" %
                         (variantSet.GetName(), newVariantSelection))
示例#23
0
def calculate_switching(df):
    tmp_df = df.head(0)
    result = trans_seq_num(tmp_df)
    meta = [(col, dtype) for col, dtype in result.dtypes.items()]
    cols = [col for col, _ in meta]
    with Timer('Trans Seq num'):
        trans_seq_num_df = df.map_partitions(trans_seq_num,
                                             cols=cols,
                                             meta=meta).compute()
    print(trans_seq_num_df)
示例#24
0
def main():
    global TIMER
    TIMER = Timer()

    oparser = argparse.ArgumentParser(description="intelligent crawling with q-learning")
    oparser.add_argument("--config-file", dest="configFile", required=True,
                         help="Path to config file (containing MySQL login etc.)")
    oparser.add_argument("--language-pair", dest="langPair", required=True,
                         help="The 2 language we're interested in, separated by ,")
    oparser.add_argument("--save-dir", dest="saveDir", default=".",
                         help="Directory that model WIP are saved to. If existing model exists then load it")
    oparser.add_argument("--save-plots", dest="saveDirPlots", default="plot",
                     help="Directory ")
    oparser.add_argument("--num-train-hosts", dest="numTrainHosts", type=int,
                         default=1, help="Number of domains to train on")
    oparser.add_argument("--num-test-hosts", dest="numTestHosts", type=int,
                         default=3, help="Number of domains to test on")
    oparser.add_argument("--max-crawl", dest="maxCrawl", type=int,
                         default=sys.maxsize, help="Maximum number of pages to crawl")
    oparser.add_argument("--gamma", dest="gamma", type=float,
                         default=0.999, help="Reward discount")
    options = oparser.parse_args()

    np.random.seed()
    np.set_printoptions(formatter={'float': lambda x: "{0:0.1f}".format(x)}, linewidth=666)

    if not os.path.exists(options.saveDir): os.makedirs(options.saveDir, exist_ok=True)
    if not os.path.exists("pickled_domains"): os.makedirs("pickled_domains", exist_ok=True)

    languages = GetLanguages(options.configFile)
    params = LearningParams(languages, options, languages.maxLangId, languages.GetLang("None"))

    print("options.numTrainHosts", options.numTrainHosts)
    #hosts = ["http://vade-retro.fr/"]
    hosts = ["http://telasmos.org/"]
    #hosts = ["http://www.buchmann.ch/", "http://telasmos.org/", "http://tagar.es/"]
    #hosts = ["http://www.visitbritain.com/"]

    #hostsTest = ["http://vade-retro.fr/"]
    #hostsTest = ["http://www.visitbritain.com/"]
    hostsTest = ["http://www.visitbritain.com/", "http://chopescollection.be/", "http://www.bedandbreakfast.eu/"]

    envs = GetEnvs(options.configFile, languages, hosts[:options.numTrainHosts])
    envsTest = GetEnvs(options.configFile, languages, hostsTest[:options.numTestHosts])

    tf.reset_default_graph()
    qn = Qnetwork(params)
    init = tf.global_variables_initializer()

    saver = None #tf.train.Saver()
    with tf.Session() as sess:
        sess.run(init)

        Train(params, sess, saver, qn, envs, envsTest)
示例#25
0
    def __get_rec_item_ids(self, item_id, sim_cat_ids):
	"""
	获取推荐item_id
	Args:
	    item_id: query的item_id
	    sim_cat_ids : item_id的相似cat_id列表
	Returns:
	    [(rec_item_id, sim_value), (rec_item_id, sim_value), ...],长度为strategy.get_total_rec_num()
	"""

	target_item = self.dim_items_index.get(item_id, -1)
	if target_item == -1:
	    write_log(sys._getframe().f_lineno, "cannot get info of item_id:%d" % (item_id) )
	    return

	rec_items = []
	sim_cat_ids = sim_cat_ids[0: self.strategy.get_max_sim_cat_process()]
	count = 0
	timer_total = Timer()
	for i in range(0, len(sim_cat_ids)):
	    (cat_id, sim_value) = sim_cat_ids[i]
	    sim_item_ids = self.cat_to_item_rindex.get(cat_id, [])
	    if len(sim_item_ids) == 0 : 
		write_log(sys._getframe().f_lineno, "cat_id:%d has no item" % (cat_id) )
		continue
	    timer = Timer()
	    count += len(sim_item_ids)
	    res_list = self.__find_sim_item_from_a_list(target_item, sim_item_ids, i)
	    write_log(msg = "__find_sim_item_from_a_list cost time:%f, sim_item_ids size:%d, i:%d" % (timer.get_diff(), len(sim_item_ids), i))
	    rec_items.extend(res_list)
	write_log(msg = "all__find_sim_item_from_a_list cost time:%f, all_sim_item_ids size:%d" % (timer_total.get_diff(), count))

	write_log(msg = 'process item_id:%d, rec_items size:%d' % (item_id, len(rec_items)) )
	rec_items.sort(lambda y,x : cmp(x[1], y[1]))
	rec_items = rec_items[0: self.strategy.get_total_rec_num()]

	final_res = []
	for (id, value) in rec_items:
	    final_res.append(id)

	return final_res
示例#26
0
    def __init__(self):
        with Timer() as self.model_load_time:
            sys.path.insert(0, "/opt/clevertagger-master")
            import clevertagger

            self.tagger = clevertagger.Clevertagger()

            def myprocessor(myinput):
                return self.tagger.tag(
                    [line for line in myinput.rstrip().split("\n")])

            self.processor = myprocessor
示例#27
0
    def __init__(self):
        with Timer() as self.model_load_time:
            import treetaggerwrapper

            self.tagger = treetaggerwrapper.TreeTagger(TAGLANG="de")

            def myprocessor(myinput):
                sents = myinput.split("\n")
                return self.tagger.tag_text(" </s> ".join(sents).split(),
                                            tagonly=True)

            self.processor = myprocessor
示例#28
0
    def __init__(self):
        with Timer() as self.model_load_time:
            from somajo import Tokenizer, SentenceSplitter

            def myprocessor(myinput):
                tokenizer = Tokenizer(language="de")
                sentsplitter = SentenceSplitter(language="de")
                tokenized = tokenizer.tokenize_paragraph(myinput)
                sentsplit = sentsplitter.split(tokenized)
                return sentsplit

            self.processor = myprocessor
示例#29
0
    def __init__(self):
        with Timer() as self.model_load_time:
            from syntok.tokenizer import Tokenizer
            import syntok.segmenter as segmenter

            def myprocessor(myinput):
                tokenizer = Tokenizer(emit_hyphen_or_underscore_sep=True,
                                      replace_not_contraction=False)
                tokenized = tokenizer.tokenize(myinput)
                return segmenter.segment(tokenized)

            self.processor = myprocessor
示例#30
0
    def __init__(self):
        self.home = Path("/opt/RFTagger")
        with Timer() as self.model_load_time:
            # just load the model
            CLASSPATH = ":".join([
                str(self.home / "jars/rft-java-beta13.jar"),
                str(self.home / "jars/jna-4.5.1.jar"),
            ])
            import jpype

            jpype.startJVM(jpype.getDefaultJVMPath(), "-ea",
                           "-Djava.class.path=" + CLASSPATH)
            rftagger = jpype.JPackage("de.sfb833.a4").RFTagger
            fn = jpype.java.io.File(str(self.home / "lib/german.par"))
            model = rftagger.Model(fn)
            self.tagger = rftagger.RFTagger(model)

            # conversion to STTS
            self.stts_converter = rftagger.tagsetconv.ConverterFactory.getConverter(
                "stts")

            # tag corrector
            self.tag_corrector = rftagger.tagcorrector.TagCorrectorFactory.getTagCorrector(
                "german")

            # lemmatizer
            self.lemmatizer = rftagger.lemmatizer.LemmatizerFactory.getLemmatizer(
                "german",
                jpype.java.io.File(
                    str(self.home /
                        "lib/german-rft-tagger-lemma-lexicon-corrected.txt")),
            )

            def myprocessor(myinput):
                # assuming myinput is one sent per line with
                # space-separated tokens
                result = list()
                for sent in myinput.split("\n"):
                    tokens = jpype.java.util.ArrayList()
                    for token in sent.split():
                        tokens.add(token)
                    newsent = list()
                    for entry in zip(tokens, self.tagger.getTags(tokens)):
                        tok, tag = entry
                        corrected_tag = self.tag_corrector.correctTag(tag)
                        stts = self.stts_converter.rftag2tag(tag)
                        lemma = self.lemmatizer.getLemma(tok, corrected_tag)
                        newsent.append((tok, tag, stts, lemma))
                    result.append(newsent)
                return result

            self.processor = myprocessor
示例#31
0
    def __init__(self):
        with Timer() as self.model_load_time:
            # https://spacy.io/usage/linguistic-features#own-annotations
            # https://spacy.io/usage/processing-pipelines#wrapping-models-libraries
            # https://github.com/explosion/spacy-stanfordnlp/blob/master/spacy_stanfordnlp/language.py
            import spacy

            self.nlp = spacy.load("de", disable=["tagger", "ner"])

            def myprocessor(myinput):
                mydoc = self.read_conlldoc(myinput)
                return self.nlp.pipeline[0][1](mydoc)

            self.processor = myprocessor
示例#32
0
def match_and_draw(win):
    with Timer('matching'):
        raw_matches = matcher.knnMatch(desc1, trainDescriptors=desc2, k=2)  # 2
    p1, p2, kp_pairs = filter_matches(kp1, kp2, raw_matches)
    if len(p1) >= 4:
        H, status = cv.findHomography(p1, p2, cv.RANSAC, 5.0)
        print('%d / %d  inliers/matched' % (np.sum(status), len(status)))
        # do not draw outliers (there will be a lot of them)
        kp_pairs = [kpp for kpp, flag in zip(kp_pairs, status) if flag]
    else:
        H, status = None, None
        print('%d matches found, not enough for homography estimation' % len(p1))

        explore_match(win, img1, img2, kp_pairs, None, H)
示例#33
0
    def __find_sim_item_from_a_list(self, target_item, sim_item_ids, idx):
	target_title = target_item[1]
	
	count = 0
	rec_items = []
	for sim_id in sim_item_ids:
	    sim_item = self.dim_items_index.get(sim_id, -1)
	    if sim_item == -1 : continue

	    count += 1
	    if count > 20000: break

	    sim_title = sim_item[1]
	    timer = Timer()
	    val = self.__cal_title_sim(target_title, sim_title)
	    write_log(msg = "__cal_title_sim cost time:%f, target_title size:%d, sim_title size:%d" % (timer.get_diff(), len(target_title), len(sim_title)))
	    rec_items.append( (sim_id, val) )

	num_to_get = self.strategy.num_to_return_of_this_sim_cat(idx)
	if len(rec_items) > num_to_get:
	    rec_items.sort(lambda y,x : cmp(x[1], y[1]))
	    rec_items = rec_items[0: num_to_get]
	return rec_items
 def __init__(self, model):
     self.model = model
     self.timer = Timer()
 def init_audio(event, **kwargs):
     timer = Timer()
     timer.on('tick', on_tick)
     timer.start(engine, 2000, interval=1000)