def OnTimer(timerid): Logger.LogDebug("Py OnTimer:%d" % timerid) from common import Timer Timer.onTimer(timerid) return 0, "success"
def _eval(mymodel, args, data_loader, data_set=None, RELOAD_MODEL=None, use_cuda=False): old_model_path = os.path.join(args.result_dir, RELOAD_MODEL) if RELOAD_MODEL is not None and os.path.exists(old_model_path): mymodel.load_model(old_model_path) LOGGER.info("Reload model successfully~") else: LOGGER.info( f'There is no such file in {old_model_path}, End evaluation') return LOGGER.info(f'reload model {RELOAD_MODEL}') hyper_param = { 'batch_size': 100, 'issave': True, 'result_dir': args.result_dir } timer = Timer() timer.set(args.time_budget) with timer.time_limit('eval'): mymodel.eval_model(data_loader, data_set, hyper_param, rebuild=True)
def __init__(self): rospy.init_node('Gateway', log_level=rospy.DEBUG) server_config_file = rospy.get_param("~server_config_file") self.config = Config(server_config_file) self.pf = ProtocolFactory(self.config) self.run_id = rospy.get_param("run_id") print("runid = ", self.run_id) self.node_list = rosnode.get_node_names() self.timer = Timer() self.monitor = Monitor(self.node_list, self.timer) self._server_request = {} # stores server_request self._event_bus = Queue() self._heartbeat_timeout_job = None self._tele_report_job = None self._report_car_job = None self._report_task_job = None self._service = DrivingTaskService(self._event_bus) self.__client = Connector(self._event_bus, self.config.host, self.config.port) self._handler_map = {} self._event_handler_map = {} self._add_command_handler() self._add_event_handler() self._web_server = WebServer(self.monitor, self._service, self.config) self._tele_control_service = TeleControlService()
def main(): import sys from common import Timer try: img_fn = sys.argv[1] except: img_fn = 'data/baboon.jpg' img = cv.imread(cv.samples.findFile(img_fn)) if img is None: print('Failed to load image file:', img_fn) sys.exit(1) filters = build_filters() with Timer('running single-threaded'): res1 = process(img, filters) with Timer('running multi-threaded'): res2 = process_threaded(img, filters) print('res1 == res2: ', (res1 == res2).all()) cv.imshow('img', img) cv.imshow('result', res2) cv.waitKey() print('Done')
def main(): import sys from common import Timer try: img_fn = sys.argv[1] except: img_fn = 'PythonData/baboon.jpg' img = cv.imread(cv.samples.findFile(img_fn)) if img is None: print('Failed to load image file:', img_fn) sys.exit(1) filters = build_filters() with Timer('running single-threaded'): res1 = process(img, filters) with Timer('running multi-threaded'): res2 = process_threaded(img, filters) print('res1 == res2: ', (res1 == res2).all()) both = np.empty((img.shape[0], img.shape[1]*2, 3), img.dtype) both = cv.hconcat([img, res2]) cv.imshow("Original and filtered image", both) cv.waitKey()
def OnClientProxy(sockfd, type, data, serverobj): """收到客户端的socket请求入口""" print("OnClientProxy:%d,%d,%s" % (sockfd, type, data)) if type == FD_TYPE_ACCEPT: logger.debug('OnClientProxy,type=FD_TYPE_ACCEPT,sock=%d' % sockfd) try: serverobj.add_client_rpc_channel(sockfd) except Exception as e: print("Exception:", e) elif type == FD_TYPE_CLIENT: logger.debug('OnClientProxy,type=FD_TYPE_READ,sock=%s' % sockfd) try: serverobj.handle_client_rpc_channel(sockfd, data) except Exception as e: print("Exception:", e) elif type == FD_TYPE_CLOSE: logger.debug('OnClientProxy,type=FD_TYPE_CLOSE,sock=%s' % sockfd) try: serverobj.del_client_rpc_channel(sockfd) except Exception as e: print("Exception:", e) elif type == FD_TYPE_TIMER: logger.debug('OnClientProxy,type=FD_TYPE_TIMER,sock=%s' % sockfd) from common import Timer Timer.onTimer(sockfd) # timerId
def _train(mymodel, args, data_loader, train_dataset=None, eval_dataset=None, RELOAD_MODEL=None, use_cuda=False): if RELOAD_MODEL is None: LOGGER.info(f'Rebuild model') else: old_model_path = os.path.join(args.result_dir, RELOAD_MODEL) if os.path.exists(old_model_path): mymodel.load_model(old_model_path) LOGGER.info("Reload model successfully~") else: LOGGER.info(f'There is no such file in {old_model_path}, Rebuild model') ##TODO: BATCH_SIZE = 8 if data_loader.sentence_max_len > 200 else 16 if use_cuda: train_param = { 'EPOCH': 30, #45 TODO:15 'batch_size': BATCH_SIZE, #512 TODO:64 'learning_rate_bert': 5e-5, 'learning_rate_upper': 1e-3, #TODO:1e-3 'bert_finetune': True, 'visualize_length': 20, #10 'isshuffle': True, 'result_dir': args.result_dir, 'model_name':'model_test.p' } else: train_param = { 'EPOCH': CPU_EPOCH, #45 'batch_size': CPU_BATCHSIZE, #512 'learning_rate_bert': 5e-5, 'learning_rate_upper': 1e-3, 'bert_finetune': False, 'visualize_length': CPU_VISUAL, #10 'isshuffle': True, 'result_dir': args.result_dir, 'model_name':'model_test.p' } timer = Timer() timer.set(args.time_budget) loss_record = None with timer.time_limit('training'): loss_record, score_record = mymodel.train_model(data_loader, hyper_param=train_param, train_dataset=train_dataset, eval_dataset=eval_dataset) loss_record = np.array(loss_record) loss_save_path = os.path.join(args.result_dir, 'loss_train.txt') loss_img_path = os.path.join(args.result_dir, 'loss.png') np.savetxt(loss_save_path, loss_record) score_record = np.array(score_record) score_save_path = os.path.join(args.result_dir, 'score_train.txt') score_img_path = os.path.join(args.result_dir, 'score.png') np.savetxt(score_save_path, score_record) ### (epochs, 3) loss = np.loadtxt(loss_save_path).reshape(1, -1) score = np.loadtxt(score_save_path).T plot_img(loss, loss_img_path) plot_img(score, score_img_path)
def main(): print('Running....') with Timer('Single threaded'): singlethreaded() with Timer('Multiple threads'): multithreaded()
def main(): print('Running....') with Timer('Single process'): singleprocessed() with Timer('Multiple processes'): multiprocessed()
class AnimationController(object): def __init__(self, model): self.model = model self.timer = Timer() def start(self): try: emitter = AnimationController.animation_timer except: print 'Animation Controller -- No Instance of Timer Running in Class' print 'Please run AnimationController.init()' emitter = None if emitter is not None: self.timer.start(emitter, interval = self.model.interval) self.timer.on('tick', self.next) def stop(self): self.timer.stop() def resume(self): self.timer.resume() def next(self): print "next frame now" self.model.next_frame() @classmethod def init(cls, emitter, message_type='tick'): if not guarded(lambda:cls.animation_timer): cls.animation_timer = Timer() cls.animation_timer.start(emitter, message_type=message_type) return cls.animation_timer
def __init__(self): self.home = Path("/opt/stanford-corenlp-full-2018-10-05/") with Timer() as self.model_load_time: import jpype jpype.startJVM( jpype.getDefaultJVMPath(), "-ea", "-Djava.class.path=" + str(self.home / "stanford-corenlp-3.9.2.jar"), ) ling = jpype.JPackage("edu").stanford.nlp.ling nndep = jpype.JPackage("edu").stanford.nlp.parser.nndep self.parser = nndep.DependencyParser.loadFromModelFile( str(self.home / "stanford-german-corenlp-2018-10-05-models" / "edu/stanford/nlp/models/parser/nndep/UD_German.gz")) def myprocessor(myinput): results = list() for sent in string2doc(myinput, hide_fields=HIDDEN_FIELDS): sent_arr = jpype.java.util.ArrayList() for tok in sent: sent_arr.add(ling.TaggedWord(tok.word, tok.xpos)) results.append(self.parser.predict(sent_arr)) return results self.processor = myprocessor
def _ClearDrawMode(self): with Timer() as t: drawModeAttr = self._modelAPI.GetModelDrawModeAttr() if drawModeAttr: sessionSpec = _GetPropertySpecInSessionLayer(drawModeAttr) if sessionSpec: self._primViewItem.drawModeWidget = None self._primViewItem.treeWidget().closePersistentEditor( self._primViewItem, PrimViewColumnIndex.DRAWMODE) sessionSpec.ClearDefaultValue() sessionSpec.layer.ScheduleRemoveIfInert(sessionSpec) self._refreshFunc(self._primViewItem) else: PrintWarning( self._modelAPI.GetPath(), "Failed to get " "session layer spec for the model:drawMode attribute") return else: PrintWarning(self._modelAPI.GetPath(), "Failed to get " "model:drawMode attribute") return if self._printTiming: t.PrintTime( "clear model:drawMode on <%s> to %s" % (self._modelAPI.GetPath(), self._comboBox.currentText()))
def main(): args = get_args() client = Client('127.0.0.1:8786') ncores = sum(client.ncores().values()) pd.set_option('display.large_repr', 'truncate') pd.set_option('display.max_columns', 0) # noqa pd.set_option('display.max_rows', 1000) # noqa cann_group_df = make_cann_group_df(num_products=100) df = read_df(args, cann_group_df['productKey']) logger.info('Setting index') df = df.set_index('customerKey', drop=True) logger.info('Repartitioning') df = df.repartition(npartitions=ncores) logger.info('Mapping Cann Group') df['cannGroupKey'] = df['productKey'].map(cann_group_df['cannGroupKey']) logger.info('Persisting') df = client.persist(df) logger.info('Cann Groups') for cann_group_key in cann_group_df['cannGroupKey'].unique().tolist(): print('Filtering Cann Group %s' % cann_group_key) cann_df = df[df['cannGroupKey'] == cann_group_key] print('This df: %s' % (len(cann_df), )) with Timer('%s' % (cann_group_key, )): calculate_switching(cann_df) return
def stage(self, value): """Sets the current Usd.Stage object, and emits a signal if it is different from the previous stage. """ validStage = (value is None) or isinstance(value, Usd.Stage) if not validStage: raise ValueError("Expected USD Stage, got: {}".format(repr(value))) if value is not self._stage: if self._pcListener: self._pcListener.Revoke() self._pcListener = None if value is None: with Timer() as t: self._stage = None if self._printTiming: t.PrintTime('close stage') else: self._stage = value if self._stage: from pxr import Tf self._pcListener = \ Tf.Notice.Register(Usd.Notice.ObjectsChanged, self.__OnPrimsChanged, self._stage) self.signalStageReplaced.emit()
def __init__(self): with Timer() as self.model_load_time: from iwnlp.iwnlp_wrapper import IWNLPWrapper from stts2upos import conv_table data_loc = "/opt/iwnlp/IWNLP.Lemmatizer_20181001.json" self.lemmatizer = IWNLPWrapper(lemmatizer_path=data_loc) def myprocessor(myinput): mydoc = string2doc(myinput) for sent in mydoc: for tok in sent: try: matching_lemmas = self.lemmatizer.lemmatize( tok.word, conv_table.get(tok.xpos)) if matching_lemmas is None: tok.lemma = "_" # elif len(matching_lemmas) > 1: # print("lots o lemmas!", matching_lemmas) else: # unclear how to select best alternative # just use first item in list tok.lemma = matching_lemmas[0] except ValueError: tok.lemma = "_" # don't repeat gold pos in output tok.hide_fields(HIDDEN_FIELDS) return mydoc self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: import spacy nlp = spacy.load("de", disable=["tagger", "parser", "ner"]) nlp.add_pipe(nlp.create_pipe("sentencizer")) self.processor = nlp
def __init__(self): self.name = "stanfordnlp" with Timer() as self.model_load_time: from stanfordnlp import Pipeline, Document from stanfordnlp.models.common.conll import CoNLLFile self.pipeline = Pipeline( lang="de", tokenize_pretokenized=True, processors="depparse", # lower batch size so our GPU can cope depparse_batch_size=1000, ) def myprocessor(myinput): # run input through converter to hide fields, etc. self.input_doc = common.Document(myinput, hidden_fields=HIDDEN_FIELDS) modified_input = doc2string(self.input_doc) self.snlp_doc = Document("") self.snlp_doc.conll_file = CoNLLFile(input_str=modified_input) self.snlp_doc.load_annotations() return self.pipeline(self.snlp_doc) self.processor = myprocessor
def Run(self): ''' The main entry point to launch a process using UsdView. ''' parser = argparse.ArgumentParser(prog=sys.argv[0], description=self.GetHelpDescription()) traceCollector = None with Timer() as totalTimer: self.RegisterPositionals(parser) self.RegisterOptions(parser) arg_parse_result = self.ParseOptions(parser) self.ValidateOptions(arg_parse_result) if arg_parse_result.traceToFile: from pxr import Trace traceCollector = Trace.Collector() traceCollector.pythonTracingEnabled = True traceCollector.enabled = True self.__LaunchProcess(arg_parse_result) if traceCollector: traceCollector.enabled = False if arg_parse_result.timing and arg_parse_result.quitAfterStartup: totalTimer.PrintTime('open and close usdview') if traceCollector: Trace.Reporter.globalReporter.ReportChromeTracingToFile( arg_parse_result.traceToFile)
def read_df(args, product_keys): d_cli = get_data_client(args) with Timer('Read'): df = d_cli.read_df() df = df[df['productKey'].isin(product_keys)] df = df.persist() logger.info('Read data: %s rows, %s mb' % (len(df), df_mem_in_mb(df).compute())) return df
def __init__(self): with Timer() as self.model_load_time: from stanfordnlp import Pipeline self.processor = Pipeline( lang="de", # 'mwt' processor would expand things # like 'am' to 'an dem' processors="tokenize", )
def __init__(self): self.name = "stanfordnlp" with Timer() as self.model_load_time: from stanfordnlp import Pipeline self.processor = Pipeline( lang="de", tokenize_pretokenized=True, processors="tokenize,mwt,pos,lemma", )
def updateVariantSelection(self, index, printTiming): variantSet = self.prim.GetVariantSet(self.variantSetName) currentVariantSelection = variantSet.GetVariantSelection() newVariantSelection = str(self.currentText()) if currentVariantSelection != newVariantSelection: with Timer() as t: variantSet.SetVariantSelection(newVariantSelection) if printTiming: t.PrintTime("change variantSet %s to %s" % (variantSet.GetName(), newVariantSelection))
def calculate_switching(df): tmp_df = df.head(0) result = trans_seq_num(tmp_df) meta = [(col, dtype) for col, dtype in result.dtypes.items()] cols = [col for col, _ in meta] with Timer('Trans Seq num'): trans_seq_num_df = df.map_partitions(trans_seq_num, cols=cols, meta=meta).compute() print(trans_seq_num_df)
def main(): global TIMER TIMER = Timer() oparser = argparse.ArgumentParser(description="intelligent crawling with q-learning") oparser.add_argument("--config-file", dest="configFile", required=True, help="Path to config file (containing MySQL login etc.)") oparser.add_argument("--language-pair", dest="langPair", required=True, help="The 2 language we're interested in, separated by ,") oparser.add_argument("--save-dir", dest="saveDir", default=".", help="Directory that model WIP are saved to. If existing model exists then load it") oparser.add_argument("--save-plots", dest="saveDirPlots", default="plot", help="Directory ") oparser.add_argument("--num-train-hosts", dest="numTrainHosts", type=int, default=1, help="Number of domains to train on") oparser.add_argument("--num-test-hosts", dest="numTestHosts", type=int, default=3, help="Number of domains to test on") oparser.add_argument("--max-crawl", dest="maxCrawl", type=int, default=sys.maxsize, help="Maximum number of pages to crawl") oparser.add_argument("--gamma", dest="gamma", type=float, default=0.999, help="Reward discount") options = oparser.parse_args() np.random.seed() np.set_printoptions(formatter={'float': lambda x: "{0:0.1f}".format(x)}, linewidth=666) if not os.path.exists(options.saveDir): os.makedirs(options.saveDir, exist_ok=True) if not os.path.exists("pickled_domains"): os.makedirs("pickled_domains", exist_ok=True) languages = GetLanguages(options.configFile) params = LearningParams(languages, options, languages.maxLangId, languages.GetLang("None")) print("options.numTrainHosts", options.numTrainHosts) #hosts = ["http://vade-retro.fr/"] hosts = ["http://telasmos.org/"] #hosts = ["http://www.buchmann.ch/", "http://telasmos.org/", "http://tagar.es/"] #hosts = ["http://www.visitbritain.com/"] #hostsTest = ["http://vade-retro.fr/"] #hostsTest = ["http://www.visitbritain.com/"] hostsTest = ["http://www.visitbritain.com/", "http://chopescollection.be/", "http://www.bedandbreakfast.eu/"] envs = GetEnvs(options.configFile, languages, hosts[:options.numTrainHosts]) envsTest = GetEnvs(options.configFile, languages, hostsTest[:options.numTestHosts]) tf.reset_default_graph() qn = Qnetwork(params) init = tf.global_variables_initializer() saver = None #tf.train.Saver() with tf.Session() as sess: sess.run(init) Train(params, sess, saver, qn, envs, envsTest)
def __get_rec_item_ids(self, item_id, sim_cat_ids): """ 获取推荐item_id Args: item_id: query的item_id sim_cat_ids : item_id的相似cat_id列表 Returns: [(rec_item_id, sim_value), (rec_item_id, sim_value), ...],长度为strategy.get_total_rec_num() """ target_item = self.dim_items_index.get(item_id, -1) if target_item == -1: write_log(sys._getframe().f_lineno, "cannot get info of item_id:%d" % (item_id) ) return rec_items = [] sim_cat_ids = sim_cat_ids[0: self.strategy.get_max_sim_cat_process()] count = 0 timer_total = Timer() for i in range(0, len(sim_cat_ids)): (cat_id, sim_value) = sim_cat_ids[i] sim_item_ids = self.cat_to_item_rindex.get(cat_id, []) if len(sim_item_ids) == 0 : write_log(sys._getframe().f_lineno, "cat_id:%d has no item" % (cat_id) ) continue timer = Timer() count += len(sim_item_ids) res_list = self.__find_sim_item_from_a_list(target_item, sim_item_ids, i) write_log(msg = "__find_sim_item_from_a_list cost time:%f, sim_item_ids size:%d, i:%d" % (timer.get_diff(), len(sim_item_ids), i)) rec_items.extend(res_list) write_log(msg = "all__find_sim_item_from_a_list cost time:%f, all_sim_item_ids size:%d" % (timer_total.get_diff(), count)) write_log(msg = 'process item_id:%d, rec_items size:%d' % (item_id, len(rec_items)) ) rec_items.sort(lambda y,x : cmp(x[1], y[1])) rec_items = rec_items[0: self.strategy.get_total_rec_num()] final_res = [] for (id, value) in rec_items: final_res.append(id) return final_res
def __init__(self): with Timer() as self.model_load_time: sys.path.insert(0, "/opt/clevertagger-master") import clevertagger self.tagger = clevertagger.Clevertagger() def myprocessor(myinput): return self.tagger.tag( [line for line in myinput.rstrip().split("\n")]) self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: import treetaggerwrapper self.tagger = treetaggerwrapper.TreeTagger(TAGLANG="de") def myprocessor(myinput): sents = myinput.split("\n") return self.tagger.tag_text(" </s> ".join(sents).split(), tagonly=True) self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: from somajo import Tokenizer, SentenceSplitter def myprocessor(myinput): tokenizer = Tokenizer(language="de") sentsplitter = SentenceSplitter(language="de") tokenized = tokenizer.tokenize_paragraph(myinput) sentsplit = sentsplitter.split(tokenized) return sentsplit self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: from syntok.tokenizer import Tokenizer import syntok.segmenter as segmenter def myprocessor(myinput): tokenizer = Tokenizer(emit_hyphen_or_underscore_sep=True, replace_not_contraction=False) tokenized = tokenizer.tokenize(myinput) return segmenter.segment(tokenized) self.processor = myprocessor
def __init__(self): self.home = Path("/opt/RFTagger") with Timer() as self.model_load_time: # just load the model CLASSPATH = ":".join([ str(self.home / "jars/rft-java-beta13.jar"), str(self.home / "jars/jna-4.5.1.jar"), ]) import jpype jpype.startJVM(jpype.getDefaultJVMPath(), "-ea", "-Djava.class.path=" + CLASSPATH) rftagger = jpype.JPackage("de.sfb833.a4").RFTagger fn = jpype.java.io.File(str(self.home / "lib/german.par")) model = rftagger.Model(fn) self.tagger = rftagger.RFTagger(model) # conversion to STTS self.stts_converter = rftagger.tagsetconv.ConverterFactory.getConverter( "stts") # tag corrector self.tag_corrector = rftagger.tagcorrector.TagCorrectorFactory.getTagCorrector( "german") # lemmatizer self.lemmatizer = rftagger.lemmatizer.LemmatizerFactory.getLemmatizer( "german", jpype.java.io.File( str(self.home / "lib/german-rft-tagger-lemma-lexicon-corrected.txt")), ) def myprocessor(myinput): # assuming myinput is one sent per line with # space-separated tokens result = list() for sent in myinput.split("\n"): tokens = jpype.java.util.ArrayList() for token in sent.split(): tokens.add(token) newsent = list() for entry in zip(tokens, self.tagger.getTags(tokens)): tok, tag = entry corrected_tag = self.tag_corrector.correctTag(tag) stts = self.stts_converter.rftag2tag(tag) lemma = self.lemmatizer.getLemma(tok, corrected_tag) newsent.append((tok, tag, stts, lemma)) result.append(newsent) return result self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: # https://spacy.io/usage/linguistic-features#own-annotations # https://spacy.io/usage/processing-pipelines#wrapping-models-libraries # https://github.com/explosion/spacy-stanfordnlp/blob/master/spacy_stanfordnlp/language.py import spacy self.nlp = spacy.load("de", disable=["tagger", "ner"]) def myprocessor(myinput): mydoc = self.read_conlldoc(myinput) return self.nlp.pipeline[0][1](mydoc) self.processor = myprocessor
def match_and_draw(win): with Timer('matching'): raw_matches = matcher.knnMatch(desc1, trainDescriptors=desc2, k=2) # 2 p1, p2, kp_pairs = filter_matches(kp1, kp2, raw_matches) if len(p1) >= 4: H, status = cv.findHomography(p1, p2, cv.RANSAC, 5.0) print('%d / %d inliers/matched' % (np.sum(status), len(status))) # do not draw outliers (there will be a lot of them) kp_pairs = [kpp for kpp, flag in zip(kp_pairs, status) if flag] else: H, status = None, None print('%d matches found, not enough for homography estimation' % len(p1)) explore_match(win, img1, img2, kp_pairs, None, H)
def __find_sim_item_from_a_list(self, target_item, sim_item_ids, idx): target_title = target_item[1] count = 0 rec_items = [] for sim_id in sim_item_ids: sim_item = self.dim_items_index.get(sim_id, -1) if sim_item == -1 : continue count += 1 if count > 20000: break sim_title = sim_item[1] timer = Timer() val = self.__cal_title_sim(target_title, sim_title) write_log(msg = "__cal_title_sim cost time:%f, target_title size:%d, sim_title size:%d" % (timer.get_diff(), len(target_title), len(sim_title))) rec_items.append( (sim_id, val) ) num_to_get = self.strategy.num_to_return_of_this_sim_cat(idx) if len(rec_items) > num_to_get: rec_items.sort(lambda y,x : cmp(x[1], y[1])) rec_items = rec_items[0: num_to_get] return rec_items
def __init__(self, model): self.model = model self.timer = Timer()
def init_audio(event, **kwargs): timer = Timer() timer.on('tick', on_tick) timer.start(engine, 2000, interval=1000)