def main(): import sys from common import Timer try: img_fn = sys.argv[1] except: img_fn = 'data/baboon.jpg' img = cv.imread(cv.samples.findFile(img_fn)) if img is None: print('Failed to load image file:', img_fn) sys.exit(1) filters = build_filters() with Timer('running single-threaded'): res1 = process(img, filters) with Timer('running multi-threaded'): res2 = process_threaded(img, filters) print('res1 == res2: ', (res1 == res2).all()) cv.imshow('img', img) cv.imshow('result', res2) cv.waitKey() print('Done')
def main(): import sys from common import Timer try: img_fn = sys.argv[1] except: img_fn = 'PythonData/baboon.jpg' img = cv.imread(cv.samples.findFile(img_fn)) if img is None: print('Failed to load image file:', img_fn) sys.exit(1) filters = build_filters() with Timer('running single-threaded'): res1 = process(img, filters) with Timer('running multi-threaded'): res2 = process_threaded(img, filters) print('res1 == res2: ', (res1 == res2).all()) both = np.empty((img.shape[0], img.shape[1]*2, 3), img.dtype) both = cv.hconcat([img, res2]) cv.imshow("Original and filtered image", both) cv.waitKey()
def main(): print('Running....') with Timer('Single threaded'): singlethreaded() with Timer('Multiple threads'): multithreaded()
def main(): print('Running....') with Timer('Single process'): singleprocessed() with Timer('Multiple processes'): multiprocessed()
def __init__(self): rospy.init_node('Gateway', log_level=rospy.DEBUG) server_config_file = rospy.get_param("~server_config_file") self.config = Config(server_config_file) self.pf = ProtocolFactory(self.config) self.run_id = rospy.get_param("run_id") print("runid = ", self.run_id) self.node_list = rosnode.get_node_names() self.timer = Timer() self.monitor = Monitor(self.node_list, self.timer) self._server_request = {} # stores server_request self._event_bus = Queue() self._heartbeat_timeout_job = None self._tele_report_job = None self._report_car_job = None self._report_task_job = None self._service = DrivingTaskService(self._event_bus) self.__client = Connector(self._event_bus, self.config.host, self.config.port) self._handler_map = {} self._event_handler_map = {} self._add_command_handler() self._add_event_handler() self._web_server = WebServer(self.monitor, self._service, self.config) self._tele_control_service = TeleControlService()
def _eval(mymodel, args, data_loader, data_set=None, RELOAD_MODEL=None, use_cuda=False): old_model_path = os.path.join(args.result_dir, RELOAD_MODEL) if RELOAD_MODEL is not None and os.path.exists(old_model_path): mymodel.load_model(old_model_path) LOGGER.info("Reload model successfully~") else: LOGGER.info( f'There is no such file in {old_model_path}, End evaluation') return LOGGER.info(f'reload model {RELOAD_MODEL}') hyper_param = { 'batch_size': 100, 'issave': True, 'result_dir': args.result_dir } timer = Timer() timer.set(args.time_budget) with timer.time_limit('eval'): mymodel.eval_model(data_loader, data_set, hyper_param, rebuild=True)
def __init__(self): self.home = Path("/opt/stanford-corenlp-full-2018-10-05/") with Timer() as self.model_load_time: import jpype jpype.startJVM( jpype.getDefaultJVMPath(), "-ea", "-Djava.class.path=" + str(self.home / "stanford-corenlp-3.9.2.jar"), ) ling = jpype.JPackage("edu").stanford.nlp.ling nndep = jpype.JPackage("edu").stanford.nlp.parser.nndep self.parser = nndep.DependencyParser.loadFromModelFile( str(self.home / "stanford-german-corenlp-2018-10-05-models" / "edu/stanford/nlp/models/parser/nndep/UD_German.gz")) def myprocessor(myinput): results = list() for sent in string2doc(myinput, hide_fields=HIDDEN_FIELDS): sent_arr = jpype.java.util.ArrayList() for tok in sent: sent_arr.add(ling.TaggedWord(tok.word, tok.xpos)) results.append(self.parser.predict(sent_arr)) return results self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: import spacy nlp = spacy.load("de", disable=["tagger", "parser", "ner"]) nlp.add_pipe(nlp.create_pipe("sentencizer")) self.processor = nlp
def _ClearDrawMode(self): with Timer() as t: drawModeAttr = self._modelAPI.GetModelDrawModeAttr() if drawModeAttr: sessionSpec = _GetPropertySpecInSessionLayer(drawModeAttr) if sessionSpec: self._primViewItem.drawModeWidget = None self._primViewItem.treeWidget().closePersistentEditor( self._primViewItem, PrimViewColumnIndex.DRAWMODE) sessionSpec.ClearDefaultValue() sessionSpec.layer.ScheduleRemoveIfInert(sessionSpec) self._refreshFunc(self._primViewItem) else: PrintWarning( self._modelAPI.GetPath(), "Failed to get " "session layer spec for the model:drawMode attribute") return else: PrintWarning(self._modelAPI.GetPath(), "Failed to get " "model:drawMode attribute") return if self._printTiming: t.PrintTime( "clear model:drawMode on <%s> to %s" % (self._modelAPI.GetPath(), self._comboBox.currentText()))
def main(): t = Timer() t.reset_cpu_time() logging.info("PREPARE DATASET") train_X, train_Y, test_X, test_Y = load_light_dataset( images_root_dir, target=target, training_set_part=0.8, extension='jpeg') logging.info("CREATE MODEL") model = create_model(network['layers_and_filters'], network['kernel_size'], network['activation'], (IMG_SIZE, IMG_SIZE, levels), network['dropout_rate'], network['optimizer'], network['learning_rate'], output_size=train_Y.shape[1]) t.get_cpu_time("PREPARATION") logging.info("TRAIN") try: if options.train == 'true': raise Exception('Force train model') model = models.load_model(model_name) except: fit_one_at_time(model, train_X, train_Y, epochs=network['epochs']) model.save(model_name) t.get_cpu_time("TRAIN") logging.info("TEST on TRAIN") score_one_at_time(model, train_X, train_Y) t.get_cpu_time("TEST on TRAIN") logging.info("TEST") score_one_at_time(model, test_X, test_Y) t.get_cpu_time("TEST")
def main(): args = get_args() client = Client('127.0.0.1:8786') ncores = sum(client.ncores().values()) pd.set_option('display.large_repr', 'truncate') pd.set_option('display.max_columns', 0) # noqa pd.set_option('display.max_rows', 1000) # noqa cann_group_df = make_cann_group_df(num_products=100) df = read_df(args, cann_group_df['productKey']) logger.info('Setting index') df = df.set_index('customerKey', drop=True) logger.info('Repartitioning') df = df.repartition(npartitions=ncores) logger.info('Mapping Cann Group') df['cannGroupKey'] = df['productKey'].map(cann_group_df['cannGroupKey']) logger.info('Persisting') df = client.persist(df) logger.info('Cann Groups') for cann_group_key in cann_group_df['cannGroupKey'].unique().tolist(): print('Filtering Cann Group %s' % cann_group_key) cann_df = df[df['cannGroupKey'] == cann_group_key] print('This df: %s' % (len(cann_df), )) with Timer('%s' % (cann_group_key, )): calculate_switching(cann_df) return
def stage(self, value): """Sets the current Usd.Stage object, and emits a signal if it is different from the previous stage. """ validStage = (value is None) or isinstance(value, Usd.Stage) if not validStage: raise ValueError("Expected USD Stage, got: {}".format(repr(value))) if value is not self._stage: if self._pcListener: self._pcListener.Revoke() self._pcListener = None if value is None: with Timer() as t: self._stage = None if self._printTiming: t.PrintTime('close stage') else: self._stage = value if self._stage: from pxr import Tf self._pcListener = \ Tf.Notice.Register(Usd.Notice.ObjectsChanged, self.__OnPrimsChanged, self._stage) self.signalStageReplaced.emit()
def __init__(self): self.name = "stanfordnlp" with Timer() as self.model_load_time: from stanfordnlp import Pipeline, Document from stanfordnlp.models.common.conll import CoNLLFile self.pipeline = Pipeline( lang="de", tokenize_pretokenized=True, processors="depparse", # lower batch size so our GPU can cope depparse_batch_size=1000, ) def myprocessor(myinput): # run input through converter to hide fields, etc. self.input_doc = common.Document(myinput, hidden_fields=HIDDEN_FIELDS) modified_input = doc2string(self.input_doc) self.snlp_doc = Document("") self.snlp_doc.conll_file = CoNLLFile(input_str=modified_input) self.snlp_doc.load_annotations() return self.pipeline(self.snlp_doc) self.processor = myprocessor
def Run(self): ''' The main entry point to launch a process using UsdView. ''' parser = argparse.ArgumentParser(prog=sys.argv[0], description=self.GetHelpDescription()) traceCollector = None with Timer() as totalTimer: self.RegisterPositionals(parser) self.RegisterOptions(parser) arg_parse_result = self.ParseOptions(parser) self.ValidateOptions(arg_parse_result) if arg_parse_result.traceToFile: from pxr import Trace traceCollector = Trace.Collector() traceCollector.pythonTracingEnabled = True traceCollector.enabled = True self.__LaunchProcess(arg_parse_result) if traceCollector: traceCollector.enabled = False if arg_parse_result.timing and arg_parse_result.quitAfterStartup: totalTimer.PrintTime('open and close usdview') if traceCollector: Trace.Reporter.globalReporter.ReportChromeTracingToFile( arg_parse_result.traceToFile)
def __init__(self): with Timer() as self.model_load_time: from iwnlp.iwnlp_wrapper import IWNLPWrapper from stts2upos import conv_table data_loc = "/opt/iwnlp/IWNLP.Lemmatizer_20181001.json" self.lemmatizer = IWNLPWrapper(lemmatizer_path=data_loc) def myprocessor(myinput): mydoc = string2doc(myinput) for sent in mydoc: for tok in sent: try: matching_lemmas = self.lemmatizer.lemmatize( tok.word, conv_table.get(tok.xpos)) if matching_lemmas is None: tok.lemma = "_" # elif len(matching_lemmas) > 1: # print("lots o lemmas!", matching_lemmas) else: # unclear how to select best alternative # just use first item in list tok.lemma = matching_lemmas[0] except ValueError: tok.lemma = "_" # don't repeat gold pos in output tok.hide_fields(HIDDEN_FIELDS) return mydoc self.processor = myprocessor
def _train(mymodel, args, data_loader, train_dataset=None, eval_dataset=None, RELOAD_MODEL=None, use_cuda=False): if RELOAD_MODEL is None: LOGGER.info(f'Rebuild model') else: old_model_path = os.path.join(args.result_dir, RELOAD_MODEL) if os.path.exists(old_model_path): mymodel.load_model(old_model_path) LOGGER.info("Reload model successfully~") else: LOGGER.info(f'There is no such file in {old_model_path}, Rebuild model') ##TODO: BATCH_SIZE = 8 if data_loader.sentence_max_len > 200 else 16 if use_cuda: train_param = { 'EPOCH': 30, #45 TODO:15 'batch_size': BATCH_SIZE, #512 TODO:64 'learning_rate_bert': 5e-5, 'learning_rate_upper': 1e-3, #TODO:1e-3 'bert_finetune': True, 'visualize_length': 20, #10 'isshuffle': True, 'result_dir': args.result_dir, 'model_name':'model_test.p' } else: train_param = { 'EPOCH': CPU_EPOCH, #45 'batch_size': CPU_BATCHSIZE, #512 'learning_rate_bert': 5e-5, 'learning_rate_upper': 1e-3, 'bert_finetune': False, 'visualize_length': CPU_VISUAL, #10 'isshuffle': True, 'result_dir': args.result_dir, 'model_name':'model_test.p' } timer = Timer() timer.set(args.time_budget) loss_record = None with timer.time_limit('training'): loss_record, score_record = mymodel.train_model(data_loader, hyper_param=train_param, train_dataset=train_dataset, eval_dataset=eval_dataset) loss_record = np.array(loss_record) loss_save_path = os.path.join(args.result_dir, 'loss_train.txt') loss_img_path = os.path.join(args.result_dir, 'loss.png') np.savetxt(loss_save_path, loss_record) score_record = np.array(score_record) score_save_path = os.path.join(args.result_dir, 'score_train.txt') score_img_path = os.path.join(args.result_dir, 'score.png') np.savetxt(score_save_path, score_record) ### (epochs, 3) loss = np.loadtxt(loss_save_path).reshape(1, -1) score = np.loadtxt(score_save_path).T plot_img(loss, loss_img_path) plot_img(score, score_img_path)
def read_df(args, product_keys): d_cli = get_data_client(args) with Timer('Read'): df = d_cli.read_df() df = df[df['productKey'].isin(product_keys)] df = df.persist() logger.info('Read data: %s rows, %s mb' % (len(df), df_mem_in_mb(df).compute())) return df
def calculate_switching(df): tmp_df = df.head(0) result = trans_seq_num(tmp_df) meta = [(col, dtype) for col, dtype in result.dtypes.items()] cols = [col for col, _ in meta] with Timer('Trans Seq num'): trans_seq_num_df = df.map_partitions(trans_seq_num, cols=cols, meta=meta).compute() print(trans_seq_num_df)
def __init__(self): with Timer() as self.model_load_time: from stanfordnlp import Pipeline self.processor = Pipeline( lang="de", # 'mwt' processor would expand things # like 'am' to 'an dem' processors="tokenize", )
def __init__(self): self.name = "stanfordnlp" with Timer() as self.model_load_time: from stanfordnlp import Pipeline self.processor = Pipeline( lang="de", tokenize_pretokenized=True, processors="tokenize,mwt,pos,lemma", )
def updateVariantSelection(self, index, printTiming): variantSet = self.prim.GetVariantSet(self.variantSetName) currentVariantSelection = variantSet.GetVariantSelection() newVariantSelection = str(self.currentText()) if currentVariantSelection != newVariantSelection: with Timer() as t: variantSet.SetVariantSelection(newVariantSelection) if printTiming: t.PrintTime("change variantSet %s to %s" % (variantSet.GetName(), newVariantSelection))
def main(): t = Timer() logging.info("PREPARE DATASET") # Create train and test set train_X, train_Y, test_X, test_Y = load_dataset_properties( dataset, target=0, training_set_part=0.8) train_names = train_X[:, 0] train_X = train_X[:, 1:] test_names = test_X[:, 0] test_X = test_X[:, 1:] train_size = train_X.shape[0] test_size = test_X.shape[0] input_size = train_X.shape[1] output_size = train_Y.shape[1] try: if options.train == 'true': raise Exception('Force train model') model = models.load_model(model_name) except: # create the model logging.info("CREATE MODEL") model = create_model(input_size=input_size, output_size=output_size, n_layers=network['n_layers'], n_neurons=network['n_neurons'], activation_function=network['activation'], learning_rate=network['learning_rate'], dropout_rate=network['dropout_rate'], optimizer=network['optimizer']) t.reset_cpu_time() # train the model results = model.fit( x=train_X, y=train_Y, epochs=network['epochs'], ) t.get_cpu_time("TRAIN") model.save(model_name) logging.info("TEST on TRAIN") t.reset_cpu_time() predict_and_score(model, np.asarray(train_X).astype(np.float32), train_Y) t.get_cpu_time("TEST on TRAIN") logging.info("TEST") predict_and_score(model, np.asarray(test_X).astype(np.float32), test_Y) t.get_cpu_time("TEST")
def main(): global TIMER TIMER = Timer() oparser = argparse.ArgumentParser(description="intelligent crawling with q-learning") oparser.add_argument("--config-file", dest="configFile", required=True, help="Path to config file (containing MySQL login etc.)") oparser.add_argument("--language-pair", dest="langPair", required=True, help="The 2 language we're interested in, separated by ,") oparser.add_argument("--save-dir", dest="saveDir", default=".", help="Directory that model WIP are saved to. If existing model exists then load it") oparser.add_argument("--save-plots", dest="saveDirPlots", default="plot", help="Directory ") oparser.add_argument("--num-train-hosts", dest="numTrainHosts", type=int, default=1, help="Number of domains to train on") oparser.add_argument("--num-test-hosts", dest="numTestHosts", type=int, default=3, help="Number of domains to test on") oparser.add_argument("--max-crawl", dest="maxCrawl", type=int, default=sys.maxsize, help="Maximum number of pages to crawl") oparser.add_argument("--gamma", dest="gamma", type=float, default=0.999, help="Reward discount") options = oparser.parse_args() np.random.seed() np.set_printoptions(formatter={'float': lambda x: "{0:0.1f}".format(x)}, linewidth=666) if not os.path.exists(options.saveDir): os.makedirs(options.saveDir, exist_ok=True) if not os.path.exists("pickled_domains"): os.makedirs("pickled_domains", exist_ok=True) languages = GetLanguages(options.configFile) params = LearningParams(languages, options, languages.maxLangId, languages.GetLang("None")) print("options.numTrainHosts", options.numTrainHosts) #hosts = ["http://vade-retro.fr/"] hosts = ["http://telasmos.org/"] #hosts = ["http://www.buchmann.ch/", "http://telasmos.org/", "http://tagar.es/"] #hosts = ["http://www.visitbritain.com/"] #hostsTest = ["http://vade-retro.fr/"] #hostsTest = ["http://www.visitbritain.com/"] hostsTest = ["http://www.visitbritain.com/", "http://chopescollection.be/", "http://www.bedandbreakfast.eu/"] envs = GetEnvs(options.configFile, languages, hosts[:options.numTrainHosts]) envsTest = GetEnvs(options.configFile, languages, hostsTest[:options.numTestHosts]) tf.reset_default_graph() qn = Qnetwork(params) init = tf.global_variables_initializer() saver = None #tf.train.Saver() with tf.Session() as sess: sess.run(init) Train(params, sess, saver, qn, envs, envsTest)
def __init__(self): with Timer() as self.model_load_time: import treetaggerwrapper self.tagger = treetaggerwrapper.TreeTagger(TAGLANG="de") def myprocessor(myinput): sents = myinput.split("\n") return self.tagger.tag_text(" </s> ".join(sents).split(), tagonly=True) self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: sys.path.insert(0, "/opt/clevertagger-master") import clevertagger self.tagger = clevertagger.Clevertagger() def myprocessor(myinput): return self.tagger.tag( [line for line in myinput.rstrip().split("\n")]) self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: from somajo import Tokenizer, SentenceSplitter def myprocessor(myinput): tokenizer = Tokenizer(language="de") sentsplitter = SentenceSplitter(language="de") tokenized = tokenizer.tokenize_paragraph(myinput) sentsplit = sentsplitter.split(tokenized) return sentsplit self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: from syntok.tokenizer import Tokenizer import syntok.segmenter as segmenter def myprocessor(myinput): tokenizer = Tokenizer(emit_hyphen_or_underscore_sep=True, replace_not_contraction=False) tokenized = tokenizer.tokenize(myinput) return segmenter.segment(tokenized) self.processor = myprocessor
def __init__(self): self.home = Path("/opt/RFTagger") with Timer() as self.model_load_time: # just load the model CLASSPATH = ":".join([ str(self.home / "jars/rft-java-beta13.jar"), str(self.home / "jars/jna-4.5.1.jar"), ]) import jpype jpype.startJVM(jpype.getDefaultJVMPath(), "-ea", "-Djava.class.path=" + CLASSPATH) rftagger = jpype.JPackage("de.sfb833.a4").RFTagger fn = jpype.java.io.File(str(self.home / "lib/german.par")) model = rftagger.Model(fn) self.tagger = rftagger.RFTagger(model) # conversion to STTS self.stts_converter = rftagger.tagsetconv.ConverterFactory.getConverter( "stts") # tag corrector self.tag_corrector = rftagger.tagcorrector.TagCorrectorFactory.getTagCorrector( "german") # lemmatizer self.lemmatizer = rftagger.lemmatizer.LemmatizerFactory.getLemmatizer( "german", jpype.java.io.File( str(self.home / "lib/german-rft-tagger-lemma-lexicon-corrected.txt")), ) def myprocessor(myinput): # assuming myinput is one sent per line with # space-separated tokens result = list() for sent in myinput.split("\n"): tokens = jpype.java.util.ArrayList() for token in sent.split(): tokens.add(token) newsent = list() for entry in zip(tokens, self.tagger.getTags(tokens)): tok, tag = entry corrected_tag = self.tag_corrector.correctTag(tag) stts = self.stts_converter.rftag2tag(tag) lemma = self.lemmatizer.getLemma(tok, corrected_tag) newsent.append((tok, tag, stts, lemma)) result.append(newsent) return result self.processor = myprocessor
def __init__(self): with Timer() as self.model_load_time: # https://spacy.io/usage/linguistic-features#own-annotations # https://spacy.io/usage/processing-pipelines#wrapping-models-libraries # https://github.com/explosion/spacy-stanfordnlp/blob/master/spacy_stanfordnlp/language.py import spacy self.nlp = spacy.load("de", disable=["tagger", "ner"]) def myprocessor(myinput): mydoc = self.read_conlldoc(myinput) return self.nlp.pipeline[0][1](mydoc) self.processor = myprocessor
def match_and_draw(win): with Timer('matching'): raw_matches = matcher.knnMatch(desc1, trainDescriptors=desc2, k=2) # 2 p1, p2, kp_pairs = filter_matches(kp1, kp2, raw_matches) if len(p1) >= 4: H, status = cv.findHomography(p1, p2, cv.RANSAC, 5.0) print('%d / %d inliers/matched' % (np.sum(status), len(status))) # do not draw outliers (there will be a lot of them) kp_pairs = [kpp for kpp, flag in zip(kp_pairs, status) if flag] else: H, status = None, None print('%d matches found, not enough for homography estimation' % len(p1)) explore_match(win, img1, img2, kp_pairs, None, H)