def main(): # Measure run time timer = StopWatch() timer.start() # Setup models db = getDBInstance(config) summaries = Summaries(db.tick_summary_db) stepSeconds = { 'minutely': 60, 'hourly': 60 * 60, 'daily': 24 * 60 * 60, 'weekly': 7 * 24 * 60 * 60 } maxCompletions = { 'minutely': None, 'hourly': MAX_COMPLETION_HOURLY, 'daily': MAX_COMPLETION_DAILY, 'weekly': None } noiseCycles = { 'minutely': None, 'hourly': COMPLETION_CYCLE_HOURLY, 'daily': None, 'weekly': None } now = datetime.datetime.now().timestamp() after = now - EXPORT_WITHIN_SECONDS for exchanger in EXCHANGERS: for unit in UNITS: sums = list(summaries.all(exchanger, unit, after=after)) logger.debug('Copying {e}\'s {u} ticks to np.array, #items={n}...' .format(e=exchanger, u=unit, n=len(sums))) step = stepSeconds[unit] maxCompletion = maxCompletions[unit] dates = seriesDate(sums, step) logger.debug('Completing {e}\'s {u} ticks to np.array, maxN={maxN}...' .format(e=exchanger, u=unit, maxN=maxCompletion)) opts = { 'maxN': maxCompletion } if noiseCycles[unit] is not None: opts['noise'] = lambda n:completionNoise(n, cycle=noiseCycles[unit], scale=COMPLETION_NOISE_SCALE) completes = { 'askMax': completion(lambda s:s.askMax, sums, step, **opts), 'askMin': completion(lambda s:s.askMin, sums, step, **opts), 'askAverage': completion(lambda s:s.askAverage, sums, step, **opts), 'askOpen': completion(lambda s:s.askOpen, sums, step, **opts), 'askClose': completion(lambda s:s.askClose, sums, step, **opts) } for ty in completes: completed = completes[ty] if len(dates) != len(completed): raise Exception('Length unmatch, #date={date}, #completed={completed}.' .format(date=len(dates), completed=len(completed))) savenpy(config, completed, exchanger, unit, ty) savenpy(config, dates, exchanger, unit, 'date') # Finished seconds = timer.stop() logger.debug('End export, elapsed={s:.2f}s'.format(s=seconds))
def prepare_data_and_vocab(cfg): load_data_file = ["books_large_p1.txt", "books_large_p2.txt"] #load_data_file = "books_100k.txt" save_data_path = os.path.join(cfg.prepro_dir, "data") save_vocab_path = os.path.join(cfg.prepro_dir, "vocab") StopWatch.go('Total') if (not os.path.exists(save_data_path) or not os.path.exists(save_vocab_path) or cfg.is_reload_prepro): print('Start preprocessing data and building vocabulary!') if isinstance(load_data_file, (list, tuple)): load_data_path = [ *map(lambda fn: os.path.join(cfg.data_dir, fn), load_data_file) ] book_procs = BookCorpusMultiProcessor.from_multiple_files( file_paths=load_data_path, min_len=cfg.prepro_min_len, max_len=cfg.prepro_max_len) sents, counter = BookCorpusMultiProcessor.multi_process(book_procs) else: load_data_path = os.path.join(cfg.data_dir, load_data_file) book_procs = BookCorpusMultiProcessor(file_path=load_data_path, min_len=cfg.prepro_min_len, max_len=cfg.prepro_max_len) sents, counter = book_procs.process() vocab = Vocab(counter=counter, max_size=cfg.vocab_size, embed_dim=cfg.embed_dim, is_load_glove=cfg.is_load_glove) sents = vocab.numericalize_sents(sents) if not os.path.exists(cfg.prepro_dir): os.makedirs(cfg.prepro_dir) with StopWatch('Saving text'): np.savetxt(save_data_path, sents, fmt="%s") with StopWatch('Pickling vocab'): vocab.pickle(save_vocab_path) else: print('Previously processed files will be used!') vocab = Vocab.unpickle(save_vocab_path) StopWatch.stop('Total') return vocab
def main(): # Measure run time timer = StopWatch() timer.start() # Execution types = ['askAverage', 'askOpen', 'askClose'] runForAll(EXCHANGERS, UNITS, types) # Finished seconds = timer.stop() logger.debug('End supervising, elapsed={s:.2f}s'.format(s=seconds))
def main(): # Measure run time timer = StopWatch() timer.start() # Setup models db = getDBInstance(config) ticksModel = Ticks(db.tick_db) dashb = Dashboard(uri=AIMAI_DB_URI, logger=logger) dashb.requestLogin(USERNAME, PASSWORD) sync(dashb, ticksModel) # Finished seconds = timer.stop() logger.debug('End synchronization, elapsed={s:.2f}s'.format(s=seconds))
def main(): # Measure run time timer = StopWatch() timer.start() # Setup models db = getDBInstance(config) ticks = Ticks(db.tick_db) summaries = Summaries(db.tick_summary_db) for exchanger in EXCHANGERS: for unit in UNITS: logger.debug('Summarizing, exchanger={exchanger}, unit={unit}.' .format(exchanger=exchanger, unit=unit)) latest = summaries.one(exchanger, unit) if latest is not None: start = latest.date else: start = None sums = ticksBy(ticks, exchanger, unit, dateStart=start) logger.info('Saving summary...') summaries.saveAll(exchanger, unit, sums) # Finished seconds = timer.stop() logger.debug('End summarization, elapsed={s:.2f}s'.format(s=seconds))
yModel.compile(optimizer='adam', loss='binary_crossentropy', metrics=[round_binary_accuracy]) yHist = fit(XbhTrain, ybh0Train, yModel, epochs=EPOCHS, batchSize=BATCH_SIZE) yTrainEnd = datetime.datetime.now() yAcc = yHist.history['val_round_binary_accuracy'][-1] plotHistory('../figures/yhist.svg', yHist, keyAccuracy='round_binary_accuracy') logger.warn(('Training done, trendAcc.={trend:.2f}, ' + '#samples={ns:.0f}, #epochs={ne:.0f}, #batchSize={nb:.0f}, ' + 'trendTime={tt:.1f}.') .format(trend=yAcc * 100, ns=trainSize, ne=EPOCHS, nb=BATCH_SIZE, tt=(yTrainEnd - yTrainStart).total_seconds())) logger.info('Training accuracies are: trend={trend:.2f}.' .format(trend=yAcc * 100)) if yAcc < ACCURACY_MIN: logger.error('Aborted because of insufficient accuracy.') exit(1) saveModel(config, yModel, 'trend') logger.debug('Predicting, Xbh.shape={x}.'.format(x=Xbh.shape)) logger.info('#Ticks={tick}, #Train={train}, #Predicted={predict}.' .format(tick=Xbh.shape[0], train=trainSize, predict=Xbh.shape[0] - trainSize)) # Finished seconds = timer.stop() logger.info('Training completed, elapsed={s:.2f}s'.format(s=seconds))
def main(): iprint("hello") def foo(): iprint("inside foo") bar() def bar(): iprint("inside bar") foo() bar() a = [1, 2, 3, 4, 5, 6] iprint(a) b = [i for i in range(1000)] iprint(b) watch = StopWatch() log1 = NameLog() log2 = NameLog() log1.track("test1", "test2") log2.track("accuracy") test1 = 4 def baz(log1, log2): test2 = 42 log1.record() for accuracy in range(10): log2.record() baz(log1, log2) print(log1.tracked) print(log2.tracked) @CodeMemo def longjob(t, ret): print("Sleeping...") sleep(t) print("Done.") return ret def worker(): return longjob(5, ret=4) watch.start() res = worker() watch.stop() iprint(f"Got {res}") iprint(f"Elapsed {watch.elapsed()}") exit() tagger = Tagger() print("Basic") print(f"# possible tags: {tagger.size():,}") for i in range(5): print(tagger.make()) print() tagger = Tagger() seen = set() tag = tagger.make() while tag not in seen: seen.add(tag) tag = tagger.make() assert len(seen) == tagger.size() print("Space size matches.") print() tagger = Tagger(10) print("Numeric 10") print(f"# possible tags: {tagger.size():,}") for i in range(5): print(tagger.make()) print() tagger = Tagger("aaa") print("Letters 3") print(f"# possible tags: {tagger.size():,}") for i in range(5): print(tagger.make()) print()