def main(): # Measure run time timer = StopWatch() timer.start() # Setup models db = getDBInstance(config) summaries = Summaries(db.tick_summary_db) stepSeconds = { 'minutely': 60, 'hourly': 60 * 60, 'daily': 24 * 60 * 60, 'weekly': 7 * 24 * 60 * 60 } maxCompletions = { 'minutely': None, 'hourly': MAX_COMPLETION_HOURLY, 'daily': MAX_COMPLETION_DAILY, 'weekly': None } noiseCycles = { 'minutely': None, 'hourly': COMPLETION_CYCLE_HOURLY, 'daily': None, 'weekly': None } now = datetime.datetime.now().timestamp() after = now - EXPORT_WITHIN_SECONDS for exchanger in EXCHANGERS: for unit in UNITS: sums = list(summaries.all(exchanger, unit, after=after)) logger.debug('Copying {e}\'s {u} ticks to np.array, #items={n}...' .format(e=exchanger, u=unit, n=len(sums))) step = stepSeconds[unit] maxCompletion = maxCompletions[unit] dates = seriesDate(sums, step) logger.debug('Completing {e}\'s {u} ticks to np.array, maxN={maxN}...' .format(e=exchanger, u=unit, maxN=maxCompletion)) opts = { 'maxN': maxCompletion } if noiseCycles[unit] is not None: opts['noise'] = lambda n:completionNoise(n, cycle=noiseCycles[unit], scale=COMPLETION_NOISE_SCALE) completes = { 'askMax': completion(lambda s:s.askMax, sums, step, **opts), 'askMin': completion(lambda s:s.askMin, sums, step, **opts), 'askAverage': completion(lambda s:s.askAverage, sums, step, **opts), 'askOpen': completion(lambda s:s.askOpen, sums, step, **opts), 'askClose': completion(lambda s:s.askClose, sums, step, **opts) } for ty in completes: completed = completes[ty] if len(dates) != len(completed): raise Exception('Length unmatch, #date={date}, #completed={completed}.' .format(date=len(dates), completed=len(completed))) savenpy(config, completed, exchanger, unit, ty) savenpy(config, dates, exchanger, unit, 'date') # Finished seconds = timer.stop() logger.debug('End export, elapsed={s:.2f}s'.format(s=seconds))
def main(): # Measure run time timer = StopWatch() timer.start() # Execution types = ['askAverage', 'askOpen', 'askClose'] runForAll(EXCHANGERS, UNITS, types) # Finished seconds = timer.stop() logger.debug('End supervising, elapsed={s:.2f}s'.format(s=seconds))
def main(): # Measure run time timer = StopWatch() timer.start() # Setup models db = getDBInstance(config) ticksModel = Ticks(db.tick_db) dashb = Dashboard(uri=AIMAI_DB_URI, logger=logger) dashb.requestLogin(USERNAME, PASSWORD) sync(dashb, ticksModel) # Finished seconds = timer.stop() logger.debug('End synchronization, elapsed={s:.2f}s'.format(s=seconds))
def test_sleep_1_iterated(): with StopWatch("time 5 times 'sleep(1)': ", ndigits=3) as tmr: for i in range(5): sleep(1) time = tmr.timelapse() print(i, time) assert time >= 1.00 assert time <= 1.01 time = tmr.time assert time > 5.00 assert time < 5.05
def __init__(self, do_print, do_plot, batch_size, data_collector, plot_every_batch=10, print_every_batch=100): self.do_print = do_print self.do_plot = do_plot self.batch_size = batch_size self.data_collector = data_collector self.plot_every = batch_size * plot_every_batch self.print_every = print_every_batch if self.do_plot: self.plotter = Plotter(2, 1) self.stop_watch = StopWatch() #self.stop_watch.start() #self.num_optimizer_updates = 0 self.plot_running_loss = 0.0 self.plot_losses, self.plot_avg_losses, self.plot_seen_samples = [],[],[] self.acc_loss, self.acc_points_reported = 0, 0
def main(): # Measure run time timer = StopWatch() timer.start() # Setup models db = getDBInstance(config) ticks = Ticks(db.tick_db) summaries = Summaries(db.tick_summary_db) for exchanger in EXCHANGERS: for unit in UNITS: logger.debug('Summarizing, exchanger={exchanger}, unit={unit}.' .format(exchanger=exchanger, unit=unit)) latest = summaries.one(exchanger, unit) if latest is not None: start = latest.date else: start = None sums = ticksBy(ticks, exchanger, unit, dateStart=start) logger.info('Saving summary...') summaries.saveAll(exchanger, unit, sums) # Finished seconds = timer.stop() logger.debug('End summarization, elapsed={s:.2f}s'.format(s=seconds))
def prepare_data_and_vocab(cfg): load_data_file = ["books_large_p1.txt", "books_large_p2.txt"] #load_data_file = "books_100k.txt" save_data_path = os.path.join(cfg.prepro_dir, "data") save_vocab_path = os.path.join(cfg.prepro_dir, "vocab") StopWatch.go('Total') if (not os.path.exists(save_data_path) or not os.path.exists(save_vocab_path) or cfg.is_reload_prepro): print('Start preprocessing data and building vocabulary!') if isinstance(load_data_file, (list, tuple)): load_data_path = [ *map(lambda fn: os.path.join(cfg.data_dir, fn), load_data_file) ] book_procs = BookCorpusMultiProcessor.from_multiple_files( file_paths=load_data_path, min_len=cfg.prepro_min_len, max_len=cfg.prepro_max_len) sents, counter = BookCorpusMultiProcessor.multi_process(book_procs) else: load_data_path = os.path.join(cfg.data_dir, load_data_file) book_procs = BookCorpusMultiProcessor(file_path=load_data_path, min_len=cfg.prepro_min_len, max_len=cfg.prepro_max_len) sents, counter = book_procs.process() vocab = Vocab(counter=counter, max_size=cfg.vocab_size, embed_dim=cfg.embed_dim, is_load_glove=cfg.is_load_glove) sents = vocab.numericalize_sents(sents) if not os.path.exists(cfg.prepro_dir): os.makedirs(cfg.prepro_dir) with StopWatch('Saving text'): np.savetxt(save_data_path, sents, fmt="%s") with StopWatch('Pickling vocab'): vocab.pickle(save_vocab_path) else: print('Previously processed files will be used!') vocab = Vocab.unpickle(save_vocab_path) StopWatch.stop('Total') return vocab
from learningUtils import validated, to2d, zscore, round_binary_accuracy, underSampling, balance, saveModel from utils import readConfig, getLogger, loadnpy, StopWatch logger = getLogger() config = readConfig('predict.ini') logger.info('Training started.') OFFSET_SAMPLES = config['train'].getint('samples.offset') INPUT_SIZE = config['train'].getint('fitting.inputsize') BATCH_SIZE = config['train'].getint('fitting.batchsize') EPOCHS = config['train'].getint('fitting.epochs') SAMPLES_PREDICT = config['train'].getint('samples.predict') ACCURACY_MIN = config['train'].getfloat('accuracy.min') # Measure run time timer = StopWatch() timer.start() def load(exchanger, unit, ty): return loadnpy(config, exchanger, unit, ty, nan=0.) def fit(X, y, model, rateTrain=0.9, epochs=1000, batchSize=64): Xu, yu = balance(X, y, thres=0.5) trainCount = int(Xu.shape[0] * rateTrain) indexRandom = np.random.permutation(Xu.shape[0]) indexTrain = indexRandom[:trainCount] XTrain = Xu[indexTrain] yTrain = yu[indexTrain] hist = model.fit(XTrain, yTrain, batch_size=batchSize, epochs=epochs, verbose=1, validation_split=0.1, shuffle=True) return hist
def test_sleep_1(): with StopWatch("time 'sleep(1)': ") as tmr: sleep(1) time = tmr.time assert time > 1.00 assert time < 1.01
def test_create_StopWatch(): tmr = StopWatch() assert tmr.time == 0.0
class LossMonitor: def __init__(self, do_print, do_plot, batch_size, data_collector, plot_every_batch=10, print_every_batch=100): self.do_print = do_print self.do_plot = do_plot self.batch_size = batch_size self.data_collector = data_collector self.plot_every = batch_size * plot_every_batch self.print_every = print_every_batch if self.do_plot: self.plotter = Plotter(2, 1) self.stop_watch = StopWatch() #self.stop_watch.start() #self.num_optimizer_updates = 0 self.plot_running_loss = 0.0 self.plot_losses, self.plot_avg_losses, self.plot_seen_samples = [],[],[] self.acc_loss, self.acc_points_reported = 0, 0 def epochStart(self, epoch): if not (self.do_print or self.do_plot): return if epoch != 0: self.data_collector.appendLosses(self.plot_losses, self.plot_seen_samples, 'running') self.acc_points_reported += len(self.plot_losses) self.acc_loss += np.sum(self.plot_losses) self.plot_losses = [] self.plot_avg_losses = [] self.plot_seen_samples = [] self.running_loss = 0.0 self.__print_thr, self.__plot_thr = self.print_every, self.plot_every def epochEnd(self, epoch, num_epochs): if self.do_print == False and epoch != 0 and epoch % 10 == 0: print('epoch #%d/%d \t %.2f, %.2f seconds elapsed' % (epoch, num_epochs, self.stop_watch.lap(), self.stop_watch.getElapsed())) def optimizerUpdate(self, loss, epoch, iteration, num_seen_samples): if not (self.do_print or self.do_plot): return self.running_loss += loss.data[0] self.plot_running_loss += loss.data[0] if self.do_print and iteration >= self.__print_thr: self.__print_thr += self.print_every print('[%d, %5d] loss: %.3f \t %.2f, %.2f seconds elapsed' % (epoch + 1, iteration * self.batch_size, self.running_loss / (self.batch_size * self.print_every), self.stop_watch.lap(), self.stop_watch.getElapsed())) self.running_loss = 0.0 if num_seen_samples >= self.__plot_thr: # plot every 100 seen samples self.__plot_thr += self.plot_every self.plot_losses.append(self.plot_running_loss / self.plot_every) self.plot_avg_losses.append( (np.sum(self.plot_losses) + self.acc_loss) / (len(self.plot_losses) + self.acc_points_reported)) self.plot_seen_samples.append(num_seen_samples) self.plot_running_loss = 0.0 if self.do_plot: self.plotter.update(self.plot_seen_samples, [self.plot_losses, self.plot_avg_losses], 0) self.plotter.update(self.plot_seen_samples[-50:], [self.plot_losses[-50:]], 1) def trainingDone(self): if self.do_print or self.do_plot: self.data_collector.appendLosses(self.plot_losses, self.plot_seen_samples, 'running')
def main(): iprint("hello") def foo(): iprint("inside foo") bar() def bar(): iprint("inside bar") foo() bar() a = [1, 2, 3, 4, 5, 6] iprint(a) b = [i for i in range(1000)] iprint(b) watch = StopWatch() log1 = NameLog() log2 = NameLog() log1.track("test1", "test2") log2.track("accuracy") test1 = 4 def baz(log1, log2): test2 = 42 log1.record() for accuracy in range(10): log2.record() baz(log1, log2) print(log1.tracked) print(log2.tracked) @CodeMemo def longjob(t, ret): print("Sleeping...") sleep(t) print("Done.") return ret def worker(): return longjob(5, ret=4) watch.start() res = worker() watch.stop() iprint(f"Got {res}") iprint(f"Elapsed {watch.elapsed()}") exit() tagger = Tagger() print("Basic") print(f"# possible tags: {tagger.size():,}") for i in range(5): print(tagger.make()) print() tagger = Tagger() seen = set() tag = tagger.make() while tag not in seen: seen.add(tag) tag = tagger.make() assert len(seen) == tagger.size() print("Space size matches.") print() tagger = Tagger(10) print("Numeric 10") print(f"# possible tags: {tagger.size():,}") for i in range(5): print(tagger.make()) print() tagger = Tagger("aaa") print("Letters 3") print(f"# possible tags: {tagger.size():,}") for i in range(5): print(tagger.make()) print()