def main():
# Measure run time
timer = StopWatch()
timer.start()
# Setup models
db = getDBInstance(config)
summaries = Summaries(db.tick_summary_db)
stepSeconds = {
'minutely': 60,
'hourly': 60 * 60,
'daily': 24 * 60 * 60,
'weekly': 7 * 24 * 60 * 60
}
maxCompletions = {
'minutely': None,
'hourly': MAX_COMPLETION_HOURLY,
'daily': MAX_COMPLETION_DAILY,
'weekly': None
}
noiseCycles = {
'minutely': None,
'hourly': COMPLETION_CYCLE_HOURLY,
'daily': None,
'weekly': None
}
now = datetime.datetime.now().timestamp()
after = now - EXPORT_WITHIN_SECONDS
for exchanger in EXCHANGERS:
for unit in UNITS:
sums = list(summaries.all(exchanger, unit, after=after))
logger.debug('Copying {e}\'s {u} ticks to np.array, #items={n}...'
.format(e=exchanger, u=unit, n=len(sums)))
step = stepSeconds[unit]
maxCompletion = maxCompletions[unit]
dates = seriesDate(sums, step)
logger.debug('Completing {e}\'s {u} ticks to np.array, maxN={maxN}...'
.format(e=exchanger, u=unit, maxN=maxCompletion))
opts = {
'maxN': maxCompletion
}
if noiseCycles[unit] is not None:
opts['noise'] = lambda n:completionNoise(n, cycle=noiseCycles[unit],
scale=COMPLETION_NOISE_SCALE)
completes = {
'askMax': completion(lambda s:s.askMax, sums, step, **opts),
'askMin': completion(lambda s:s.askMin, sums, step, **opts),
'askAverage': completion(lambda s:s.askAverage, sums, step, **opts),
'askOpen': completion(lambda s:s.askOpen, sums, step, **opts),
'askClose': completion(lambda s:s.askClose, sums, step, **opts)
}
for ty in completes:
completed = completes[ty]
if len(dates) != len(completed):
raise Exception('Length unmatch, #date={date}, #completed={completed}.'
.format(date=len(dates), completed=len(completed)))
savenpy(config, completed, exchanger, unit, ty)
savenpy(config, dates, exchanger, unit, 'date')
# Finished
seconds = timer.stop()
logger.debug('End export, elapsed={s:.2f}s'.format(s=seconds))
def prepare_data_and_vocab(cfg):
load_data_file = ["books_large_p1.txt", "books_large_p2.txt"]
#load_data_file = "books_100k.txt"
save_data_path = os.path.join(cfg.prepro_dir, "data")
save_vocab_path = os.path.join(cfg.prepro_dir, "vocab")
StopWatch.go('Total')
if (not os.path.exists(save_data_path)
or not os.path.exists(save_vocab_path) or cfg.is_reload_prepro):
print('Start preprocessing data and building vocabulary!')
if isinstance(load_data_file, (list, tuple)):
load_data_path = [
*map(lambda fn: os.path.join(cfg.data_dir, fn), load_data_file)
]
book_procs = BookCorpusMultiProcessor.from_multiple_files(
file_paths=load_data_path,
min_len=cfg.prepro_min_len,
max_len=cfg.prepro_max_len)
sents, counter = BookCorpusMultiProcessor.multi_process(book_procs)
else:
load_data_path = os.path.join(cfg.data_dir, load_data_file)
book_procs = BookCorpusMultiProcessor(file_path=load_data_path,
min_len=cfg.prepro_min_len,
max_len=cfg.prepro_max_len)
sents, counter = book_procs.process()
vocab = Vocab(counter=counter,
max_size=cfg.vocab_size,
embed_dim=cfg.embed_dim,
is_load_glove=cfg.is_load_glove)
sents = vocab.numericalize_sents(sents)
if not os.path.exists(cfg.prepro_dir):
os.makedirs(cfg.prepro_dir)
with StopWatch('Saving text'):
np.savetxt(save_data_path, sents, fmt="%s")
with StopWatch('Pickling vocab'):
vocab.pickle(save_vocab_path)
else:
print('Previously processed files will be used!')
vocab = Vocab.unpickle(save_vocab_path)
StopWatch.stop('Total')
return vocab
def main():
# Measure run time
timer = StopWatch()
timer.start()
# Execution
types = ['askAverage', 'askOpen', 'askClose']
runForAll(EXCHANGERS, UNITS, types)
# Finished
seconds = timer.stop()
logger.debug('End supervising, elapsed={s:.2f}s'.format(s=seconds))
def main():
# Measure run time
timer = StopWatch()
timer.start()
# Setup models
db = getDBInstance(config)
ticksModel = Ticks(db.tick_db)
dashb = Dashboard(uri=AIMAI_DB_URI, logger=logger)
dashb.requestLogin(USERNAME, PASSWORD)
sync(dashb, ticksModel)
# Finished
seconds = timer.stop()
logger.debug('End synchronization, elapsed={s:.2f}s'.format(s=seconds))
def main():
# Measure run time
timer = StopWatch()
timer.start()
# Setup models
db = getDBInstance(config)
ticks = Ticks(db.tick_db)
summaries = Summaries(db.tick_summary_db)
for exchanger in EXCHANGERS:
for unit in UNITS:
logger.debug('Summarizing, exchanger={exchanger}, unit={unit}.'
.format(exchanger=exchanger, unit=unit))
latest = summaries.one(exchanger, unit)
if latest is not None:
start = latest.date
else:
start = None
sums = ticksBy(ticks, exchanger, unit, dateStart=start)
logger.info('Saving summary...')
summaries.saveAll(exchanger, unit, sums)
# Finished
seconds = timer.stop()
logger.debug('End summarization, elapsed={s:.2f}s'.format(s=seconds))
yModel.compile(optimizer='adam', loss='binary_crossentropy',
metrics=[round_binary_accuracy])
yHist = fit(XbhTrain, ybh0Train, yModel, epochs=EPOCHS, batchSize=BATCH_SIZE)
yTrainEnd = datetime.datetime.now()
yAcc = yHist.history['val_round_binary_accuracy'][-1]
plotHistory('../figures/yhist.svg', yHist, keyAccuracy='round_binary_accuracy')
logger.warn(('Training done, trendAcc.={trend:.2f}, ' +
'#samples={ns:.0f}, #epochs={ne:.0f}, #batchSize={nb:.0f}, ' +
'trendTime={tt:.1f}.')
.format(trend=yAcc * 100,
ns=trainSize, ne=EPOCHS, nb=BATCH_SIZE,
tt=(yTrainEnd - yTrainStart).total_seconds()))
logger.info('Training accuracies are: trend={trend:.2f}.'
.format(trend=yAcc * 100))
if yAcc < ACCURACY_MIN:
logger.error('Aborted because of insufficient accuracy.')
exit(1)
saveModel(config, yModel, 'trend')
logger.debug('Predicting, Xbh.shape={x}.'.format(x=Xbh.shape))
logger.info('#Ticks={tick}, #Train={train}, #Predicted={predict}.'
.format(tick=Xbh.shape[0], train=trainSize,
predict=Xbh.shape[0] - trainSize))
# Finished
seconds = timer.stop()
logger.info('Training completed, elapsed={s:.2f}s'.format(s=seconds))
def main():
iprint("hello")
def foo():
iprint("inside foo")
bar()
def bar():
iprint("inside bar")
foo()
bar()
a = [1, 2, 3, 4, 5, 6]
iprint(a)
b = [i for i in range(1000)]
iprint(b)
watch = StopWatch()
log1 = NameLog()
log2 = NameLog()
log1.track("test1", "test2")
log2.track("accuracy")
test1 = 4
def baz(log1, log2):
test2 = 42
log1.record()
for accuracy in range(10):
log2.record()
baz(log1, log2)
print(log1.tracked)
print(log2.tracked)
@CodeMemo
def longjob(t, ret):
print("Sleeping...")
sleep(t)
print("Done.")
return ret
def worker():
return longjob(5, ret=4)
watch.start()
res = worker()
watch.stop()
iprint(f"Got {res}")
iprint(f"Elapsed {watch.elapsed()}")
exit()
tagger = Tagger()
print("Basic")
print(f"# possible tags: {tagger.size():,}")
for i in range(5):
print(tagger.make())
print()
tagger = Tagger()
seen = set()
tag = tagger.make()
while tag not in seen:
seen.add(tag)
tag = tagger.make()
assert len(seen) == tagger.size()
print("Space size matches.")
print()
tagger = Tagger(10)
print("Numeric 10")
print(f"# possible tags: {tagger.size():,}")
for i in range(5):
print(tagger.make())
print()
tagger = Tagger("aaa")
print("Letters 3")
print(f"# possible tags: {tagger.size():,}")
for i in range(5):
print(tagger.make())
print()