def main():
# Measure run time
timer = StopWatch()
timer.start()
# Setup models
db = getDBInstance(config)
summaries = Summaries(db.tick_summary_db)
stepSeconds = {
'minutely': 60,
'hourly': 60 * 60,
'daily': 24 * 60 * 60,
'weekly': 7 * 24 * 60 * 60
}
maxCompletions = {
'minutely': None,
'hourly': MAX_COMPLETION_HOURLY,
'daily': MAX_COMPLETION_DAILY,
'weekly': None
}
noiseCycles = {
'minutely': None,
'hourly': COMPLETION_CYCLE_HOURLY,
'daily': None,
'weekly': None
}
now = datetime.datetime.now().timestamp()
after = now - EXPORT_WITHIN_SECONDS
for exchanger in EXCHANGERS:
for unit in UNITS:
sums = list(summaries.all(exchanger, unit, after=after))
logger.debug('Copying {e}\'s {u} ticks to np.array, #items={n}...'
.format(e=exchanger, u=unit, n=len(sums)))
step = stepSeconds[unit]
maxCompletion = maxCompletions[unit]
dates = seriesDate(sums, step)
logger.debug('Completing {e}\'s {u} ticks to np.array, maxN={maxN}...'
.format(e=exchanger, u=unit, maxN=maxCompletion))
opts = {
'maxN': maxCompletion
}
if noiseCycles[unit] is not None:
opts['noise'] = lambda n:completionNoise(n, cycle=noiseCycles[unit],
scale=COMPLETION_NOISE_SCALE)
completes = {
'askMax': completion(lambda s:s.askMax, sums, step, **opts),
'askMin': completion(lambda s:s.askMin, sums, step, **opts),
'askAverage': completion(lambda s:s.askAverage, sums, step, **opts),
'askOpen': completion(lambda s:s.askOpen, sums, step, **opts),
'askClose': completion(lambda s:s.askClose, sums, step, **opts)
}
for ty in completes:
completed = completes[ty]
if len(dates) != len(completed):
raise Exception('Length unmatch, #date={date}, #completed={completed}.'
.format(date=len(dates), completed=len(completed)))
savenpy(config, completed, exchanger, unit, ty)
savenpy(config, dates, exchanger, unit, 'date')
# Finished
seconds = timer.stop()
logger.debug('End export, elapsed={s:.2f}s'.format(s=seconds))
def main():
# Measure run time
timer = StopWatch()
timer.start()
# Execution
types = ['askAverage', 'askOpen', 'askClose']
runForAll(EXCHANGERS, UNITS, types)
# Finished
seconds = timer.stop()
logger.debug('End supervising, elapsed={s:.2f}s'.format(s=seconds))
def main():
# Measure run time
timer = StopWatch()
timer.start()
# Setup models
db = getDBInstance(config)
ticksModel = Ticks(db.tick_db)
dashb = Dashboard(uri=AIMAI_DB_URI, logger=logger)
dashb.requestLogin(USERNAME, PASSWORD)
sync(dashb, ticksModel)
# Finished
seconds = timer.stop()
logger.debug('End synchronization, elapsed={s:.2f}s'.format(s=seconds))
def test_sleep_1_iterated():
with StopWatch("time 5 times 'sleep(1)': ", ndigits=3) as tmr:
for i in range(5):
sleep(1)
time = tmr.timelapse()
print(i, time)
assert time >= 1.00
assert time <= 1.01
time = tmr.time
assert time > 5.00
assert time < 5.05
def __init__(self,
do_print,
do_plot,
batch_size,
data_collector,
plot_every_batch=10,
print_every_batch=100):
self.do_print = do_print
self.do_plot = do_plot
self.batch_size = batch_size
self.data_collector = data_collector
self.plot_every = batch_size * plot_every_batch
self.print_every = print_every_batch
if self.do_plot:
self.plotter = Plotter(2, 1)
self.stop_watch = StopWatch()
#self.stop_watch.start()
#self.num_optimizer_updates = 0
self.plot_running_loss = 0.0
self.plot_losses, self.plot_avg_losses, self.plot_seen_samples = [],[],[]
self.acc_loss, self.acc_points_reported = 0, 0
def main():
# Measure run time
timer = StopWatch()
timer.start()
# Setup models
db = getDBInstance(config)
ticks = Ticks(db.tick_db)
summaries = Summaries(db.tick_summary_db)
for exchanger in EXCHANGERS:
for unit in UNITS:
logger.debug('Summarizing, exchanger={exchanger}, unit={unit}.'
.format(exchanger=exchanger, unit=unit))
latest = summaries.one(exchanger, unit)
if latest is not None:
start = latest.date
else:
start = None
sums = ticksBy(ticks, exchanger, unit, dateStart=start)
logger.info('Saving summary...')
summaries.saveAll(exchanger, unit, sums)
# Finished
seconds = timer.stop()
logger.debug('End summarization, elapsed={s:.2f}s'.format(s=seconds))
def prepare_data_and_vocab(cfg):
load_data_file = ["books_large_p1.txt", "books_large_p2.txt"]
#load_data_file = "books_100k.txt"
save_data_path = os.path.join(cfg.prepro_dir, "data")
save_vocab_path = os.path.join(cfg.prepro_dir, "vocab")
StopWatch.go('Total')
if (not os.path.exists(save_data_path)
or not os.path.exists(save_vocab_path) or cfg.is_reload_prepro):
print('Start preprocessing data and building vocabulary!')
if isinstance(load_data_file, (list, tuple)):
load_data_path = [
*map(lambda fn: os.path.join(cfg.data_dir, fn), load_data_file)
]
book_procs = BookCorpusMultiProcessor.from_multiple_files(
file_paths=load_data_path,
min_len=cfg.prepro_min_len,
max_len=cfg.prepro_max_len)
sents, counter = BookCorpusMultiProcessor.multi_process(book_procs)
else:
load_data_path = os.path.join(cfg.data_dir, load_data_file)
book_procs = BookCorpusMultiProcessor(file_path=load_data_path,
min_len=cfg.prepro_min_len,
max_len=cfg.prepro_max_len)
sents, counter = book_procs.process()
vocab = Vocab(counter=counter,
max_size=cfg.vocab_size,
embed_dim=cfg.embed_dim,
is_load_glove=cfg.is_load_glove)
sents = vocab.numericalize_sents(sents)
if not os.path.exists(cfg.prepro_dir):
os.makedirs(cfg.prepro_dir)
with StopWatch('Saving text'):
np.savetxt(save_data_path, sents, fmt="%s")
with StopWatch('Pickling vocab'):
vocab.pickle(save_vocab_path)
else:
print('Previously processed files will be used!')
vocab = Vocab.unpickle(save_vocab_path)
StopWatch.stop('Total')
return vocab
from learningUtils import validated, to2d, zscore, round_binary_accuracy, underSampling, balance, saveModel
from utils import readConfig, getLogger, loadnpy, StopWatch
logger = getLogger()
config = readConfig('predict.ini')
logger.info('Training started.')
OFFSET_SAMPLES = config['train'].getint('samples.offset')
INPUT_SIZE = config['train'].getint('fitting.inputsize')
BATCH_SIZE = config['train'].getint('fitting.batchsize')
EPOCHS = config['train'].getint('fitting.epochs')
SAMPLES_PREDICT = config['train'].getint('samples.predict')
ACCURACY_MIN = config['train'].getfloat('accuracy.min')
# Measure run time
timer = StopWatch()
timer.start()
def load(exchanger, unit, ty):
return loadnpy(config, exchanger, unit, ty, nan=0.)
def fit(X, y, model, rateTrain=0.9, epochs=1000, batchSize=64):
Xu, yu = balance(X, y, thres=0.5)
trainCount = int(Xu.shape[0] * rateTrain)
indexRandom = np.random.permutation(Xu.shape[0])
indexTrain = indexRandom[:trainCount]
XTrain = Xu[indexTrain]
yTrain = yu[indexTrain]
hist = model.fit(XTrain, yTrain, batch_size=batchSize, epochs=epochs,
verbose=1, validation_split=0.1, shuffle=True)
return hist
def test_sleep_1():
with StopWatch("time 'sleep(1)': ") as tmr:
sleep(1)
time = tmr.time
assert time > 1.00
assert time < 1.01
def test_create_StopWatch():
tmr = StopWatch()
assert tmr.time == 0.0
class LossMonitor:
def __init__(self,
do_print,
do_plot,
batch_size,
data_collector,
plot_every_batch=10,
print_every_batch=100):
self.do_print = do_print
self.do_plot = do_plot
self.batch_size = batch_size
self.data_collector = data_collector
self.plot_every = batch_size * plot_every_batch
self.print_every = print_every_batch
if self.do_plot:
self.plotter = Plotter(2, 1)
self.stop_watch = StopWatch()
#self.stop_watch.start()
#self.num_optimizer_updates = 0
self.plot_running_loss = 0.0
self.plot_losses, self.plot_avg_losses, self.plot_seen_samples = [],[],[]
self.acc_loss, self.acc_points_reported = 0, 0
def epochStart(self, epoch):
if not (self.do_print or self.do_plot):
return
if epoch != 0:
self.data_collector.appendLosses(self.plot_losses,
self.plot_seen_samples, 'running')
self.acc_points_reported += len(self.plot_losses)
self.acc_loss += np.sum(self.plot_losses)
self.plot_losses = []
self.plot_avg_losses = []
self.plot_seen_samples = []
self.running_loss = 0.0
self.__print_thr, self.__plot_thr = self.print_every, self.plot_every
def epochEnd(self, epoch, num_epochs):
if self.do_print == False and epoch != 0 and epoch % 10 == 0:
print('epoch #%d/%d \t %.2f, %.2f seconds elapsed' %
(epoch, num_epochs, self.stop_watch.lap(),
self.stop_watch.getElapsed()))
def optimizerUpdate(self, loss, epoch, iteration, num_seen_samples):
if not (self.do_print or self.do_plot):
return
self.running_loss += loss.data[0]
self.plot_running_loss += loss.data[0]
if self.do_print and iteration >= self.__print_thr:
self.__print_thr += self.print_every
print('[%d, %5d] loss: %.3f \t %.2f, %.2f seconds elapsed' %
(epoch + 1, iteration * self.batch_size, self.running_loss /
(self.batch_size * self.print_every), self.stop_watch.lap(),
self.stop_watch.getElapsed()))
self.running_loss = 0.0
if num_seen_samples >= self.__plot_thr: # plot every 100 seen samples
self.__plot_thr += self.plot_every
self.plot_losses.append(self.plot_running_loss / self.plot_every)
self.plot_avg_losses.append(
(np.sum(self.plot_losses) + self.acc_loss) /
(len(self.plot_losses) + self.acc_points_reported))
self.plot_seen_samples.append(num_seen_samples)
self.plot_running_loss = 0.0
if self.do_plot:
self.plotter.update(self.plot_seen_samples,
[self.plot_losses, self.plot_avg_losses],
0)
self.plotter.update(self.plot_seen_samples[-50:],
[self.plot_losses[-50:]], 1)
def trainingDone(self):
if self.do_print or self.do_plot:
self.data_collector.appendLosses(self.plot_losses,
self.plot_seen_samples, 'running')
def main():
iprint("hello")
def foo():
iprint("inside foo")
bar()
def bar():
iprint("inside bar")
foo()
bar()
a = [1, 2, 3, 4, 5, 6]
iprint(a)
b = [i for i in range(1000)]
iprint(b)
watch = StopWatch()
log1 = NameLog()
log2 = NameLog()
log1.track("test1", "test2")
log2.track("accuracy")
test1 = 4
def baz(log1, log2):
test2 = 42
log1.record()
for accuracy in range(10):
log2.record()
baz(log1, log2)
print(log1.tracked)
print(log2.tracked)
@CodeMemo
def longjob(t, ret):
print("Sleeping...")
sleep(t)
print("Done.")
return ret
def worker():
return longjob(5, ret=4)
watch.start()
res = worker()
watch.stop()
iprint(f"Got {res}")
iprint(f"Elapsed {watch.elapsed()}")
exit()
tagger = Tagger()
print("Basic")
print(f"# possible tags: {tagger.size():,}")
for i in range(5):
print(tagger.make())
print()
tagger = Tagger()
seen = set()
tag = tagger.make()
while tag not in seen:
seen.add(tag)
tag = tagger.make()
assert len(seen) == tagger.size()
print("Space size matches.")
print()
tagger = Tagger(10)
print("Numeric 10")
print(f"# possible tags: {tagger.size():,}")
for i in range(5):
print(tagger.make())
print()
tagger = Tagger("aaa")
print("Letters 3")
print(f"# possible tags: {tagger.size():,}")
for i in range(5):
print(tagger.make())
print()