def main():
coronaTexts = CoronaTexts()
coronaTexts.constructCountries() # 6 will be quit
currTime = datetime.now()
Session = Messaging.Session()
Session.login()
updateTimer = helpers.Timer(
helpers.minutesToSeconds(numMinsBetweenUpdates))
cPrintPollingAlert()
while True:
try:
if hasTimeElapsed(updateTimer):
# refresh all
countryPopulationScraper.updatePops(coronaTexts)
print("\t{}\t|\t{}\t".format("Name", "Population"))
for country in coronaTexts.countries:
print("\t{}\t|\t{}\t".format(country.getCountryName(),
int(country.getPopulation())))
coronaTexts.updateAllCountries()
except KeyboardInterrupt:
printMenu()
userInput = getUsrMenuInput()
if userInput == 0:
coronaTexts.addEmailAddress()
if userInput == 1:
coronaTexts.showAllEmailAddresses()
cPrintPollingAlert()
def test(opt):
log = helpers.Logger(opt.verbose)
timer = helpers.Timer()
# Load data =========================================================
log.info('Reading corpora')
# Read vocabs
widss, ids2ws, widst, ids2wt = helpers.get_dictionaries(opt, test=True)
# Read test
tests_data = np.asarray(data.read_corpus(opt.test_src, widss), dtype=list)
# Test output
if not opt.test_out:
opt.test_out = helpers.exp_filename(opt, 'test.out')
# Get target language model
lang_model = helpers.get_language_model(opt, None, widst, test=True)
# Create model ======================================================
log.info('Creating model')
s2s = helpers.build_model(opt, widss, widst, lang_model, test=True)
# Print configuration ===============================================
if opt.verbose:
options.print_config(opt,
src_dict_size=len(widss),
trg_dict_size=len(widst))
# Start testing =====================================================
log.info('Start running on test set, buckle up!')
timer.restart()
translations = []
s2s.set_test_mode()
for i, x in enumerate(tests_data):
y = s2s.translate(x, beam_size=opt.beam_size)
translations.append(' '.join([ids2wt[w] for w in y[1:-1]]))
np.savetxt(opt.test_out, translations, fmt='%s')
translations = np.asarray(translations, dtype=str)
BLEU, details = evaluation.bleu_score(opt.test_dst, opt.test_out)
log.info('Finished running on test set %.2f elapsed.' % timer.tick())
log.info(details)
def __build_result_assign_step(hs, res, cx2_kpts, cx2_desc, cx2_rchip_size, assign_matches, verbose):
'1) Assign matches with the chosen function (vsone) or (vsmany)'
if verbose:
#helpers.printvar(locals(), 'cx2_desc')
#helpers.printvar(locals(), 'res.qcx')
num_qdesc = len(cx2_desc[res.qcx])
print('[mc2] assign %d desc' % (num_qdesc))
tt1 = helpers.Timer(verbose=False)
assign_output = assign_matches(res.qcx, cx2_kpts, cx2_desc, cx2_rchip_size)
(cx2_fm, cx2_fs, cx2_score) = assign_output
# Record initial assignments
res.assign_time = tt1.toc()
res.cx2_fm = np.array(cx2_fm)
res.cx2_fs = np.array(cx2_fs)
res.cx2_score = cx2_score
def parallelize_tasks(task_list, num_procs, task_lbl='', verbose=True):
'''
Used for embarissingly parallel tasks, which write output to disk
'''
nTasks = len(task_list)
msg = ('Distributing %d %s tasks to %d processes' %
(nTasks, task_lbl, num_procs) if num_procs > 1 else
'Executing %d %s tasks in serial' % (nTasks, task_lbl))
with helpers.Timer(msg=msg):
if num_procs > 1:
# Parallelize tasks
return _compute_in_parallel(task_list, num_procs, task_lbl,
verbose)
else:
return _compute_in_serial(task_list, task_lbl, verbose)
def main(source: str, remove: bool = False) -> None:
"""
Executes an NRN stage.
:param str source: abbreviation for the source province / territory.
:param bool remove: removes pre-existing validation log within the data/processed directory for the specified
source, default False.
"""
try:
with helpers.Timer():
stage = Stage(source, remove)
stage.execute()
except KeyboardInterrupt:
logger.exception("KeyboardInterrupt: Exiting program.")
sys.exit(1)
def __build_result_verify_step(hs, res, cx2_kpts, cx2_rchip_size, verbose):
' 2) Spatially verify the assigned matches'
cx2_fm = res.cx2_fm
cx2_fs = res.cx2_fs
cx2_score = res.cx2_score
if verbose:
num_assigned = np.array([len(fm) for fm in cx2_fm]).sum()
print('[mc2] verify %d assigned matches' % (num_assigned))
tt2 = helpers.Timer(verbose=False)
sv_output = spatially_verify_matches(res.qcx, cx2_kpts, cx2_rchip_size, cx2_fm, cx2_fs, cx2_score)
(cx2_fm_V, cx2_fs_V, cx2_score_V) = sv_output
# Record verified assignments
res.verify_time = tt2.toc()
res.cx2_fm_V = np.array(cx2_fm_V)
res.cx2_fs_V = np.array(cx2_fs_V)
res.cx2_score_V = cx2_score_V
if verbose:
num_verified = np.array([len(fm) for fm in cx2_fm_V]).sum()
print('[mc2] verified %d matches' % (num_verified))
def main(source: str, remove: bool = False, exclude_old: bool = False) -> None:
"""
Executes an NRN stage.
:param str source: abbreviation for the source province / territory.
:param bool remove: removes pre-existing files within the data/interim directory for the specified source, default
False.
:param bool exclude_old: excludes the previous NRN vintage for the specified source from being removed if
remove=True, default False. Option has no effect if remove=False.
"""
try:
with helpers.Timer():
stage = Stage(source, remove, exclude_old)
stage.execute()
except KeyboardInterrupt:
logger.exception("KeyboardInterrupt: Exiting program.")
sys.exit(1)
def test(options):
log = helpers.Logger(options.verbose)
timer = helpers.Timer()
# Load data =========================================================
log.info('Reading corpora')
# Read vocabs
vocab = helpers.get_dictionaries(options, test=True)
src_dic, trg_dic = vocab['src'], vocab['trg']
# Read test
tests_data = src_dic.read_corpus(options.test_src)
# Test output
if not options.test_out:
options.test_out = helpers.exp_filename(options, 'test.out')
# Get target language model
lang_model = helpers.get_language_model(options,
None,
trg_dic.size(),
test=True)
# Create model ======================================================
log.info('Creating model')
s2s = helpers.build_model(options, vocab, lang_model, test=True)
# Print configuration ===============================================
if options.verbose:
Opts.print_config(options,
src_dict_size=src_dic.size(),
trg_dict_size=trg_dic.size())
# Start testing =====================================================
log.info('Start running on test set, buckle up!')
timer.restart()
translations = []
s2s.set_test_flag()
for i, x in enumerate(tests_data):
y = s2s.translate(x, beam_size=options.beam_size)
translations.append(' '.join([trg_dic.get_word(w) for w in y[1:-1]]))
translations = np.asarray(translations, dtype=str)
np.savetxt(options.test_out, translations, fmt='%s')
if options.test_dst is not None:
BLEU, details = evaluation.bleu_score(options.test_dst,
options.test_out)
log.info(details)
log.info('Finished running on test set %.2f elapsed.' % timer.tick())
def parallelize_tasks(task_list, num_procs, task_lbl, verbose=True):
'''
Used for embarissingly parallel tasks, which write output to disk
'''
nTasks = len(task_list)
msg = ('Distributing %d %s tasks to %d processes' %
(nTasks, task_lbl, num_procs) if num_procs > 1 else
'Executing %d %s tasks in serial' % (nTasks, task_lbl))
with helpers.Timer(msg=msg):
if num_procs >= 1:
print('[parallel] Forced serial processing')
# Parallelize tasks
return _compute_in_serial(
task_list, task_lbl,
verbose) #trying to force serial computation -MD
#return _compute_in_parallel(task_list, num_procs, task_lbl, verbose)
#return _compute_in_parallel(task_list, num_procs, task_lbl, verbose)
''' Hacky patch to hopefully avoid segfaults '''
return _compute_in_serial(task_list, task_lbl, verbose)
else:
print('[parallel] Natural serial processing')
return _compute_in_serial(task_list, task_lbl, verbose)
def train(options):
log = helpers.Logger(options.verbose)
timer = helpers.Timer()
# Load data =========================================================
log.info('Reading corpora')
# Read vocabs
vocab = helpers.get_dictionaries(options)
src_dic, trg_dic = vocab['src'], vocab['trg']
# Read training
train_src_data = src_dic.read_corpus(options.train_src)
train_trg_data = trg_dic.read_corpus(options.train_dst)
max_src_len, max_trg_len = options.max_src_len, options.max_trg_len
if max_src_len > 0 or max_trg_len > 0:
train_src_data, train_trg_data = corpus_filter(train_src_data,
train_trg_data,
max_src_len,
max_trg_len)
assert len(train_src_data) == len(
train_trg_data
), 'Size of source corpus and the target corpus must be the same!!'
# Read validation
valid_src_data = src_dic.read_corpus(options.valid_src)
valid_trg_data = trg_dic.read_corpus(options.valid_dst)
# Validation output
if not options.valid_out:
options.valid_out = helpers.exp_filename(options, 'valid.out')
# Get target language model
lang_model = helpers.get_language_model(options, train_trg_data,
trg_dic.size())
# Create model ======================================================
log.info('Creating model')
s2s = helpers.build_model(options, vocab, lang_model)
# Trainer ==========================================================
trainer = helpers.get_trainer(options, s2s)
log.info('Using ' + options.trainer + ' optimizer')
# Print configuration ===============================================
if options.verbose:
Opts.print_config(options,
src_dict_size=src_dic.size(),
trg_dict_size=trg_dic.size())
# Creat batch loaders ===============================================
log.info('Creating batch loaders')
trainbatchloader = BatchLoader(train_src_data, train_trg_data,
options.batch_size)
devbatchloader = BatchLoader(valid_src_data, valid_trg_data,
options.dev_batch_size)
# Start training ====================================================
log.info('starting training')
timer.restart()
train_loss = 0.
processed = 0
best_bleu = -1
bleu = -1
deadline = 0
i = 0
for epoch in xrange(options.num_epochs):
for x, y in trainbatchloader:
s2s.set_train_flag()
processed += sum(map(len, y))
bsize = len(y)
# Compute loss
loss = s2s.calculate_loss(x, y)
# Backward pass and parameter update
train_loss += loss.scalar_value() * bsize
loss.backward()
trainer.update()
if (i + 1) % options.check_train_error_every == 0:
# Check average training error from time to time
logloss = train_loss / processed
ppl = np.exp(logloss)
trainer.status()
log.info(
" Training_loss=%f, ppl=%f, time=%f s, tokens processed=%d"
% (logloss, ppl, timer.tick(), processed))
train_loss = 0
processed = 0
if (i + 1) % options.check_valid_error_every == 0:
# Check generalization error on the validation set from time to time
s2s.set_test_flag()
dev_loss = 0
dev_processed = 0
timer.restart()
for x, y in devbatchloader:
dev_processed += sum(map(len, y))
bsize = len(y)
loss = s2s.calculate_loss(x, y)
dev_loss += loss.scalar_value() * bsize
dev_logloss = dev_loss / dev_processed
dev_ppl = np.exp(dev_logloss)
log.info(
"[epoch %d] Dev loss=%f, ppl=%f, time=%f s, tokens processed=%d"
%
(epoch, dev_logloss, dev_ppl, timer.tick(), dev_processed))
if (i + 1) % options.valid_bleu_every == 0:
# Check BLEU score on the validation set from time to time
s2s.set_test_flag()
log.info('Start translating validation set, buckle up!')
timer.restart()
with open(options.valid_out, 'w+') as fp:
for x in valid_src_data:
y_hat = s2s.translate(x, beam_size=options.beam_size)
translation = [
trg_dic.get_word(w) for w in y_hat[1:-1]
]
fp.write(' '.join(translation))
fp.write('\n')
bleu, details = evaluation.bleu_score(options.valid_dst,
options.valid_out)
log.info('Finished translating validation set %.2f elapsed.' %
timer.tick())
log.info(details)
# Early stopping : save the latest best model
if bleu > best_bleu:
best_bleu = bleu
log.info('Best BLEU score up to date, saving model to %s' %
options.model)
s2s.save(options.model)
deadline = 0
else:
deadline += 1
if options.patience > 0 and deadline > options.patience:
log.info('No improvement since %d epochs, early stopping '
'with best validation BLEU score: %.3f' %
(deadline, best_bleu))
sys.exit()
# i += 1
# trainer.update()
#if bleu > best_bleu:
# s2s.save(options.model)
s2s.save(options.model)
@autojit
def sum2d5(arr):
M, N = arr.shape
result = 0.0
result = arr.sum()
return result
if __name__ == '__main__':
input_sizes = [1, 10, 100, 1000, 10000]
for sz in input_sizes:
print('\n--------------------\nInput Size: ' + str(sz))
arr = np.random.rand(sz, sz)
with helpers.Timer('with numba'):
res = sum2d(arr)
print res
with helpers.Timer('with numpy'):
res2 = sum2d4(arr)
print res2
with helpers.Timer('without numpy and numba'):
res2 = sum2d5(arr)
print res2
'''
with helpers.Timer('without numba (but smart)'):
res3 = sum2d3(arr)
print res3
with helpers.Timer('without numba (naive)'):
res2 = sum2d2(arr)
print res2
def test_realdata2():
from helpers import printWARN, printINFO
import warnings
import numpy.linalg as linalg
import numpy as np
import scipy.sparse as sparse
import scipy.sparse.linalg as sparse_linalg
import load_data2
import params
import draw_func2 as df2
import helpers
import spatial_verification
#params.reload_module()
#load_data2.reload_module()
#df2.reload_module()
db_dir = load_data2.MOTHERS
hs = load_data2.HotSpotter(db_dir)
assign_matches = hs.matcher.assign_matches
qcx = 0
cx = hs.get_other_cxs(qcx)[0]
fm, fs, score = hs.get_assigned_matches_to(qcx, cx)
# Get chips
rchip1 = hs.get_chip(qcx)
rchip2 = hs.get_chip(cx)
# Get keypoints
kpts1 = hs.get_kpts(qcx)
kpts2 = hs.get_kpts(cx)
# Get feature matches
kpts1_m = kpts1[fm[:, 0], :].T
kpts2_m = kpts2[fm[:, 1], :].T
title='(qx%r v cx%r)\n #match=%r' % (qcx, cx, len(fm))
df2.show_matches2(rchip1, rchip2, kpts1, kpts2, fm, fs, title=title)
np.random.seed(6)
subst = helpers.random_indexes(len(fm),len(fm))
kpts1_m = kpts1[fm[subst, 0], :].T
kpts2_m = kpts2[fm[subst, 1], :].T
df2.reload_module()
df2.SHOW_LINES = True
df2.ELL_LINEWIDTH = 2
df2.LINE_ALPHA = .5
df2.ELL_ALPHA = 1
df2.reset()
df2.show_keypoints(rchip1, kpts1_m.T, fignum=0, plotnum=121)
df2.show_keypoints(rchip2, kpts2_m.T, fignum=0, plotnum=122)
df2.show_matches2(rchip1, rchip2, kpts1_m.T, kpts2_m.T, title=title,
fignum=1, vert=True)
spatial_verification.reload_module()
with helpers.Timer():
aff_inliers1 = spatial_verification.aff_inliers_from_ellshape2(kpts1_m, kpts2_m, xy_thresh_sqrd)
with helpers.Timer():
aff_inliers2 = spatial_verification.aff_inliers_from_ellshape(kpts1_m, kpts2_m, xy_thresh_sqrd)
# Homogonize+Normalize
xy1_m = kpts1_m[0:2,:]
xy2_m = kpts2_m[0:2,:]
(xyz_norm1, T1) = spatial_verification.homogo_normalize_pts(xy1_m[:,aff_inliers1])
(xyz_norm2, T2) = spatial_verification.homogo_normalize_pts(xy2_m[:,aff_inliers1])
H_prime = spatial_verification.compute_homog(xyz_norm1, xyz_norm2)
H = linalg.solve(T2, H_prime).dot(T1) # Unnormalize
Hdet = linalg.det(H)
# Estimate final inliers
acd1_m = kpts1_m[2:5,:] # keypoint shape matrix [a 0; c d] matches
acd2_m = kpts2_m[2:5,:]
# Precompute the determinant of lower triangular matrix (a*d - b*c); b = 0
det1_m = acd1_m[0] * acd1_m[2]
det2_m = acd2_m[0] * acd2_m[2]
# Matrix Multiply xyacd matrix by H
# [[A, B, X],
# [C, D, Y],
# [E, F, Z]]
# dot
# [(a, 0, x),
# (c, d, y),
# (0, 0, 1)]
# =
# [(a*A + c*B + 0*E, 0*A + d*B + 0*X, x*A + y*B + 1*X),
# (a*C + c*D + 0*Y, 0*C + d*D + 0*Y, x*C + y*D + 1*Y),
# (a*E + c*F + 0*Z, 0*E + d*F + 0*Z, x*E + y*F + 1*Z)]
# =
# [(a*A + c*B, d*B, x*A + y*B + X),
# (a*C + c*D, d*D, x*C + y*D + Y),
# (a*E + c*F, d*F, x*E + y*F + Z)]
# # IF x=0 and y=0
# =
# [(a*A + c*B, d*B, 0*A + 0*B + X),
# (a*C + c*D, d*D, 0*C + 0*D + Y),
# (a*E + c*F, d*F, 0*E + 0*F + Z)]
# =
# [(a*A + c*B, d*B, X),
# (a*C + c*D, d*D, Y),
# (a*E + c*F, d*F, Z)]
# ---
# A11 = a*A + c*B
# A21 = a*C + c*D
# A31 = a*E + c*F
# A12 = d*B
# A22 = d*D
# A32 = d*F
# A31 = X
# A32 = Y
# A33 = Z
#
# det(A) = A11*(A22*A33 - A23*A32) - A12*(A21*A33 - A23*A31) + A13*(A21*A32 - A22*A31)
det1_mAt = det1_m * Hdet
# Check Error in position and scale
xy_sqrd_err = (x1_mAt - x2_m)**2 + (y1_mAt - y2_m)**2
scale_sqrd_err = det1_mAt / det2_m
# Check to see if outliers are within bounds
xy_inliers = xy_sqrd_err < xy_thresh_sqrd
s1_inliers = scale_sqrd_err > scale_thresh_low
s2_inliers = scale_sqrd_err < scale_thresh_high
_inliers, = np.where(np.logical_and(np.logical_and(xy_inliers, s1_inliers), s2_inliers))
xy1_mHt = transform_xy(H, xy1_m) # Transform Kpts1 to Kpts2-space
sqrd_dist_error = np.sum( (xy1_mHt - xy2_m)**2, axis=0) # Final Inlier Errors
inliers = sqrd_dist_error < xy_thresh_sqrd
df2.show_matches2(rchip1, rchip2, kpts1_m.T[best_inliers1], kpts2_m.T[aff_inliers1], title=title, fignum=2, vert=False)
df2.show_matches2(rchip1, rchip2, kpts1_m.T[best_inliers2], kpts2_m.T[aff_inliers2], title=title, fignum=3, vert=False)
df2.present(wh=(600,400))
#### Modify the below to get different distributions!
from comp_implt import *
import helpers
import time
s1_belief = beliefs[12]
given_statement = beliefs[0]
correction = phrase([rubicon])
qud = "hang"
alpha = 1. #try different values: 1.0: normal optimality; higher settings: sharper distribution
TIME = helpers.Timer()
smoketest = True #1: calculate small utterance prior in s_1; 0: calculate full big utterance prior in s_1
ext = "reverse" #useful plot file mnemonic
if __name__ == "__main__":
s1_attorney = S(alpha, swk, beliefs, s1_belief, quds)
Listener = L(alpha, swk, beliefs, given_statement, quds)
with TIME("l0 calculation", x=True):
#lit listener
l0_info = "Lit. Listener distribution.\n\n- Correction: " + str(
correction)
l0_dist = Listener.L0(correction)
helpers.plotter(l0_dist,
output="plots/l0_" + str(ext) + ".png",
addinfo=l0_info)
def train(opt):
log = helpers.Logger(opt.verbose)
timer = helpers.Timer()
# Load data =========================================================
log.info('Reading corpora')
# Read vocabs
widss, ids2ws, widst, ids2wt = helpers.get_dictionaries(opt)
# Read training
trainings_data = data.read_corpus(opt.train_src, widss)
trainingt_data = data.read_corpus(opt.train_dst, widst)
# Read validation
valids_data = data.read_corpus(opt.valid_src, widss)
validt_data = data.read_corpus(opt.valid_dst, widst)
# Validation output
if not opt.valid_out:
opt.valid_out = helpers.exp_filename(opt, 'valid.out')
# Get target language model
lang_model = helpers.get_language_model(opt, trainingt_data, widst)
# Create model ======================================================
log.info('Creating model')
s2s = helpers.build_model(opt, widss, widst, lang_model)
# Trainer ==========================================================
trainer = helpers.get_trainer(opt, s2s)
log.info('Using ' + opt.trainer + ' optimizer')
# Print configuration ===============================================
if opt.verbose:
options.print_config(opt,
src_dict_size=len(widss),
trg_dict_size=len(widst))
# Creat batch loaders ===============================================
log.info('Creating batch loaders')
trainbatchloader = data.BatchLoader(trainings_data, trainingt_data,
opt.batch_size)
devbatchloader = data.BatchLoader(valids_data, validt_data,
opt.dev_batch_size)
# Start training ====================================================
log.info('starting training')
timer.restart()
train_loss = 0
processed = 0
best_bleu = -1
deadline = 0
i = 0
for epoch in range(opt.num_epochs):
for x, y in trainbatchloader:
s2s.set_train_mode()
processed += sum(map(len, y))
bsize = len(y)
# Compute loss
loss = s2s.calculate_loss(x, y)
# Backward pass and parameter update
loss.backward()
trainer.update()
train_loss += loss.scalar_value() * bsize
if (i + 1) % opt.check_train_error_every == 0:
# Check average training error from time to time
logloss = train_loss / processed
ppl = np.exp(logloss)
trainer.status()
log.info(
" Training_loss=%f, ppl=%f, time=%f s, tokens processed=%d"
% (logloss, ppl, timer.tick(), processed))
train_loss = 0
processed = 0
if (i + 1) % opt.check_valid_error_every == 0:
# Check generalization error on the validation set from time to time
s2s.set_test_mode()
dev_loss = 0
dev_processed = 0
timer.restart()
for x, y in devbatchloader:
dev_processed += sum(map(len, y))
bsize = len(y)
loss = s2s.calculate_loss(x, y, test=True)
dev_loss += loss.scalar_value() * bsize
dev_logloss = dev_loss / dev_processed
dev_ppl = np.exp(dev_logloss)
log.info(
"[epoch %d] Dev loss=%f, ppl=%f, time=%f s, tokens processed=%d"
%
(epoch, dev_logloss, dev_ppl, timer.tick(), dev_processed))
if (i + 1) % opt.valid_bleu_every == 0:
# Check BLEU score on the validation set from time to time
s2s.set_test_mode()
log.info('Start translating validation set, buckle up!')
timer.restart()
with open(opt.valid_out, 'w+') as f:
for x in valids_data:
y_hat = s2s.translate(x, beam_size=opt.beam_size)
translation = [ids2wt[w] for w in y_hat[1:-1]]
print(' '.join(translation), file=f)
bleu, details = evaluation.bleu_score(opt.valid_dst,
opt.valid_out)
log.info('Finished translating validation set %.2f elapsed.' %
timer.tick())
log.info(details)
# Early stopping : save the latest best model
if bleu > best_bleu:
best_bleu = bleu
log.info('Best BLEU score up to date, saving model to %s' %
s2s.model_file)
s2s.save()
deadline = 0
else:
deadline += 1
if opt.patience > 0 and deadline > opt.patience:
log.info('No improvement since %d epochs, early stopping '
'with best validation BLEU score: %.3f' %
(deadline, best_bleu))
exit()
i = i + 1
trainer.update_epoch()
import helpers
M1 = 3
M2 = 5
N = 1000
res = set()
t1 = helpers.Timer(True)
for i in range((N // M1)+1):
for j in range((N // M2)+1):
S = (i * M1,1)[i == 0] * (j * M2,1)[j == 0]
if S > 1 and S < N:
res.add(S)
t1.stop()
t1.show()
sumi = 0
for i in res:
sumi += i
print(res)
print(sumi)
