def traj_obs(print_num=1000):
""" Returns the trajectory observations.
If the obs have never been computed before, also stores them in a file.
Otherwise reads the cached copy from the disk.
"""
fname = "%s/traj_obs.pkl"%experiment_directory(experiment_name)
fname_test = "%s/traj_obs_test.pkl"%experiment_directory(experiment_name)
if not os.path.exists(fname):
tic("traj_obs: Saving trajectory obs in %s"%fname, experiment_name)
if num_jobs == 1:
seq = (traj_ob for date in dates
for traj_ob in getDayTrajs(data_source['feed'],
basic_geometry['nid'],
date,
basic_geometry['net_type'],
basic_geometry['box'],
experiment_design['trajectory_conversion'],
traj_conv(), net))
else:
from joblib import Parallel, delayed
tic("Using concurrent job code with {0} jobs".format(num_jobs),"learn_procedure")
ls = Parallel(n_jobs=num_jobs)(delayed(wrapper)(data_source['feed'],
basic_geometry['nid'],
date,
basic_geometry['net_type'],
basic_geometry['box'],
experiment_design['trajectory_conversion'],
traj_conv(), net) for date in dates)
seq = [traj_ob for l in ls
for traj_ob in l]
# seq = (traj_ob for tspots_seq in tspots_seqs()
# for traj_ob in traj_conv().mapTrajectory(tspots_seq))
kfold_cross_validation = data_source['kfold_cross_validation']
test_k = data_source['test_k']
assert kfold_cross_validation == 0 or test_k < kfold_cross_validation
f = open(fname, 'w')
if kfold_cross_validation > 0:
tic("traj_obs: Saving test trajectory obs in %s"%fname_test, experiment_name)
f_test = open(fname_test, 'w')
idx = 0
for traj_ob in seq:
idx += 1
if print_num > 0 and idx % print_num == 0:
tic("traj_obs: Converted so far {0} observations".format(idx), experiment_name)
if kfold_cross_validation > 0 and idx % kfold_cross_validation == test_k:
s_dump_elt(traj_ob, f_test)
else:
s_dump_elt(traj_ob, f)
yield traj_ob
else:
tic("traj_obs: opening trajectory obs in %s"%fname, experiment_name)
f = open(fname, 'r')
for traj_ob in s_load(f):
yield traj_ob
def wrapper(*args,**kwargs):
tic("Wrapper called")
res = getDayTrajs(*args, **kwargs)
tic("Cached {0} trajs".format(len(res)))
return res