def make_rows_for_stats(stats):
_rows = [None] * 2
_rows[0] = [''] * (1 + len(all_rooms) * len(all_kinds) * len(all_methods))
for i_roomkind, (room, kind) in enumerate(iterprod(all_rooms, all_kinds)):
_rows[0][1 + len(all_methods) * i_roomkind] = f'room{room} ({kind})'
_rows[1] = [''] + all_methods * len(all_rooms) * len(all_kinds)
for metric in all_metrics:
_row = [f'Δ{metric}' if method_rev == 'delta' else metric]
for room, kind, method in iterprod(all_rooms, all_kinds, all_methods):
_row.append(stats.get(Metadata(method, room, kind, metric), '-'))
_rows.append(_row)
return _rows
def _generate_conformers(self):
"""
Generate rigid body conformers of a complex by (1) Fixing the first m
olecule, (2) initialising the second molecule's COM evenly on the points
of a sphere around the first with a random rotation and (3) iterating
until all molecules in the complex have been added
"""
if len(self.molecules) < 2:
# Single (or zero) molecule complex only has a single *rigid body*
# conformer
self.conformers = [get_conformer(name=self.name, species=self)]
return None
n_molecules = len(self.molecules) # Number of molecules in the complex
self.conformers = []
n = 0 # Current conformer number
points_on_sphere = get_points_on_sphere(
n_points=Config.num_complex_sphere_points)
for _ in iterprod(range(Config.num_complex_random_rotations),
repeat=n_molecules - 1):
# Generate the rotation thetas and axes
rotations = [
np.random.uniform(-np.pi, np.pi, size=4)
for _ in range(n_molecules - 1)
]
for points in iterprod(points_on_sphere, repeat=n_molecules - 1):
conformer = get_conformer(species=self,
name=f'{self.name}_conf{n}')
atoms = get_complex_conformer_atoms(self.molecules, rotations,
points)
conformer.set_atoms(atoms)
self.conformers.append(conformer)
n += 1
if n == Config.max_num_complex_conformers:
logger.warning(
f'Generated the maximum number of complex conformers ({n})'
)
return None
logger.info(f'Generated {n} conformers')
return None
def generate(self, ds):
# check whether the ds is balanced
unique_super = ds.sa[self.attr].unique
nunique_subord = []
for usuper in unique_super:
mask = ds.sa[self.attr].value == usuper
nunique_subord.append(len(np.unique(ds[mask].sa[self.partitioner.attr].value)))
if len(np.unique(nunique_subord)) != 1:
warnings.warn('One or more superordinate attributes do not have the same '
'number of subordinate attributes. This could yield to '
'unbalanced partitions.', category=RuntimeWarning)
# make a fake ds from the first feature to use the attributes
fakeds = ds[:, 0]
if self.selection_strategy != 'equidistant':
raise NotImplementedError("This strategy is not yet implemented")
attr_value = ds.sa[self.attr].value
uattr = ds.sa[self.attr].unique
uattr_masks = [attr_value == u for u in uattr]
for partitionings in iterprod(*[self.partitioner.generate(fakeds[uattr_mask]) for uattr_mask in uattr_masks]):
pds = ds.copy(deep=False)
target_partitioning = np.zeros(len(pds), dtype=int)
for uattr_mask, partitioning in zip(uattr_masks, partitionings):
target_partitioning[uattr_mask] = partitioning.sa[self.partitioner.space].value
pds.sa[self.space] = target_partitioning
yield pds
def make_wtvsparamval(self, allparams=True, paramlist=None):
'''
template usage vs parameter value
'''
if allparams:
paramlist = set(self.test_metadata.colnames) & set(
pca.metadata.colnames)
paramlist = [
n for n in paramlist
if self.test_metadata[n].shape == (len(self.test_metadata), )
]
paramlist = sorted(paramlist)
gs, fig = figures_tools.gen_gridspec_fig(len(paramlist),
border=(1., 1., 0.5, 0.5),
space=(0.7, 0.4),
spsize=(2.5, 1.25))
subplot_inds = iterprod(range(gs._nrows), range(gs._ncols))
fig_axes = {
n: fig.add_subplot(gs[ii, jj])
for (ii, jj), n in zip(subplot_inds, paramlist)
}
for n, ax in fig_axes.items():
ax.hist(self.trn_metadata[n].data,
weights=self.trn_usage_cts,
histtype='step',
density=True)
ax.set_yscale('log')
ax.set_ylabel(r'density', size='x-small')
ax.tick_params(labelsize='x-small')
ax.set_xlabel(n.replace('_', '\_'))
figures_tools.savefig(fig, fname='wtsvsparams.png', fdir=self.workdir)
def make_wtvsparamval(self, allparams=True, paramlist=None):
'''
template usage vs parameter value
'''
if allparams:
paramlist = set(self.test_metadata.colnames) & set(pca.metadata.colnames)
paramlist = [n for n in paramlist
if self.test_metadata[n].shape == (len(self.test_metadata), )]
paramlist = sorted(paramlist)
gs, fig = figures_tools.gen_gridspec_fig(
len(paramlist), border=(1., 1., 0.5, 0.5), space=(0.7, 0.4),
spsize=(2.5, 1.25))
subplot_inds = iterprod(range(gs._nrows), range(gs._ncols))
fig_axes = {n: fig.add_subplot(gs[ii, jj])
for (ii, jj), n in zip(subplot_inds, paramlist)}
for n, ax in fig_axes.items():
ax.hist(self.trn_metadata[n].data, weights=self.trn_usage_cts,
histtype='step', density=True)
ax.set_yscale('log')
ax.set_ylabel(r'density', size='x-small')
ax.tick_params(labelsize='x-small')
ax.set_xlabel(n.replace('_', '\_'))
figures_tools.savefig(fig, fname='wtsvsparams.png', fdir=self.workdir)
def convert_xticklabels(suffix, selected_rooms, selected_kinds):
if suffix == 'unseens':
return [str(i + 1) for i in range(len(selected_rooms))]
else:
return [f'room{i // 2 * 2 + 2}\n({kind})'
for i, kind in iterprod(range(len(selected_rooms)), selected_kinds)
]
def generate(self, ds):
# check whether the ds is balanced
unique_super = ds.sa[self.attr].unique
nunique_subord = []
for usuper in unique_super:
mask = ds.sa[self.attr].value == usuper
nunique_subord.append(
len(np.unique(ds[mask].sa[self.partitioner.attr].value)))
if len(np.unique(nunique_subord)) != 1:
warnings.warn(
'One or more superordinate attributes do not have the same '
'number of subordinate attributes. This could yield to '
'unbalanced partitions.',
category=RuntimeWarning)
# make a fake ds from the first feature to use the attributes
fakeds = ds[:, 0]
if self.selection_strategy != 'equidistant':
raise NotImplementedError("This strategy is not yet implemented")
attr_value = ds.sa[self.attr].value
uattr = ds.sa[self.attr].unique
uattr_masks = [attr_value == u for u in uattr]
for partitionings in iterprod(*[
self.partitioner.generate(fakeds[uattr_mask])
for uattr_mask in uattr_masks
]):
pds = ds.copy(deep=False)
target_partitioning = np.zeros(len(pds), dtype=int)
for uattr_mask, partitioning in zip(uattr_masks, partitionings):
target_partitioning[uattr_mask] = partitioning.sa[
self.partitioner.space].value
pds.sa[self.space] = target_partitioning
yield pds
def reduce(field,target):
outfield = field.copy()
maxx, maxy = field.shape
for i in iterprod(arange(0,maxx),arange(0,maxy)):
if field[i] == target:
if identical_neighbours(field,i):
outfield[i] += 1
return outfield
def wordbreak(s):
s = s.lower()
if s in arpabet:
return arpabet[s]
middle = len(s) / 2
partition = sorted(list(range(len(s))), key=lambda x: (x - middle)**2 - x)
for i in partition:
pre, suf = (s[:i], s[i:])
if pre in arpabet and wordbreak(suf) is not None:
return [x + y for x, y in iterprod(arpabet[pre], wordbreak(suf))]
return None
def make_rows_for_stats(stats):
_rows = [None] * 2
_rows[0] = [''] * (1 + len(all_kinds) * len(all_methods))
for i_kind, kind in enumerate(all_kinds):
_rows[0][1 + len(all_methods) * i_kind] = kind
_rows[1] = [''] + all_methods * len(all_kinds)
for metric in all_metrics:
_row = [metric]
for kind, method in iterprod(all_kinds, all_methods):
_row.append(stats[MethodKindMetric(method, kind, metric)])
_rows.append(_row)
return _rows
def buildorder(formation):
""" Determine bounds of formation """
minx = formation[0][0]
maxx = formation[0][0]
miny = formation[0][1]
maxy = formation[0][1]
for x,y in formation:
if minx > x:
minx = x
elif maxx < x:
maxx = x
if miny > y:
miny = y
elif maxy < y:
maxy = y
""" Create Required Arrays """
""" Both arrays are 2 larger than te formation requires"""
field = ones((maxx - minx + 3, maxy - miny + 3))
form = zeros((maxx - minx + 3, maxy - miny + 3))
fmaxx, fmaxy = field.shape
""" Filling array with truth values according to formation """
for x,y in map(lambda x: x - array([minx - 1,miny - 1]), formation):
form[x,y] = 1
form = form == 1
""" Paint-like fill operation on 0,0, sets them at 0 """
field = fill(field, form)
""" Will perform a reduction operation on each square with value of target"""
""" It will add 1 to the square if all it's neighbors also have the value of target"""
""" This results in a height/distance-from-border map type situation"""
target = 0
while target != amax(field):
target += 1
field = reduce(field, target)
m = amax(field)
outformation = []
""" Recreates formation as a tuple of coordinates and the height/distance"""
for i in iterprod(arange(0,field.shape[0]),arange(0,field.shape[1])):
if form[i]:
outformation.append((m - field[i],array(i)))
""" Sorts the order based on lexicographical ordering and height/distance"""
outformation = sorted(outformation, key=lambda x: x[1][0] + x[1][1]*fmaxx + x[0]*fmaxx*fmaxy)
""" Restore Origin"""
return [(x[0], x[1] + array([minx -1, miny -1])) for x in outformation]
def wordbreak(s):
# print(f's: "{s}"')
s = s.replace("'", "").replace('"', '')
s = s.lower()
if s in arpabet:
return arpabet[s]
middle = len(s) / 2
partition = sorted(list(range(len(s))), key=lambda x: (x - middle)**2 - x)
# print(f'partition: "{partition}"')
for i in partition:
pre, suf = (s[:i], s[i:])
if pre in arpabet and wordbreak(suf) is not None:
return [x + y for x, y in iterprod(arpabet[pre], wordbreak(suf))]
return None
def requires(self):
dates = utils.get_valid_dates(self.start, self.end) # Add Mods Product
if self.edos:
munis = pu.get_munis(self.edos)
for cutoff, cve_muni in iterprod(self.cutoffs, munis):
yield CreateCluster(dates=dates,
cutoff=cutoff,
cve_muni=cve_muni)
elif cve_muni:
for cutoff in self.cutoffs:
yield CreateCluster(dates=dates,
cutoff=cutoff,
cve_muni=self.cve_muni)
else:
pass
def wordbreak(s):
try:
arpabet = nltk.corpus.cmudict.dict()
except LookupError:
nltk.download('cmudict')
arpabet = nltk.corpus.cmudict.dict()
s = s.lower()
if s in arpabet:
return arpabet[s]
middle = len(s) / 2
partition = sorted(list(range(len(s))), key=lambda x: (x - middle)**2 - x)
for i in partition:
pre, suf = (s[:i], s[i:])
if pre in arpabet and wordbreak(suf) is not None:
return [x + y for x, y in iterprod(arpabet[pre], wordbreak(suf))]
return None
def __init__(self, fracs):
self.fracs = fracs
self.nfracs = len(fracs)
# load figure and gridspec object
self.gs, self.fig = gen_gridspec_fig(
self.nfracs, spsize=(2.5, 1.))
self.subplot_inds = list(iterprod(range(self.gs._nrows), range(self.gs._ncols)))
self.fracs_plot_ix = dict(zip(sorted(fracs), self.subplot_inds))
self.fig_axes = {f: self.fig.add_subplot(self.gs[ix[0], ix[1]])
for f, ix in self.fracs_plot_ix.items()}
for f_, ax_ in self.fig_axes.items():
ax_.tick_params(labelsize='x-small')
ax_.set_xscale('log')
ax_.set_xlim([.1, 200.])
ax_.text(x=.2, y=.1, s='f = {:.2f}'.format(f_), size='x-small')
self.fig.suptitle('SNR vs PDF population', size='x-small')
def test_xrandom_iterprod():
# just some exhaustive one
all_12ab = [[1, 'a'], [1, 'b'], [2, 'a'], [2, 'b']]
assert_equal(sorted(xrandom_iterprod(10, [1, 2], 'ab')), all_12ab)
# Let's do a few of some long ones, random ones and verify that come out correctly
for i in xrange(10):
ns = random.randint(1, 5)
seqs = [range(random.randint(1, 5)) for i in range(ns)]
all_prods = set(map(tuple, iterprod(*seqs)))
for count in [random.randint(0, 8) for i in range(3)]:
real_count = min(count, np.prod(map(len, seqs)))
r_prods = set(map(tuple, xrandom_iterprod(count, *seqs)))
assert_equal(len(r_prods), real_count)
# assert that they are all unique
assert_equal(len(set(r_prods)), real_count)
# assert that they are all a part of all prods
assert all_prods.issuperset(r_prods)
def test_xrandom_iterprod():
# just some exhaustive one
all_12ab = [[1, 'a'], [1, 'b'], [2, 'a'], [2, 'b']]
assert_equal(sorted(xrandom_iterprod(10, [1, 2], 'ab')), all_12ab)
# Let's do a few of some long ones, random ones and verify that come out correctly
for i in xrange(10):
ns = random.randint(1, 5)
seqs = [range(random.randint(1, 5)) for i in range(ns)]
all_prods = set(map(tuple, iterprod(*seqs)))
for count in [random.randint(0, 8) for i in range(3)]:
real_count = min(count, np.prod(map(len, seqs)))
r_prods = set(map(tuple, xrandom_iterprod(count, *seqs)))
assert_equal(len(r_prods), real_count)
# assert that they are all unique
assert_equal(len(set(r_prods)), real_count)
# assert that they are all a part of all prods
assert all_prods.issuperset(r_prods)
def rec_int(f, ranges, max_num_evals):
"""Evaluates the integral of the function 'f' over the coordinate ranges
indicated by 'ranges'. Uses the rectangle rule and no more than
'max_num_evals' evaluations."""
from itertools import product as iterprod
from math import pow, floor, fsum
num_dims = len(ranges)
points_per_dim = floor(pow(max_num_evals, 1.0 / num_dims))
num_evals = points_per_dim**num_dims
coord_ranges = [[lo + (hi - lo) / (2 * points_per_dim) +\
((hi - lo) / points_per_dim) * i
for i in range(points_per_dim)]
for lo, hi in ranges]
accum = fsum(f(x) for x in iterprod(*coord_ranges))
volume = mul(hi - lo for lo, hi in ranges)
return accum * volume / num_evals
def rec_int(f, ranges, max_num_evals):
"""Evaluates the integral of the function 'f' over the coordinate ranges
indicated by 'ranges'. Uses the rectangle rule and no more than
'max_num_evals' evaluations."""
from itertools import product as iterprod
from math import pow, floor, fsum
num_dims = len(ranges)
points_per_dim = floor(pow(max_num_evals, 1.0 / num_dims))
num_evals = points_per_dim ** num_dims
coord_ranges = [[lo + (hi - lo) / (2 * points_per_dim) +\
((hi - lo) / points_per_dim) * i
for i in range(points_per_dim)]
for lo, hi in ranges]
accum = fsum(f(x) for x in iterprod(*coord_ranges))
volume = mul(hi - lo for lo, hi in ranges)
return accum * volume / num_evals
def get_raw_hjortevillt_df():
"""Get the data from hjorteviltregisteret
"""
animals = [
'Elg',
'Hjort']
data_sets = [
'Sett',
'Felt',
'Slaktevekt',
]
# download the data and turn each table into an individual frame
with ThreadPool(6) as pool:
frames = pool.map(download_from_hjortevilltregisteret,
iterprod(animals, data_sets))
# join the frames into one single large frame
df = pd.concat([frames[0].iloc[:, :3]]
+ [i.iloc[:, 3:] for i in frames],
axis=1)
df.to_csv(RAW_FILENAME, index=False)
return df
def generate_order_formation(self, n):
n = n + 1
mindist = maxsize
for i in range(1,n):
if abs(floor(n/i + 1) - (i*2 -1)) < mindist:
mindist = abs(floor(n/i + 0.5) - (i*2 -1))
else:
y = floor(n/(i - 1) + 1)
x = i - 1
break
l = list(array(x) for x in iterprod(arange(-1*floor(y/2),floor(y/2)+1),
arange(-1*floor((x*2-1)/2), floor((x*2-1)/2)+1, 2)))
for i in range(0,len(l)):
if array_equal(l[i],array([0,0])):
del l[i]
break
n = n - 1
return l[0:n]
def drawocclusionshading(self, cc, state):
if self.selectedbee is not None:
pos = next((x[0] for x in state if x[1] is self.selectedbee), None)
positions, bees, movement, communication = map(list, zip(*state))
xmin,ymin = self.fi(array([0,0]))
xmax,ymax = self.fi(array([1,1]))
for square in iterprod(range(int(xmin)-2,int(xmax)+2),
range(int(ymin)-2,int(ymax)+2)):
square = array(square)
if not lineofsight(pos, square, positions):
x,y = self.f(square)
cc.save()
cc.translate(x,y)
cc.scale(1/self.worldsize[0], 1/self.worldsize[1])
cc.rectangle(-0.5,-0.5,1,1)
cc.restore()
cc.set_source_rgb(0.9, 0.9, 0.9)
cc.fill()
_rows[0][1 + len(all_methods) * i_roomkind] = f'room{room} ({kind})'
_rows[1] = [''] + all_methods * len(all_rooms) * len(all_kinds)
for metric in all_metrics:
_row = [f'Δ{metric}' if method_rev == 'delta' else metric]
for room, kind, method in iterprod(all_rooms, all_kinds, all_methods):
_row.append(stats.get(Metadata(method, room, kind, metric), '-'))
_rows.append(_row)
return _rows
path_csv = (path_root / fname_result).with_suffix('.csv')
with path_csv.open('w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
rows = []
for method in all_methods:
rows.append([method])
for room, kind, (i, metric) in iterprod(all_rooms, all_kinds, enumerate(all_metrics)):
values = all_arrays.get(Metadata(method, room, kind, metric), ['-'])
rows.append([
f'room{room} ({kind})' if i == 0 else '',
f'Δ{metric}' if method_rev == 'delta' else metric,
*values,
])
rows += [['means'], *make_rows_for_stats(all_means)]
rows += [['stds'], *make_rows_for_stats(all_stds)]
for r in rows:
writer.writerow(r)
# %% plotting functions
def main(task_info, paths):
print """
"""
task = {
'KIND_RUN' : 'TONAME',
'FILE_INIT': 'GENERATOR_GLOBAL',
'LENGTH_FS': 3,
'INITIALIZATION': 'DIABATIC',
'NUMBER_CONFIG' : 1,
'NUMBER_REPEAT' : 1,
'LIGHT' : False,
'FIRST_MOL_CHAIN' : [1, 1, 1],
'LAST_MOL_CHAIN' : [2, 1, 1]
}
task_info.update(task)
cp2k_param = [
#[ 'PROPAGATION', 'FSSH'],
['PROPAGATION', 'FSSH'],
#['DECO', 'NO_DECO_CORR','INSTANT_COLLAPSE','DAMPING']
[ 'DECO', 'DAMPING'],
#['METHOD_RESCALING','NACV','SIMPLE_QSYS']
[ 'METHOD_RESCALING', 'NACV'],
#[ 'METHOD_ADIAB_NACV', 'FAST','TOTAL']
[ 'METHOD_ADIAB_NACV', 'FAST'],
#['METHOD_REVERSAL', 'NEVER', 'ALWAYS', 'TRUHLAR', 'SUBOTNIK']
[ 'METHOD_REVERSAL', 'ALWAYS'],
#[ 'SCALING', 0.05, 0.03, 0.01, 0.008, 0.005, 0.003, 0.001, 0.0005, 0.0001, 0.00005],
[ 'SCALING', 0.00005],
#['SCALING', 0.003, 0.00005],
#[ 'TIMESTEP', 0.01, 0.05, 0.1, 0.5],
[ 'TIMESTEP', 0.1],
#['EDC_E0', 0.01, 0.1, 1.0],
['EDC_E0', 0.1],
#['ELECTRONIC_STEPS', 5, 10, 50],
['ELECTRONIC_STEPS', 5],
[ 'TEMPLATE_FILE', 'FSSH_CORE.template'],
[ 'FORCEFIELD_FILE', 'FSSH_FF.template'],
['INITIALIZATION', 'DIABATIC'],
['INIT'] + range(1, task_info.get('NUMBER_CONFIG') + 1),
['REPEAT'] + range(task_info.get('NUMBER_REPEAT'))
]
# BUILD THE MEGA_LISTS
second_list = [ sublist[1:] for sublist in cp2k_param]
total_list = list(iterprod(*second_list))
mega_list = []
for sublist in total_list:
subdict = {}
for index in range(len(sublist)):
subdict.update({
cp2k_param[index][0] : sublist[index]
})
mega_list.append(subdict)
# SET_UP THE DIRECTORY, CHECK ANY SUBDIR IS PRESENT
bucket = Bucket(task_info)
bucket.name()
paths.update({'bucket': bucket.path})
task = Dir(task_info.get('INPUT_INFO'))
paths.update( {'task' : task.path} )
templates = Dir('templates', paths)
templates.checkdir()
templates.clean()
supinitial = Dir('initial')
supinitial.checkdir()
bin = Dir('bin', paths)
bin.checkdir()
# PREPARE THE MEGA_LIST FOR POOL
for ndir in range(len(mega_list)):
mega_list[ndir].update({ 'NDIR' : ndir,
'PATHS_DICT' : paths,
'INPUTS_DICT' : task_info
})
# RUN THE CALCULATIONS, SERIE OR PARALLEL ACCORDING TO THE NWORKER VARIABLE
nworker = task_info['NWORKER']
if nworker == 0:
for cp2k_info in mega_list:
run_fssh(cp2k_info)
else:
from multiprocessing import Pool, cpu_count
if nworker == -1:
nworker = cpu_count()
pool = Pool(nworker)
pool.map( run_fssh, mega_list)
def formations():
formations = []
formations.append(list(iterprod(arange(0,3),arange(0,3))))
formations.append(list(iterprod(arange(0,4),arange(0,4))))
f = list(iterprod(arange(0,5),arange(0,5)))
for x in [18, 17, 16, 13, 11, 8, 7, 6 ]:
del f[x]
formations.append(f)
formations.append(list(iterprod(arange(0,8,2),arange(0,8,2))))
f = list(iterprod(arange(0,3),arange(0,3)))
f = f + list(iterprod(arange(8,11),arange(5,8)))
formations.append(f)
f = list(iterprod(arange(0,3),arange(0,3)))
f = f + list(iterprod(arange(1,4),arange(1,4)))
for x in [14, 13, 11, 10]:
del f[x - 1]
formations.append(f)
i = 101
random.seed(i)
formations.append(random.sample(
list(iterprod(arange(0,5),arange(0,5)))
, 10))
i += 1
random.seed(i)
formations.append(random.sample(
list(iterprod(arange(0,10),arange(0,10)))
, 10))
i +=1
random.seed(i)
formations.append(random.sample(
list(iterprod(arange(0,10),arange(0,10)))
, 15))
i += 1
random.seed(i)
formations.append(random.sample(
list(iterprod(arange(0,15),arange(0,15)))
, 15))
i +=1
return formations
def iter_bin_vec(n):
for i, z in enumerate(iterprod(*[[0, 1]] * n)):
yield z
def plot_all_predictions(self, path_stem):
a_dict = {}
params_path = path_stem + ".params"
with open(params_path,'r') as f:
for step, line in enumerate(f.readlines()):
a_dict[step] = literal_eval(line)
#print(a_dict)
# For the moment, if a list of years is given, we are only looking at the first year.
year_path = f"{self.vis_target}/{a_dict[0]['YEAR'][0]}"
def make_month_path(base_path, month, dict0, seed):
suffix = ""
if dict0["SUPER"]:
suffix += "-LvlUp"
if dict0["BUFFER"]:
suffix += f"-{dict0['BUFFER']}meter"
if dict0["USE_PCA"]:
suffix += "-PCA"
if dict0["VALIDATE"]:
suffix += f"-{dict0['VALIDATE'] - 1:.2f}_{seed}"
return f"{base_path}/month{month}{suffix}"
month_paths = {}
for month in a_dict[1]:
month_paths[month] = make_month_path(year_path, month, a_dict[0], a_dict[1][month]["seed"])
#print(month_paths)
region_paths = {}
for month in month_paths:
region_paths[month] = {}
for region in a_dict[0]["REG_LIST"]:
reg_string = "_".join(region)
region_paths[month][reg_string] = f"{month_paths[month]}/{reg_string}"
#print(region_paths)
method_paths = {}
for month in a_dict[1]:
print(f"month: {month}")
method_paths[month] = {}
reg_list = f"{path_stem}.{month}reg"
with open(reg_list, 'r') as regions:
for line in regions.readlines():#region in a_dict[0]["REG_LIST"]:
region = "_".join(line.strip().split(","))
#print(f"region: {region}")
method_paths[month][region] = {}
image_origen = f"{region_paths[month][region]}/{SUB_FIGS}/{SUB_PRED}/origen.png"
display(Image(filename=image_origen))
for method in a_dict[0]["MODICT"]:
#print(f"method: {method}")
paramdict = a_dict[0]["MODICT"][method]
params = paramdict.keys()
argums = paramdict.values()
arg_combos = iterprod(*argums)
# Don't print the following! It exhausts the generator.
##print(f"The possible combinations of arguments are: {list(arg_combos)}")
bash_suffixes = [zip(params, ac) for ac in arg_combos]
file_suffixes = ["".join([f"{par}{arg}" for par, arg in bashix]) for bashix in bash_suffixes]
for suffix in file_suffixes:
image_path = f"{region_paths[month][region]}/{SUB_FIGS}/{SUB_PRED}/{method}{suffix}-plot.png"
if os.path.exists(image_path):
#print(image_path)
display(Image(filename=image_path))
with open(f"{region_paths[month][region]}/{SUB_PRED}/{method}{suffix}.log", 'r') as f:
print("Method completion time:", f.readlines()[-1])
def main(task_info, paths):
print """
"""
task = {
'KIND_RUN': 'TONAME',
'FILE_INIT':
'TASK279-SAMPLE-BO-CORRECT-TEMP-100ps-20dabab-170601-8da556911f813378f0577abfd206e148',
'FILE_DICT':
'TASK279-SAMPLE-BO-CORRECT-TEMP-100ps-20dabab-170601-8da556911f813378f0577abfd206e148-1706121812',
'LENGTH_FS': 1,
'INITIALIZATION': 'ADIABATIC',
'NUMBER_ADIABAT': 2,
'NUMBER_CONFIG': 100,
'FIRST_CONFIG': 200,
'FINAL_CONFIG': 500,
'NUMBER_REPEAT': 10,
'LIGHT': True
}
task_info.update(task)
seed()
cp2k_param = [
['PROPAGATION', 'FSSH'],
#['TRIVIAL', 'TRIVIAL_HOP_CORRECT','UNMODIFIED_SURF_HOP']
['TRIVIAL', 'TRIVIAL_HOP_CORRECT', 'UNMODIFIED_SURF_HOP'],
# ['DECO', 'NO_DECO_CORR','INSTANT_COLLAPSE','DAMPING']
['DECO', 'DAMPING'],
#['METHOD_RESCALING','NACV','SIMPLE_QSYS']
['METHOD_RESCALING', 'NACV'],
#[ 'METHOD_ADIAB_NACV', 'FAST','TOTAL']
['METHOD_ADIAB_NACV', 'FAST'],
#['METHOD_REVERSAL', 'NEVER', 'ALWAYS', 'TRUHLAR', 'SUBOTNIK']
['METHOD_REVERSAL', 'ALWAYS'],
#[ 'SCALING', 0.05, 0.03, 0.01, 0.008, 0.005, 0.003, 0.001, 0.0005, 0.0001, 0.00005],
#[ 'SCALING', 0.03, 0.02, 0.01, 0.008, 0.003, 0.0005, 0.00005],
['SCALING', 0.03],
#['SCALING', 0.003, 0.00005],
#[ 'TIMESTEP', 0.01, 0.05, 0.1, 0.5],
['TIMESTEP', 0.5],
#['EDC_E0', 0.01, 0.1, 1.0],
['EDC_E0', 0.1],
#['ELECTRONIC_STEPS', 5, 10, 50],
['ELECTRONIC_STEPS', 5],
['TEMPLATE_FILE', 'FSSH_CORE.template'],
['FORCEFIELD_FILE', 'FSSH_FF.template'],
['INITIALIZATION', 'ADIABATIC'],
['INIT_CONFIG'] +
[(ind + 1,
range(task_info['FIRST_CONFIG'], task_info['FINAL_CONFIG'],
(task_info['FINAL_CONFIG'] - task_info['FIRST_CONFIG']) /
task_info['NUMBER_CONFIG'])[ind])
for ind in range(task_info['NUMBER_CONFIG'])],
['REPEAT'] + range(task_info.get('NUMBER_REPEAT')),
]
print "How many run?", len(cp2k_param)
dict_for_equilibrium = {}
with open(
'initial/result-thermodynamics-population-extract-scaling-adiabat-%s/Simulation-1.dat'
% task_info['FILE_DICT']) as file_dict:
for line in file_dict.readlines()[1:]:
if '#' in line:
pass
else:
dict_for_equilibrium[float(
line.split()[0])] = [float(x) for x in line.split()[1:]]
print dict_for_equilibrium
# BUILD THE MEGA_LISTS
second_list = [sublist[1:] for sublist in cp2k_param]
total_list = list(iterprod(*second_list))
mega_list = []
for sublist in total_list:
subdict = {}
for index in range(len(sublist)):
subdict.update({cp2k_param[index][0]: sublist[index]})
mega_list.append(subdict)
# SET_UP THE DIRECTORY, CHECK ANY SUBDIR IS PRESENT
bucket = Bucket(task_info)
bucket.name()
paths.update({'bucket': bucket.path})
task = Dir(task_info.get('INPUT_INFO'))
paths.update({'task': task.path})
templates = Dir('templates', paths)
templates.checkdir()
templates.clean()
supinitial = Dir('initial')
supinitial.checkdir()
bin = Dir('bin', paths)
bin.checkdir()
# PREPARE THE MEGA_LIST FOR POOL
for ndir in range(len(mega_list)):
mega_list[ndir].update({
'NDIR': ndir,
'PATHS_DICT': paths,
'INPUTS_DICT': task_info,
'DICT_EQUILIBRIUM': dict_for_equilibrium
})
# RUN THE CALCULATIONS, SERIE OR PARALLEL ACCORDING TO THE NWORKER VARIABLE
nworker = task_info['NWORKER']
if nworker == 0:
for cp2k_info in mega_list:
run_fssh(cp2k_info)
else:
from multiprocessing import Pool, cpu_count
if nworker == -1:
nworker = cpu_count()
pool = Pool(nworker)
pool.map(run_fssh, mega_list)
with (path_root / fname).open('rb') as f:
dict_pickled = pickle.load(f)
rooms = set()
kinds = set()
for key, value in dict_pickled.items():
_, room, kind, metric = key
rooms.add(room)
kinds.add(kind)
all_metrics.add(metric)
for metric in all_metrics:
means = []
for i_method, method in enumerate(all_methods):
array = []
for room, kind in iterprod(rooms, kinds):
array += dict_pickled[method, room, kind, metric]
means.append(np.mean(array))
if metric not in all_means:
all_means[metric] = dict()
all_means[metric][phasetype] = np.array(means)
# %% draw bar plots
plt.style.use('default')
plt.rc('font', family='Arial', size=18)
figs = []
for metric, means in all_means.items():
fig: plt.Figure = None
def model(REGION, TRAIN_DIR, EVAL_DIR, OUT_DIR, MODELS, NOTE):
TRAVIS_MODEL = pathlib.Path("hypppo7.py").resolve()
MARIO_MODEL = pathlib.Path("2b-kknn.R").resolve()
RF_MODEL = pathlib.Path("2c-rf.R").resolve()
# If 0 is given instead of a specified region,
# create list of all regions in the TRAIN_DIR
if not REGION:
REGIONS = listdir(TRAIN_DIR)
else:
REGIONS = [REGION]
#print(REGIONS)
# Prepare the list of parameter combinations for each model
suffixes = {}
for MODEL in MODELS:
paramdict = MODELS[MODEL]
params = paramdict.keys()
argums = paramdict.values()
arg_combos = iterprod(*argums)
# Don't print the following! It exhausts the generator.
##print(f"The possible combinations of arguments are: {list(arg_combos)}")
bash_suffixes = [list(zip(params, ac)) for ac in arg_combos]
file_suffixes = [
"".join([f"{par}{arg}" for par, arg in bashix])
for bashix in bash_suffixes
]
suffixes[MODEL] = list(zip(bash_suffixes, file_suffixes))
for REGION in REGIONS:
# Specify train and predi files
TR = TRAIN_DIR.joinpath(REGION)
EV = EVAL_DIR.joinpath(REGION)
# Create an output folder for the region, with predictions and logs subfolders
OUT = OUT_DIR.joinpath(REGION).with_suffix("")
PRED = OUT.joinpath(SUB_PRED)
if not PRED.is_dir():
PRED.mkdir(parents=True)
for MODEL in MODELS:
# The following has been extracted outside the REGIONS loop for a one-time creation
#paramdict = MODELS[MODEL]
#params = paramdict.keys()
#argums = paramdict.values()
#arg_combos = iterprod(*argums)
## Don't print the following! It exhausts the generator.
###print(f"The possible combinations of arguments are: {list(arg_combos)}")
#
#for ac in arg_combos:
# file_name = f"{MODEL}"#{NOTE}"
# bash_suffix = []
# for p, a in zip(params, ac):
# file_name += f"{p}{a}"
# bash_suffix.extend([p, a])
for bash_suf, file_suf in suffixes[MODEL]:
t0 = time()
# Specify paths of output files
file_name = MODEL + file_suf
PRD = PRED.joinpath(f"{file_name}.csv")
LOG = PRED.joinpath(f"{file_name}.log")
# Open the log file and start writing
with open(LOG, "w") as log:
log.write(f"t0={t0}\n")
log.write(f"Prediction file: {PRD}\n")
log.write(f"Log file: {LOG}\n")
log.write(f"{REGION} {MODEL}\n")
log.write(f"bash_suffix: {bash_suf}\n")
if (MODEL in ["HYPPO", "KNN", "SBM"]):
bash_args = [
TRAVIS_MODEL, "-t", TR, "-e", EV, "-m", MODEL, "-o",
PRD, "-l", LOG
]
elif (MODEL == "1NN"):
bash_args = [
TRAVIS_MODEL, "-t", TR, "-e", EV, "-m", "KNN", "-o",
PRD, "-s", 0, "-S", "1,2", "-v", 2, "-k", 1, "-l", LOG
]
elif (MODEL == "KKNN"):
bash_args = [
MARIO_MODEL, "-t", TR, "-e", EV, "-o", PRD, "-l", LOG
]
elif (MODEL == "RF"):
bash_args = [
RF_MODEL, "-t", TR, "-e", EV, "-o", PRD, "-l", LOG
]
else:
print(f"Model \"{MODEL}\" unknown!")
bash_args = ["echo", f"Model \"{MODEL}\" unknown!"]
for bashix in bash_suf:
bash_args.extend(bashix)
with open(LOG, "a") as log:
log.write(f"bash_args: {bash_args}\n")
bash(bash_args)
t1 = time()
with open(LOG, "a") as log:
log.write(f"t1={t1}\n")
log.write(f"t={t1 - t0}\n")
def main(task_info, paths):
task = {
'KIND_RUN' : 'TASK234-SAMPLE-BO',
'INITIALIZATION' : 'SAMPLE_BO',
'NUMBER_CONFIG' : 1,
'FILE_INIT' : {
1 :'config-GENERATOR_GLOBAL',
2 :'config-GENERATOR_GLOBAL',
3 :'config-GENERATOR_GLOBAL'
},
'LENGTH_FS': 100000,
'OUTPUT_CONFIG' : 1
}
task_info.update(task)
cp2k_param = [
[ 'PROPAGATION', 'BORN_OPPENHEIMER'],
[ 'INIT'] + range(1, task_info.get('NUMBER_CONFIG') + 1),
['SCALING', 0.00005],
#[ 'SCALING', 0.05, 0.03, 0.02, 0.01, 0.008, 0.005, 0.003, 0.001, 0.0005, 0.0001, 0.00005],
[ 'TIMESTEP', 0.1],
[ 'TEMPLATE_FILE', 'FSSH_CORE.template'],
[ 'FORCEFIELD_FILE', 'FSSH_FF.template'],
['FIRST_ADIABAT',1,2,3],
['DENSITY', 0.001],
['CC_CHARGED', 1.369],
['INITIALIZATION', 'ADIABATIC']
]
# BUILD THE MEGA_LISTS
second_list = [ sublist[1:] for sublist in cp2k_param]
total_list = list(iterprod(*second_list))
mega_list = []
for sublist in total_list:
subdict = {}
for index in range(len(sublist)):
subdict.update({
cp2k_param[index][0] : sublist[index]
})
mega_list.append(subdict)
# SET_UP THE DIRECTORY, CHECK ANY SUBDIR IS PRESENT
bucket = Bucket(task_info)
bucket.name()
paths.update({'bucket': bucket.path})
task = Dir(task_info.get('INPUT_INFO'))
paths.update( {'task' : task.path} )
templates = Dir('templates', paths)
templates.checkdir()
templates.clean()
bin = Dir('bin', paths)
bin.checkdir()
output_total = Dir('output')
output_total.rm_mkdir()
# PREPARE THE MEGA_LIST FOR POOL
for ndir in range(len(mega_list)):
mega_list[ndir].update({ 'NDIR' : ndir,
'PATHS_DICT' : paths,
'INPUTS_DICT' : task_info
})
# RUN THE CALCULATIONS, SERIE OR PARALLEL ACCORDING TO THE NWORKER VARIABLE
nworker = task_info['NWORKER']
if nworker == 0:
for cp2k_info in mega_list:
run_fssh(cp2k_info)
else:
from multiprocessing import Pool, cpu_count
if nworker == -1:
nworker = cpu_count()
pool = Pool(nworker)
pool.map( run_fssh, mega_list)
'density': [150],
'coh_mean': [40],
'coh_stdev': [10],
'direction': ['left'],
'num_frames': [24],
'diameter': [8],
'cp_time': [.2]
})
num_trials = 100
# edit params so that shorter entries are recycled
# I found inspiration here: https://stackoverflow.com/a/12913336
dummy_list = list(params.values())
combinations = list(iterprod(*dummy_list))
num_comb = len(combinations)
import pprint
for dset_idx in range(num_comb):
curr_dict = {k: combinations[dset_idx][ix] for ix, k in enumerate(params.keys())}
S = DotsStimulus(**curr_dict)
# only create the file at first iteration
write_stimulus_to_file(S, num_trials, file_name, create_file=file_does_not_exist)
print("--- {} seconds ---".format(time.time() - start_time))
pprint.pprint(inspect_db(file_name), width=120)
for init in info['FILE_INIT']:
system_info = (InputFile('%s/system.info' % init).dict)
system_info.update({'AOM_RADIUS': info['AOM_RADIUS']})
system_info.update({'FILE_INIT': init})
system_info = add_list_activated(system_info, init)
list_config_init = range(
0, system_info['NPROD_INIT'],
system_info['NPROD_INIT'] / system_info['NCONFIG_INIT'])
cp2k_param_here = list(cp2k_param)
cp2k_param_here.append(['INIT'] + [
list_config_init[x]
for x in range(0, len(list_config_init),
len(list_config_init) / info['NUMBER_CONFIG'])
])
second_list = [sublist[1:] for sublist in cp2k_param_here]
total_list = list(iterprod(*second_list))
for sublist in total_list:
subdict = {}
for index in range(len(sublist)):
subdict.update({cp2k_param_here[index][0]: sublist[index]})
subdict.update(system_info)
subdict.update({'ARCHER': archer})
mega_list.append(subdict)
for ndir in range(len(mega_list)):
mega_list[ndir].update({
'NDIR': ndir,
'PATHS_DICT': paths,
'INPUTS_DICT': info
})
# RUN THE CALCULATIONS, SERIE OR PARALLEL ACCORDING TO THE NWORKER VARIABLE
'list_fname'),
chars_as_strings=True,
squeeze_me=True)
flist_speech = metadata_ref['path_all_speech']
flist_speech = [Path(p.rstrip()) for p in flist_speech]
n_speech = len(flist_speech)
list_fname_ref = metadata_ref['list_fname']
list_feature: List[Tuple] = list_fname_to_feature(list_fname_ref)
n_feature = len(list_feature)
else:
flist_speech = list(path_speech.glob('**/*.WAV')) + list(
path_speech.glob('**/*.wav'))
n_speech = len(flist_speech)
list_feature = [
(i_speech, hp.room_create, i_loc)
for i_speech, i_loc in iterprod(range(n_speech), range(n_loc))
]
# uniformly random sample
if args.kind_data.lower() == 'train':
n_feature = hp.n_data_per_room
else:
n_feature = hp.n_test_per_room
idx_choice = np.random.choice(len(list_feature),
n_feature,
replace=False)
idx_choice.sort()
list_feature: List[Tuple] = [list_feature[i] for i in idx_choice]
if n_feature < args.from_idx:
raise ArgumentError
def iter_bin_vec(n):
for i,z in enumerate(iterprod(*[[0,1]]*n)):
yield z
_rows[1] = [''] + all_methods * len(all_kinds)
for metric in all_metrics:
_row = [metric]
for kind, method in iterprod(all_kinds, all_methods):
_row.append(stats[MethodKindMetric(method, kind, metric)])
_rows.append(_row)
return _rows
path_csv = (path_root / fstem_analysis).with_suffix('.csv')
with path_csv.open('w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
rows = []
for method in all_methods:
rows.append([method])
for kind, (i, metric) in iterprod(all_kinds, enumerate(all_metrics)):
rows.append([
kind if i == 0 else '',
metric,
*all_arrays[MethodKindMetric(method, kind, metric)],
])
rows += [['means'], *make_rows_for_stats(all_means)]
rows += [['stds'], *make_rows_for_stats(all_stds)]
for r in rows:
writer.writerow(r)
# %% plotting functions
def _graph_initialize(means: ndarray, stds: ndarray,
