def plot_cats_vars_errors(dbs_dir, err_var_labs, n_cpus):
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
n_cpus = min(n_cats, n_cpus)
cats_paths_gen = ((cat_db, err_var_labs) for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_vars_errors, cats_paths_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for cat_paths in cats_paths_gen:
plot_cat_vars_errors(cat_paths)
return
def plot_cats_hbv_sim(dbs_dir,
water_bal_step_size,
full_flag=False,
wat_bal_flag=False,
show_warm_up_steps_flag=False,
n_cpus=1):
'''Plot hbv simulations for every catchment for every kfold.'''
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
const_args = (water_bal_step_size, full_flag, wat_bal_flag,
show_warm_up_steps_flag)
plot_gen = ((cat_db, const_args) for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_hbv_sim, plot_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for plot_args in plot_gen:
plot_cat_hbv_sim(plot_args)
return
def plot_cats_qsims(dbs_dir, n_cpus=1):
'''Plot discharge simulations for every catchment for every
kfold using its prm_vecs.'''
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
plot_gen = (cat_db for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_qsims, plot_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for plot_args in plot_gen:
plot_cat_qsims(plot_args)
return
def plot_cats_prm_vecs(dbs_dir, n_cpus):
'''Plot final parameter set from kfold for every catchments along with
objective function value distribution.
'''
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
n_cpus = min(n_cats, n_cpus)
opt_res_gen = (cat_db for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_prm_vecs, opt_res_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for opt_res in opt_res_gen:
plot_cat_prm_vecs(opt_res)
return
def plot_cats_kfold_effs(dbs_dir, hgs_db_path, compare_ann_cyc_flag, n_cpus):
'''Plot the k-fold efficiency results.'''
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
n_cpus = min(n_cats, n_cpus)
const_args = (compare_ann_cyc_flag, hgs_db_path)
cats_paths_gen = ((cat_db, const_args) for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_kfold_effs, cats_paths_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for cat_paths in cats_paths_gen:
plot_cat_kfold_effs(cat_paths)
return
def plot_cats_best_prms_1d(dbs_dir, n_cpus):
'''Plot every best kfold parameter set for all catchments.'''
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
cats_paths_gen = (cat_db for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_best_prms_1d, cats_paths_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for cat_paths in cats_paths_gen:
plot_cat_best_prms_1d(cat_paths)
return
def normalize(self) -> None:
""" Normalize ensemble """
if not self.regenerate:
try:
self.load()
return
except FileNotFoundError:
pass
assert xor((self.normalizer_nld is not None
and self.normalizer_gsf is not None),
self.normalizer_simultan is not None), \
"Either 'normalizer_nld' and 'normalizer_gsf' must be set, or " \
"normalizer_simultan"
gsfs = self.extractor.gsf
nlds = self.extractor.nld
self.LOG.info(f"Start normalization with {self.nprocesses} cpus")
pool = ProcessPool(nodes=self.nprocesses)
N = len(nlds)
iterator = pool.imap(self.step, range(N), nlds, gsfs)
self.res = list(tqdm(iterator, total=N))
pool.close()
pool.join()
pool.clear()
self.save()
def _prep_anomaly_bjs_mp(anoms_arr, bjs_arr, n_cpus, fig_out_dir):
assert anoms_arr.shape == bjs_arr.shape
_idxs = ret_mp_idxs(anoms_arr.shape[1], n_cpus)
_idxs_list = [_idxs[i:i + 2] for i in range(n_cpus)]
_anoms_gen = ((anoms_arr[:, _idxs_list[i][0]:_idxs_list[i][1]])
for i in range(n_cpus))
_bjs_gen = ((bjs_arr[:, _idxs_list[i][0]:_idxs_list[i][1]])
for i in range(n_cpus))
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
try:
print(
list(
mp_pool.uimap(Anomaly._plot_anomaly_bjs_cdf, _idxs_list,
_anoms_gen, _bjs_gen,
[fig_out_dir] * n_cpus)))
mp_pool.clear()
except Exception as msg:
mp_pool.close()
mp_pool.join()
print('Error in _plot_anomaly_bjs_cdf:', msg)
return
def correction_factor(p, number_of_runs, method, X, y, n_jobs=None):
# Setup parallel job
if n_jobs == -1:
n_jobs = cpu_count()
elif n_jobs == None:
n_jobs = 1
pool = Pool(n_jobs, maxtasksperchild=1000)
def run(_):
# Artificially falsify
y_f = falsify(y, p, random_state=_)
# Correct labels
y_corrected = method.fit_transform(X, y_f)
N = X.shape[0]
return ((y == y_corrected).sum() - (1 - p) * N) / (p * N)
factor = np.array(pool.map(run, range(number_of_runs)))
# Close the pool again
pool.close()
pool.join()
pool.clear()
return np.mean(factor), np.std(factor)
def _parallel(ordered: bool, function: Callable, *iterables: Iterable, **kwargs: Any) -> Generator:
"""Returns a generator for a parallel map with a progress bar.
Arguments:
ordered(bool): True for an ordered map, false for an unordered map.
function(Callable): The function to apply to each element of the given Iterables.
iterables(Tuple[Iterable]): One or more Iterables containing the data to be mapped.
Returns:
A generator which will apply the function to each element of the given Iterables
in parallel in order with a progress bar.
"""
# Extract num_cpus
num_cpus = kwargs.pop('num_cpus', None)
# Determine num_cpus
if num_cpus is None:
num_cpus = cpu_count()
elif type(num_cpus) == float:
num_cpus = int(round(num_cpus * cpu_count()))
# Determine length of tqdm (equal to length of shortest iterable)
length = min(len(iterable) for iterable in iterables if isinstance(iterable, Sized))
# Create parallel generator
map_type = 'imap' if ordered else 'uimap'
pool = Pool(num_cpus)
map_func = getattr(pool, map_type)
for item in tqdm(map_func(function, *iterables), total=length, **kwargs):
yield item
pool.clear()
def Scrap_landing(allowed_domains, start_urls):
pool = ProcessPool(nodes=4)
def f_runner(spider):
ScrapperSpider.allowed_domains = [allowed_domains]
ScrapperSpider.start_urls = [start_urls]
from twisted.internet import reactor
from scrapy.settings import Settings
import Scrapper.settings as my_settings
from scrapy.crawler import CrawlerProcess, CrawlerRunner
crawler_settings = Settings()
crawler_settings.setmodule(my_settings)
runner = CrawlerRunner(settings=crawler_settings)
deferred = runner.crawl(spider)
deferred.addBoth(lambda _: reactor.stop())
reactor.run()
#ScrapperSpider.allowed_domains = [allowed_domains]
#ScrapperSpider.start_urls = [start_urls]
#print("\nstart URLS:{}".format(ScrapperSpider.start_urls))
results = pool.amap(f_runner, [ScrapperSpider])
t = 0
while not results.ready():
time.sleep(5);
print(".", end=' ');
t = t + 5
if t == 30:
print("\nProcess limited to 30 seconds...EXITING\n");
return None
pool.clear()
def _parallel_final_score(self, smiles: List[str]) -> FinalSummary:
molecules, valid_indices = self._smiles_to_mols(smiles)
component_smiles_pairs = [[
component, molecules, valid_indices, smiles
] for component in self.scoring_components]
pool = ProcessPool(nodes=len(self.scoring_components))
mapped_pool = pool.map(parallel_run, component_smiles_pairs)
pool.clear()
return self._score_summary(mapped_pool, smiles, valid_indices)
def _prepare_eigen(self):
"""
calculate eigen values and vectors for all kpts and save.
Note that the convention 2 is used here, where the
phase factor is e^(ik.R), not e^(ik.(R+rj-ri))
"""
nkpts = len(self.kpts)
self.evals = np.zeros((nkpts, self.nbasis), dtype=float)
self.nkpts = nkpts
self.H0 = np.zeros((self.nbasis, self.nbasis), dtype=complex)
self.evecs = np.zeros((nkpts, self.nbasis, self.nbasis), dtype=complex)
H = np.zeros((nkpts, self.nbasis, self.nbasis), dtype=complex)
if not self.is_orthogonal:
self.S = np.zeros((nkpts, self.nbasis, self.nbasis), dtype=complex)
else:
self.S = None
if self.nproc == 1:
results = map(self.tbmodel.HSE_k, self.kpts)
else:
executor = ProcessPool(nodes=self.nproc)
results = executor.map(self.tbmodel.HSE_k, self.kpts,
[2] * len(self.kpts))
executor.close()
executor.join()
executor.clear()
for ik, result in enumerate(results):
if self.is_orthogonal:
H[ik], _, self.evals[ik], self.evecs[ik] = result
else:
H[ik], self.S[ik], self.evals[ik], self.evecs[ik] = result
self.H0 += H[ik] / self.nkpts
self.evals, self.evecs = self._reduce_eigens(self.evals,
self.evecs,
emin=self.efermi - 10.0,
emax=self.efermi + 10.1)
if self._use_cache:
evecs = self.evecs
self.evecs_shape = self.evecs.shape
self.evecs = np.memmap(os.path.join(self.cache_path, 'evecs.dat'),
mode='w+',
shape=self.evecs.shape,
dtype=complex)
self.evecs[:, :, :] = evecs[:, :, :]
if self.is_orthogonal:
self.S = None
else:
S = self.S
self.S = np.memmap(os.path.join(self.cache_path, 'S.dat'),
mode='w+',
shape=(nkpts, self.nbasis, self.nbasis),
dtype=complex)
self.S[:] = S[:]
del self.evecs
if not self.is_orthogonal:
del self.S
def runIteration(self, task, pop, fpop, xb, fxb, A, A_f, B, B_f, D, D_f,
**dparams):
r"""Core funciton of GreyWolfOptimizer algorithm.
Args:
task (Task): Optimization task.
pop (numpy.ndarray): Current population.
fpop (numpy.ndarray): Current populations function/fitness values.
xb (numpy.ndarray):
fxb (float):
A (numpy.ndarray):
A_f (float):
B (numpy.ndarray):
B_f (float):
D (numpy.ndarray):
D_f (float):
**dparams (Dict[str, Any]): Additional arguments.
Returns:
Tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray, float, Dict[str, Any]]:
1. New population
2. New population fitness/function values
3. Additional arguments:
* A (): TODO
"""
def eval_task(args):
i, w = args
A1, C1 = 2 * a * self.rand(task.D) - a, 2 * self.rand(task.D)
X1 = A - A1 * fabs(C1 * A - w)
A2, C2 = 2 * a * self.rand(task.D) - a, 2 * self.rand(task.D)
X2 = B - A2 * fabs(C2 * B - w)
A3, C3 = 2 * a * self.rand(task.D) - a, 2 * self.rand(task.D)
X3 = D - A3 * fabs(C3 * D - w)
pop = task.repair((X1 + X2 + X3) / 3, self.Rand)
fpop = task.eval(pop[i])
return i, pop, fpop
a = 2 - task.Evals * (2 / task.nFES)
pool = ProcessPool(nodes=self.nodes)
pool.clear()
results = pool.map(eval_task, [[i, w] for i, w in enumerate(pop)])
for i, _pop, _fpop in results:
pop[i] = _pop
fpop[i] = _fpop
for i, f in enumerate(fpop):
if f < A_f: A, A_f = pop[i].copy(), f
elif A_f < f < B_f: B, B_f = pop[i].copy(), f
elif B_f < f < D_f: D, D_f = pop[i].copy(), f
xb, fxb = self.getBest(A, A_f, xb, fxb)
return pop, fpop, xb, fxb, {
'A': A,
'A_f': A_f,
'B': B,
'B_f': B_f,
'D': D,
'D_f': D_f
}
def sample(self, horizon, act=None, nodes=8):
act = [None] * len(horizon) if act is None else act
seeds = [i for i in range(len(horizon))]
pool = ProcessPool(nodes=nodes)
res = pool.map(self._sample, horizon, act, seeds)
pool.clear()
state, obs = list(map(list, zip(*res)))
return state, obs
def accuracy(p, number_of_runs, method, X, y, clf = None, n_jobs = None):
'''
Returns: (array{mean_acc_corrected, mean_acc_false}, array{stddev_acc_corrected, stddev_acc_false})
'''
# Setup parallel job
if n_jobs == -1:
n_jobs = cpu_count()
elif n_jobs == None:
n_jobs = 1
pool = Pool(n_jobs, maxtasksperchild = 1000)
if clf is None:
clf = NNC(n_neighbors= 1)
def run(_):
# Artificially falsify
y_f = falsify(y, p, random_state = _)
# Correct labels
y_corrected = method.fit_transform(X, y_f)
# Set up 10-fold-Cross validation
train_corr, test_corr = kfold(10, X, y, y_corrected)
train_f, test_f = kfold(10, X, y, y_f)
score = np.zeros((2, 10))
# Calc scores
for fold in range(10):
train_X, train_y = train_corr[fold]
test_X, test_y = test_corr[fold]
clf.fit(train_X, train_y)
score[0, fold] = clf.score(test_X, test_y)
train_X, train_y = train_f[fold]
test_X, test_y = test_f[fold]
clf.fit(train_X, train_y)
score[1, fold] = clf.score(test_X, test_y)
# Average
return np.mean(score, axis = 1).tolist()
acc = np.array(pool.map(run, range(number_of_runs)))
# Close the pool again
pool.close()
pool.join()
pool.clear()
return np.mean(acc, axis = 0), np.std(acc, axis = 0)
def parallelize_simulations(simulation_execs: List[Callable],
var_dict_list: List[VarDictType],
states_lists: List[StatesListsType],
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range], SimIDs, Ns: List[int],
ExpIDs: List[int], SubsetIDs, SubsetWindows,
configured_n):
print(f'Execution Mode: parallelized')
params = list(
zip(simulation_execs, var_dict_list, states_lists, configs_structs,
env_processes_list, Ts, SimIDs, Ns, SubsetIDs, SubsetWindows))
len_configs_structs = len(configs_structs)
unique_runs = Counter(SimIDs)
sim_count = max(unique_runs.values())
highest_divisor = int(len_configs_structs / sim_count)
new_configs_structs, new_params = [], []
for count in range(sim_count):
if count == 0:
new_params.append(params[count:highest_divisor])
new_configs_structs.append(configs_structs[count:highest_divisor])
elif count > 0:
new_params.append(params[count * highest_divisor:(count + 1) *
highest_divisor])
new_configs_structs.append(
configs_structs[count * highest_divisor:(count + 1) *
highest_divisor])
def threaded_executor(params):
tp = TPool()
if len_configs_structs > 1:
results = tp.map(
lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n), params)
else:
t = params[0]
results = t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n)
tp.close()
return results
pp = PPool()
results = flatten(
list(pp.map(lambda params: threaded_executor(params), new_params)))
pp.close()
pp.join()
pp.clear()
# pp.restart()
return results
def MyProcessPool(nodes=None):
if nodes is None or nodes > 1:
p = ProcessPool(nodes)
try:
yield p
finally:
p.close()
p.join()
p.clear()
else:
#print("Using PseudoPool!")
yield PseudoPool()
def calculate_all(self):
"""
The top level.
"""
print("Green's function Calculation started.")
widgets = [
' [',
progressbar.Timer(),
'] ',
progressbar.Bar(),
' (',
progressbar.ETA(),
') ',
]
bar = progressbar.ProgressBar(maxval=self.contour.npoints,
widgets=widgets)
bar.start()
rhoRs = []
GRs = []
AijRs = {}
if self.np > 1:
executor = ProcessPool(nodes=self.np)
results = executor.map(self.get_AijR_rhoR, self.contour.path)
else:
results = map(self.get_AijR_rhoR, self.contour.path)
for i, result in enumerate(results):
bar.update(i)
for iR, R in enumerate(self.R_ijatom_dict):
for (iatom, jatom) in self.R_ijatom_dict[R]:
if (R, iatom, jatom) in AijRs:
AijRs[(R, iatom, jatom)].append(result[0][R, iatom,
jatom])
else:
AijRs[(R, iatom, jatom)] = []
AijRs[(R, iatom, jatom)].append(result[0][R, iatom,
jatom])
rhoRs.append(result[1])
if self.np > 1:
executor.close()
executor.join()
executor.clear()
#self.save_AijRs(AijRs)
self.integrate(rhoRs, AijRs)
self.get_rho_atom()
self.A_to_Jtensor()
bar.finish()
def process_executor(params):
if len_configs_structs > 1:
pp = PPool(processes=len_configs_structs)
results = pp.map(
lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n), params)
pp.close()
pp.join()
pp.clear()
else:
t = params[0]
results = t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n)
return results
def call_processes(url, path, pages):
urls = create_urls(50, int(pages), url)
# I want a better way to do this
paths = []
for _ in range(len(urls)):
paths.append(path)
start = time.time()
p = Pool(multiprocessing.cpu_count() - 1)
result = p.map(process_page, urls, paths)
end = time.time()
print('Full Run: ', end - start)
p.clear()
def multiprocessing_imap(
func: Callable,
iterable: Iterable[Any],
*,
n_workers: int = -1,
progressbar: bool = True,
args=(),
**kwargs
) -> List[Any]:
"""Execute func on each element in iterable on n_workers, ensuring order.
Args:
func: Function to apply to each element in iterable.
iterable: Input iterable on which to execute func.
n_workers: Amount of workers (processes) to spawn.
progressbar: Whether to wrap the chunks in a tqdm.auto.tqdm.
args: Additional positional arguments to pass to func.
kwargs: Additional keyword arguments to pass to func.
Returns:
Results in same order as input iterable.
"""
iterable = list(iterable) # exhaust if iterable is a generator
n_chunks = len(iterable)
func = partial(func, *args, **kwargs)
if n_chunks == 1 or n_workers == 1:
# no sense spawning pool
pool = None
stage = map(func, iterable)
else:
n_workers = _choose_n_workers(n_chunks, n_workers)
logger.debug("Starting ProcessPool with %d workers", n_workers)
pool = ProcessPool(n_workers)
stage = pool.imap(func, iterable)
if progressbar:
stage = tqdm(stage, total=n_chunks)
try:
return list(stage)
finally:
if pool:
logger.debug("Closing ProcessPool")
pool.clear()
def apply_data(self, data: 'Data', method: Callable) -> 'Data':
"""Applies 'method' to 'data' across several cores.
Args:
data ('Data'): instance with a stored pandas DataFrame.
method (Callable): callable method or function to apply to 'data'.
Returns:
'Data': with 'method' applied.
"""
dfs = np.array_split(data.data, mp.cpu_count(), axis=0)
pool = Pool()
data.data = np.vstack(pool.map(method, dfs))
pool.close()
pool.join()
pool.clear()
return data
def avaliacao(self, populacao):
n = len(populacao)
def steps(k):
individuo = populacao[k, :]
obj = self.funcao_objetivo(individuo)
return obj
ncpu = cpu_count()
pool = ProcessPool(nodes=ncpu)
pesos = array(pool.map(steps, range(n)))
pool.close()
pool.join()
pool.clear()
shutdown()
return pesos
def parallel_map(f, X, nof_process):
""" Runs a map of X to f in parallel.
Parameters
f: a callable object to which X should be applied.
X: a list of objects to which apply f.
nof_process: a integer representing the number of process that
should be spawned to calculate f(X)
Returns
results: a list containing the resulting application of f to
every element in X.
"""
pool = ProcessPool(nof_process)
pool.clear() # why would you clear something you just created?
# anser: pathos.multiprocessing caches worker pools. (yeahhh...
# maybe this should be a class) see issue #49
results = pool.map(f, X)
return results
def calculate_all(self):
"""
The top level.
"""
print("Green's function Calculation started.")
widgets = [
' [',
progressbar.Timer(),
'] ',
progressbar.Bar(),
' (',
progressbar.ETA(),
') ',
]
bar = progressbar.ProgressBar(maxval=len(self.contour.path),
widgets=widgets)
bar.start()
if self.np == 1:
results = map(self.get_AijR_rhoR, self.contour.path)
else:
pool = ProcessPool(nodes=self.np)
results = pool.map(self.get_AijR_rhoR, self.contour.path)
for i, result in enumerate(results):
bar.update(i)
rup, rdn, Jorb_list, JJ_list = result
self.rho_up_list.append(rup)
self.rho_dn_list.append(rdn)
for iR, R in enumerate(self.R_ijatom_dict):
for (iatom, jatom) in self.R_ijatom_dict[R]:
key = (R, iatom, jatom)
self.Jorb_list[key].append(Jorb_list[key])
self.JJ_list[key].append(JJ_list[key])
if self.np > 1:
pool.close()
pool.join()
pool.clear()
self.integrate()
self.get_rho_atom()
self.A_to_Jtensor()
bar.finish()
def compute_seq_distances(sequences, affinity=sequence_distance,
nb_jobs=NB_THREADS):
""" compute matrix of all distances
:param [] sequences: list of all sequences
:param func affinity: function specify the sample affinity
:param int nb_jobs: number jobs running in parallel
:return ndarray:
>>> ss = [['a', 'b', 'a', 'c'], ['a', 'a', 'b', 'a'], ['b', None, 'b', 'a']]
>>> compute_seq_distances(ss, affinity=sequence_distance)
array([[0. , 0.25, 0.5 ],
[0.25, 0. , 0.25],
[0.5 , 0.25, 0. ]])
>>> ss = [['hi', 'there', 'how', 'are', 'you'],
... ['hi', 'how', 'are', 'you'],
... ['hi', 'are', 'you', 'there']]
>>> compute_seq_distances(ss)
array([[0. , 0.2, 0.6],
[0.2, 0. , 0.5],
[0.6, 0.5, 0. ]])
"""
idxs = [(i, j) for i in range(len(sequences))
for j in range(i, len(sequences))]
idx_lt = (((i, j), (sequences[i], sequences[j])) for i, j in idxs)
dists = np.zeros((len(sequences), len(sequences)))
_wrap_dist = partial(wrap_distance, similar_distance=affinity)
pool = ProcessPool(nb_jobs)
for idx, d in pool.imap(_wrap_dist, idx_lt):
dists[idx[0], idx[1]] = d
dists[idx[1], idx[0]] = d
pool.close()
pool.join()
pool.clear()
return dists
def plot_cats_prm_vecs_evo(dbs_dir,
save_obj_flag,
save_png_flag,
save_gif_flag,
anim_secs,
n_cpus=1):
'''Plot the evolution of parameter vectors and convex hull for every
catchment for every kfold.
'''
cats_dbs = glob(os.path.join(dbs_dir, 'cat_*.hdf5'))
assert cats_dbs
n_cats = len(cats_dbs)
n_cpus = min(n_cats, n_cpus)
n_cpus = min(n_cats, n_cpus)
opt_res_gen = ((cat_db, save_obj_flag, save_png_flag, save_gif_flag,
anim_secs) for cat_db in cats_dbs)
if (n_cpus > 1) and (n_cats > 1):
mp_pool = ProcessPool(n_cpus)
mp_pool.restart(True)
print(list(mp_pool.uimap(plot_cat_prm_vecs_evo, opt_res_gen)))
mp_pool.clear()
mp_pool.close()
mp_pool.join()
else:
for opt_res in opt_res_gen:
plot_cat_prm_vecs_evo(opt_res)
return
def data_generator(annotation_lines,
input_shape,
anchors,
nb_classes,
batch_size=1,
augment=True,
max_boxes=20,
jitter=0.3,
img_scaling=1.2,
resize_img=True,
allow_rnd_shift=True,
color_hue=0.1,
color_sat=1.5,
color_val=1.5,
flip_horizontal=True,
flip_vertical=False,
bbox_overlap=0.95,
nb_threads=1):
"""data generator for fit_generator
:param list(str) annotation_lines:
:param int batch_size:
:param ndarray anchors:
:param int nb_classes:
:param tuple(int,int) input_shape: CNN input size
:param bool augment: perform augmentation
:param int max_boxes: maximal number of training bounding boxes
:param float jitter:
:param float color_hue: range of change of HSV color HUE
:param float color_sat: range of change of HSV color SAT
:param float color_val: range of change of HSV color value
:param float img_scaling: upper image scaling
:param bool flip_horizontal: allow random flop image/boxes vertical
:param bool flip_vertical: allow random flop image/boxes horizontal
:param bool resize_img: resize image to fit fully to CNN
:param bool allow_rnd_shift: allow shifting image not only centered crop
:param float bbox_overlap: threshold in case cut image, drop all boxes with lower overlap
:param float|int nb_threads: nb threads running in parallel
:return:
>>> np.random.seed(0)
>>> path_img = os.path.join(update_path('model_data'), 'bike-car-dog.jpg')
>>> line = path_img + ' 100,150,200,250,0 300,50,400,200,1'
>>> anchors = get_anchors(os.path.join(update_path('model_data'), 'yolo_anchors.csv'))
>>> gen = data_generator([line], (416, 416), anchors, 3, nb_threads=2)
>>> batch = next(gen)
>>> len(batch)
2
>>> [b.shape for b in batch[0]]
[(1, 416, 416, 3), (1, 13, 13, 3, 8), (1, 26, 26, 3, 8), (1, 52, 52, 3, 8)]
>>> gen = data_generator([line], (416, 416), anchors, 3, augment=False)
>>> batch = next(gen)
>>> len(batch)
2
>>> [b.shape for b in batch[0]]
[(1, 416, 416, 3), (1, 13, 13, 3, 8), (1, 26, 26, 3, 8), (1, 52, 52, 3, 8)]
"""
nb_lines = len(annotation_lines)
circ_i = 0
if nb_lines == 0 or batch_size <= 0:
return None
color_hue = abs(color_hue)
color_sat = color_sat if color_sat > 1 else 1. / color_sat
color_val = color_val if color_val > 1 else 1. / color_val
nb_threads = nb_workers(nb_threads)
pool = ProcessPool(nb_threads) if nb_threads > 1 else None
_wrap_rand_data = partial(
get_augmented_data,
input_shape=input_shape,
augment=augment,
max_boxes=max_boxes,
jitter=jitter,
resize_img=resize_img,
img_scaling=img_scaling,
allow_rnd_shift=allow_rnd_shift,
hue=color_hue,
sat=color_sat,
val=color_val,
flip_horizontal=flip_horizontal,
flip_vertical=flip_vertical,
bbox_overlap=bbox_overlap,
)
while True:
if circ_i < batch_size:
# shuffle while you are starting new cycle
np.random.shuffle(annotation_lines)
batch_image_data = []
batch_box_data = []
# create the list of lines to be loaded in batch
annot_lines = annotation_lines[circ_i:circ_i + batch_size]
batch_offset = (circ_i + batch_size) - nb_lines
# chekck if the loaded batch size have sufficient size
if batch_offset > 0:
annot_lines += annotation_lines[:batch_offset]
# multiprocessing loading of batch data
map_process = pool.imap if pool else map
for image, box in map_process(_wrap_rand_data, annot_lines):
batch_image_data.append(image)
batch_box_data.append(box)
circ_i = (circ_i + batch_size) % nb_lines
batch_image_data = np.array(batch_image_data)
batch_box_data = np.array(batch_box_data)
y_true = preprocess_true_boxes(batch_box_data, input_shape, anchors,
nb_classes)
batch = [batch_image_data, *y_true], np.zeros(batch_size)
yield batch
gc.collect()
if pool:
pool.close()
pool.join()
pool.clear()
def recursive(self):
"""
Recursively computes a series of interpolating instances
Generates a dictionary with the emax instances
There is a sequence of Emax for each child age (1-10)
"""
def emax_gen(j):
for t in range(j,0,-1):
if t==j:#last period
emax_bigt_ins=self.emax_bigt(j)
emax_dic={'emax'+str(t): emax_bigt_ins[0]}
elif t==j-1: #at T-1
emax_t1_ins=self.emax_t(t,j,emax_bigt_ins[0])
emax_dic['emax'+str(t)]=emax_t1_ins[0]
else:
emax_t1_ins=self.emax_t(t,j,emax_t1_ins[0])
emax_dic['emax'+str(t)]=emax_t1_ins[0]
return [emax_dic]
pool = ProcessPool(nodes = 18)
#7: old child (11 years old) solves for 7 emax
#19: young child (0 years old) solves for 18 emax
list_emax = pool.map(emax_gen,range(8,18))
pool.close()
pool.join()
pool.clear()
"""
list_emax = []
for j in range(7,19):
print ('Im in emax j ', j)
for t in range(j,0,-1):
print ('In period t ', t)
if t==j:#last period
emax_bigt_ins=self.emax_bigt(j)
emax_dic={'emax'+str(t): emax_bigt_ins[0]}
#emax_values={'emax'+str(t): emax_bigt_ins[1]}
elif t==j-1: #at T-1
emax_t1_ins=self.emax_t(t,j,emax_bigt_ins[0])
emax_dic['emax'+str(t)]=emax_t1_ins[0]
#emax_values['emax'+str(t)]=emax_t1_ins[1]
else:
emax_t1_ins=self.emax_t(t,j,emax_t1_ins[0])
emax_dic['emax'+str(t)]=emax_t1_ins[0]
#emax_values['emax'+str(t)]=emax_t1_ins[1]
list_emax.append([emax_dic])
"""
return list_emax
