def _s7_metric(self):
"""
TODO
"""
def _reid_multi(month):
return self._find_k_guess(*self._subset(month))
Df_list = list()
F_list = list()
with PPool(SIZE_POOL) as p:
F_list = p.map(_reid_multi, [i for i in range(13)])
F_list = sorted(F_list, key=lambda x: x['month'], reverse=False)
dtypes = {'id_user': str}
user_id = pd.DataFrame(sorted(self._ground_truth["id_user"].unique()))
user_id.columns = ['id_user']
user_id = user_id.set_index('id_user')
Df_list.append(user_id)
for dict_month in F_list:
col = dict_month.pop("month")
df = pd.DataFrame.from_dict(dict_month, orient='index')
df.columns = [col]
Df_list.append(df)
F_file = pd.concat(Df_list, axis=1, join_axes=[Df_list[0].index])
F_file = F_file.reset_index()
F_file = F_file.fillna('DEL')
score = self.compare_f_files(F_file)
self._current_score.append(score)
return score
def parallelize_simulations(simulation_execs: List[Callable],
var_dict_list: List[VarDictType],
states_lists: List[StatesListsType],
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range], SimIDs, Ns: List[int],
ExpIDs: List[int], SubsetIDs, SubsetWindows,
configured_n):
print(f'Execution Mode: parallelized')
params = list(
zip(simulation_execs, var_dict_list, states_lists, configs_structs,
env_processes_list, Ts, SimIDs, Ns, SubsetIDs, SubsetWindows))
len_configs_structs = len(configs_structs)
unique_runs = Counter(SimIDs)
sim_count = max(unique_runs.values())
highest_divisor = int(len_configs_structs / sim_count)
new_configs_structs, new_params = [], []
for count in range(sim_count):
if count == 0:
new_params.append(params[count:highest_divisor])
new_configs_structs.append(configs_structs[count:highest_divisor])
elif count > 0:
new_params.append(params[count * highest_divisor:(count + 1) *
highest_divisor])
new_configs_structs.append(
configs_structs[count * highest_divisor:(count + 1) *
highest_divisor])
def threaded_executor(params):
tp = TPool()
if len_configs_structs > 1:
results = tp.map(
lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n), params)
else:
t = params[0]
results = t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n)
tp.close()
return results
pp = PPool()
results = flatten(
list(pp.map(lambda params: threaded_executor(params), new_params)))
pp.close()
pp.join()
pp.clear()
# pp.restart()
return results
def alignAllShapes( self ):
start = time.time()
w, W = self.calcWs()
freeze_support()
allShapes = PPool().map( lambda x : self.alignOneShape( x, w ), self.asm.allShapes )
mn1, sd1, mn2,sd2 = self.shapeDist()
with open( os.path.join( self.out, 'log.txt' ), 'a' ) as of:
of.write('point-wise mean:\t%f\n' % mn1 )
of.write('point-wise std:\t%f\n' % sd1 )
of.write('shape-wise mean:\t%f\n'% mn2 )
of.write('shape-wise std:\t%f\n' % sd2 )
of.write( 'alignAllShapes: %f\n' % (time.time() - start ) )
return allShapes
def parallelize_simulations(
simulation_execs: List[Callable],
var_dict_list: List[VarDictType],
states_lists: List[StatesListsType],
configs_structs: List[ConfigsType],
env_processes_list: List[EnvProcessesType],
Ts: List[range],
Ns: List[int]
):
l = list(zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns))
with PPool(len(configs_structs)) as p:
results = p.map(lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6]), l)
return results
def process_executor(params):
if len_configs_structs > 1:
pp = PPool(processes=len_configs_structs)
results = pp.map(
lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n), params)
pp.close()
pp.join()
pp.clear()
else:
t = params[0]
results = t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8],
t[9], configured_n)
return results
def get_pool(self,mpi=False,nthreads=1):
import emcee
from emcee.utils import MPIPool
from pathos.multiprocessing import ProcessingPool as PPool
#from multiprocessing import Pool as PPool
if mpi:
pool = MPIPool(loadbalance=True)
if not pool.is_master():
pool.wait()
sys.exit(0)
self.logger.info('Creating MPI pool with {:d} workers.'.format(pool.size+1))
elif nthreads > 1:
pool = PPool(nthreads)
self.logger.info('Creating multiprocessing pool with {:d} threads.'.format(nthreads))
else:
pool = None
return pool
def scale( self, scaling ):
return PPool().map( lambda line: SimpleShape( PPool().map( lambda x : x.scale( scaling ) , line)), map( lambda y : y.pointList, self.shapeList ) )
def translate( self, translation ):
return map( lambda line: SimpleShape( PPool().map( lambda x : x.scale( translation ) , line)), PPool().map( lambda y : y.pointList, self.shapeList ) )
def alignAllShapes( self ):
start = time.time()
self.calcWs()
self.allShapes = PPool().map( self.alignOneShape, self.allShapes )
print 'alignAllShapes: %f' % (time.time() - start )
return