def setupGlobals():
multiprocess.current_process().authkey = b"176778741"
manager = multiprocess.Manager()
globals_dict = {"OUTPUT_QUEUE": manager.Queue()}
return globals_dict
def compute_feature_extraction(extractor, data):
manager = mp.Manager()
q = manager.Queue(maxsize=8)
process_list = []
pre_compute = mp.Process(target=pre_compute_sillhouttes,
args=(data, q, extractor))
pre_compute.start()
num_processes = 4
for i in range(num_processes):
time.sleep(random.random() * 0.1)
p = mp.Process(
target=listener,
args=(extractor, q, i),
)
process_list.append(p)
p.start()
pre_compute.join()
for i in range(num_processes + 5):
q.put((None, None))
for p in process_list:
p.join()
return True
def run_parallel_async(graph, nprocs=None, sleep=0.2):
if nprocs == 1:
return run_async(graph)
nprocs = nprocs or mp.cpu_count() // 2
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
ioq = mp.Queue(len(graph.funcs.keys()))
cpuq = mp.Queue(len(graph.funcs.keys()))
for _ in range(nprocs):
proc = mp.Process(target=run_scheduler,
args=(graph, sleep, ioq, cpuq))
proc.start()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: queueing task {}'.format(
os.getpid(), task))
if task in graph.io_bound:
ioq.put(task)
else:
cpuq.put(task)
time.sleep(sleep)
return tgraph.recover_values_from_manager(graph)
def __init__(self, isThread=1):
self.threadtool = ThreadTool(isThread)
self.isThread = isThread
# self.threadtool.add_task(self.task)
if isThread == 1:
self.lock = Lock() #线程锁
self.q_request = Queue.Queue() #任务所处理的对象队列
self.q_finish = Queue.Queue() #任务所处理的对象完成队列
else:
self.lock = multiprocessing.Manager().Lock()
self.q_request = multiprocess.Manager().Queue()
self.q_finish = multiprocess.Manager().Queue()
self.running = 0
def run_parallel(graph, nprocs=None, sleep=0.2, raise_errors=False):
nprocs = nprocs or mp.cpu_count() - 1
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
with mp.Pool(nprocs) as pool:
exception_q = mp.Queue(10)
def error_callback(exception):
exception_q.put_nowait(exception)
pool.terminate()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph, reverse=False):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: assigning task {}'.format(
os.getpid(), task))
pool.apply_async(run_task,
args=(graph, task, raise_errors),
error_callback=error_callback)
time.sleep(sleep)
if not exception_q.empty():
raise exception_q.get()
return tgraph.recover_values_from_manager(graph)
def __init__(self,
time_unit=TIME_UNIT,
max_cpu_count=MAX_CPU_COUNT,
sleep_time=SLEEP_TIME,
empty_count_limit=EMPTY_COUNT_LIMIT,
max_sleep_time=MAX_SLEEP_TIME):
super().__init__(time_unit=time_unit)
self._manager = multiprocess.Manager()
# in parent class:
self._to_do_queue = self._manager.Queue()
self._done_queue = self._manager.Queue()
self._tasks_status = self._manager.dict()
self._incomplete_task_ids = self._manager.dict()
# not in parent class:
self._workers_doing = self._manager.dict(
) # key: worker_id, value: task_id
self._worker_reports = self._manager.dict()
self._cpu_usage = self._manager.Value('i', 0)
self._keep_workers_alive = self._manager.Value('i', 1)
self._workers = dict()
self._max_cpu_count = max_cpu_count or self.system_cpu_count
self._sleep_time = sleep_time
self._empty_count_limit = empty_count_limit
self._max_sleep_time = max_sleep_time
atexit.register(self.terminate)
def __init__(self,isThread=1):
self.isThread=isThread
self.Threads=[]
self.idletask={}
# self.running = 0
# def __del__(self): #解构时需等待两个队列完成
# time.sleep(0.5)
# if self.isThread==1:
# self.q_request.join()
# self.q_finish.join()
if self.isThread==1:
# self.lock = Lock() #线程锁
self.work_queue = Queue.Queue()#任务队列
self.lock = Lock() #线程锁
# self.q_request = Queue() #任务所处理的对象队列
# self.q_finish = Queue() #任务所处理的对象完成队列
else :
# self.lock = multiprocessing.Lock()
# self.q_request=multiprocessing.Queue()
# self.q_finish=multiprocessing.Queue()
self.lock = multiprocessing.Manager().Lock()
temp=multiprocess.Manager()
self.work_queue = temp.Queue()#任务队列
def run_parallel_async(graph, nprocs=None, sleep=0.2, raise_errors=False):
if nprocs == 1:
return run_async(graph, sleep=sleep, raise_errors=raise_errors)
nprocs = nprocs or mp.cpu_count() // 2
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
ioq = mp.Queue(len(graph.funcs.keys()))
cpuq = mp.Queue(len(graph.funcs.keys()))
procs = [mp.Process(target=run_scheduler,
args=(graph, sleep, ioq, cpuq, raise_errors))
for _ in range(nprocs)]
for proc in procs:
proc.start()
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info(
'pid {}: queueing task {}'.format(os.getpid(), task))
if task in graph.io_bound:
ioq.put(task)
else:
cpuq.put(task)
time.sleep(sleep)
if raise_errors and sum(not p.exitcode for p in procs):
raise RuntimeError('An async task has failed. Please check your logs')
return tgraph.recover_values_from_manager(graph)
def options_mocks(mocker):
mgr = multiprocess.Manager()
q = mgr.Queue()
def on_subscribe(self, msg, params, websocket):
new_params = copy.deepcopy(params)
new_params.update({'context': msg.get('context', {})})
q.put(self)
return new_params
def on_connect(self, message, websocket):
q.put(self)
def on_disconnect(self, websocket):
q.put(self)
def on_unsubscribe(self, websocket):
q.put(self)
options_mocks = {
'on_subscribe':
PickableMock(side_effect=promisify(on_subscribe), name='on_subscribe'),
'on_unsubscribe':
PickableMock(side_effect=on_unsubscribe, name='on_unsubscribe'),
'on_connect':
PickableMock(return_value={'test': 'test_context'},
side_effect=on_connect,
name='on_connect'),
'on_disconnect':
PickableMock(side_effect=on_disconnect, name='on_disconnect')
}
return options_mocks, q
def main():
initialize()
game = setup_game()
with mp.Manager() as manager:
exc = manager.Queue()
arg_list = []
for i in range(0, 3):
arg_list.append(((i, i + 3), game, False, exc))
arg_list.append(((5, 5), game, True, exc))
proc_list = []
for arg in arg_list:
proc_list.append(mp.Process(target=function, args=arg))
proc_list[-1].start()
print("Number of active children post start: %d" %
len(mp.active_children()))
for p in proc_list:
p.join()
if (not exc.empty()):
e = exc.get()
print(e.message)
print("Number active children post join: %d " % len(mp.active_children()))
print(mp.active_children())
print(mp.current_process())
def __init__(self, task_data={}):
self.id = str(uuid.uuid4())
self.data = task_data
self.process = None
manager = multiprocess.Manager()
self.status_dict = manager.dict()
self.set_status('')
self.logs = manager.list()
self.logger = utilities.create_logger(self.id)
def run(self):
with mb.Manager() as m:
results = m.dict()
p = mb.Process(target=QLWinSingleTest._run_test, args=(self, results))
p.start()
p.join()
if "exception" not in results:
return results['result']
else:
raise RuntimeError(f"\n\nGot an exception during subprocess:\n\n{results['exception']}")
def run(self):
with mb.Manager() as m:
results = m.dict()
p = mb.Process(target=QLWinSingleTest._run_test,
args=(self, results))
p.start()
p.join()
if "exception" not in results:
return results['result']
else:
raise results['exception']
def test():
manager = processing.Manager()
gc.disable()
print('\n\t######## testing Queue.Queue\n')
test_queuespeed(threading.Thread, Queue.Queue(),
threading.Condition())
print('\n\t######## testing processing.Queue\n')
test_queuespeed(processing.Process, processing.Queue(),
processing.Condition())
print('\n\t######## testing Queue managed by server process\n')
test_queuespeed(processing.Process, manager.Queue(),
manager.Condition())
print('\n\t######## testing processing.Pipe\n')
test_pipespeed()
print
print('\n\t######## testing list\n')
test_seqspeed(range(10))
print('\n\t######## testing list managed by server process\n')
test_seqspeed(manager.list(range(10)))
print('\n\t######## testing Array("i", ..., lock=False)\n')
test_seqspeed(processing.Array('i', range(10), lock=False))
print('\n\t######## testing Array("i", ..., lock=True)\n')
test_seqspeed(processing.Array('i', range(10), lock=True))
print()
print('\n\t######## testing threading.Lock\n')
test_lockspeed(threading.Lock())
print('\n\t######## testing threading.RLock\n')
test_lockspeed(threading.RLock())
print('\n\t######## testing processing.Lock\n')
test_lockspeed(processing.Lock())
print('\n\t######## testing processing.RLock\n')
test_lockspeed(processing.RLock())
print('\n\t######## testing lock managed by server process\n')
test_lockspeed(manager.Lock())
print('\n\t######## testing rlock managed by server process\n')
test_lockspeed(manager.RLock())
print()
print('\n\t######## testing threading.Condition\n')
test_conditionspeed(threading.Thread, threading.Condition())
print('\n\t######## testing processing.Condition\n')
test_conditionspeed(processing.Process, processing.Condition())
print('\n\t######## testing condition managed by a server process\n')
test_conditionspeed(processing.Process, manager.Condition())
gc.enable()
def create_jobs(fns, args):
manager = multiprocessing.Manager()
return_dict = manager.dict()
fn_arg = zip(fns, args)
jobs = []
for i, (f, a) in enumerate(fn_arg):
arg = (i, f, a, return_dict)
job = multiprocessing.Process(target=target, args=arg)
jobs.append(job)
return jobs, return_dict
def __init__(self, filename, mode):
logging.Handler.__init__(self)
self.handler = logging.FileHandler(filename, mode)
manager = multiprocess.Manager()
self.queue = manager.Queue(-1)
# self.queue = multiprocess.Queue(-1)
self.is_closed = False
self.t = threading.Thread(target=self.receive)
self.t.daemon = True
self.t.start()
def run_parallel(graph, nprocs=None, sleep=0.2):
nprocs = nprocs or mp.cpu_count() - 1
with mp.Manager() as manager:
graph = tgraph.create_parallel_compatible_graph(graph, manager)
with mp.Pool(nprocs) as pool:
while not tgraph.all_done(graph):
for task in tgraph.get_ready_tasks(graph, reverse=False):
graph = tgraph.mark_as_in_progress(graph, task)
mlog(graph).info('pid {}: assigning task {}'.format(
os.getpid(), task))
pool.apply_async(run_task, args=(graph, task))
time.sleep(sleep)
return tgraph.recover_values_from_manager(graph)
def __init__(self, num_workers, eval_function, timeout=None):
self.num_workers = num_workers
self.eval_function = eval_function
self.timeout = timeout
self.manager = mp.Manager()
self.connection_strings = self.manager.Queue()
for i in range(num_workers):
# Connect to SITL directly wthout mavproxy, can only do one instance
# port = 5760 + i*10
# self.connection_strings.put('tcp:127.0.0.1:' + str(port))
# Connect to mavproxy, uses more rescources, max about 20 instances
port = 14550 + i * 10
self.connection_strings.put('127.0.0.1:' + str(port))
self.pool = mp.Pool(num_workers,
initializer=self.initializer,
initargs=(self.connection_strings, ))
def on_sub_mock(mocker):
mgr = multiprocess.Manager()
q = mgr.Queue()
def on_subscribe(self, msg, params, websocket):
new_params = copy.deepcopy(params)
new_params.update({'context': msg.get('context', {})})
q.put(self)
return new_params
on_sub_mock = {
'on_subscribe':
PickableMock(side_effect=promisify(on_subscribe), name='on_subscribe')
}
return on_sub_mock, q
def kline_data(self, pair_list, interval, **kwargs):
start_date = kwargs.get('start_date', '')
end_date = kwargs.get('end_date', '')
storage = kwargs.get('storage', '')
output = kwargs.get('output', '')
progress_statements = kwargs.get('progress_statements', '')
if start_date:
start_date = datetime.datetime.strptime(start_date, '%m/%d/%Y')
if end_date:
end_date = datetime.datetime.strptime(end_date, '%m/%d/%Y')
valid_kline_intervals = [
'1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h'
]
if interval not in set(valid_kline_intervals):
raise ValueError(
'Invalid Interval: Kline interval should be one of the following - {}'
.format(','.join(valid_kline_intervals)))
output = self.process_kline_output(output)
if not storage:
storage = ['csv', None]
try:
storage_method, intended_dir = storage
except ValueError:
storage_method = storage[0]
intended_dir = None
if progress_statements:
self.progress_statements = progress_statements
if storage_method.lower() == 'csv':
kline_interval_directory = self.create_csv_directories(
pair_list, interval, intended_dir)
csv_file_info = mp.Manager().list()
pair = [currency_pair for i, currency_pair in enumerate(pair_list)]
lock = mp.Lock()
pool = mp.Pool(processes=3, initargs=(lock, ))
# data = pool.starmap(self.kline_to_csv,zip(pair,re(start_date),re(end_date),re(kline_interval_directory),re(interval),re(titles),re(fields),re(csv_file_info)))
data = pool.starmap(
self.kline_to_csv,
zip(pair, re(start_date), re(end_date),
re(kline_interval_directory), re(interval),
re(csv_file_info)))
pool.close()
pool.join()
self.concatenate_csvs(set(list(csv_file_info)))
else:
raise ValueError(
'Invalid Storage Type: Currently only csv storage supported')
def _set_alphas(self):
manager = mp.Manager()
shared_alphas = manager.dict()
shared_test_counts = manager.dict()
self.alphas = dict()
self.test_counts = dict()
test_anno = self.anno.groupby('id')['test_data'].agg(is_test=any)
exclude_test_samples = self.exclude_test_samples
def load_alphas(entry):
data = pd.read_csv(entry[1],
sep="\t",
skip_blank_lines=False,
keep_default_na=False)
samp_names = [re.sub('\.[0-9]+', '', x) for x in data.columns]
if not samp_names:
return entry[0], False
is_test_sample = [
test_anno.loc[x, 'is_test'] if x in test_anno.index else False
for x in samp_names
]
is_test_sample = np.array(is_test_sample)
if exclude_test_samples is True and any(is_test_sample):
shared_alphas[entry[0]] = data.loc[:, ~is_test_sample] + 1
shared_test_counts[entry[0]] = data.loc[:, is_test_sample]
else:
try:
shared_alphas[entry[0]] = data + 1
except Exception as e:
raise Exception(f'Unable to deal with {entry[1]}')
return entry[0], True
tasks = list(self.expression_tsv.items())
with mp.Pool() as pool:
for key, success in tqdm(pool.imap(load_alphas, tasks),
total=len(tasks),
desc='loading counts'):
if success is False:
warnings.warn('No samples found for {}.'.format(key),
RuntimeWarning)
continue
self.alphas[key] = shared_alphas[key]
if key in shared_test_counts.keys():
self.test_counts[key] = shared_test_counts[key]
return self.alphas, self.test_counts
def evaluate_population(self, population, n_cores=0):
manager = multiprocess.Manager()
cache = manager.dict()
eval_fun = lambda ind: self.evaluate(ind, make_copy=True, cache=cache)
if n_cores == 1:
for ind in population:
try:
eval_fun(ind)
except CADETProcessError:
print(ind)
else:
if n_cores == 0:
n_cores = None
with pathos.multiprocessing.ProcessPool(ncpus=n_cores) as pool:
pool.map(eval_fun, population)
return cache
def __init__(
self,
event: mp.synchronize.Event=mp.Event(),
# worker: t.Callable=_worker,
**kargs
):
super(FormalLoader, self).__init__(**kargs)
self.event = event
self.manager = mp.Manager()
self.queue_in, self.queue_out = (
self.manager.Queue(),
self.manager.Queue(self.num_workers * 3)
)
for smiles_block_i in self.smiles_blocks:
self.queue_in.put(smiles_block_i)
# self.worker = worker
self.pool = mp.Pool(self.num_workers)
def create_feat_mtx_parallel(features,
h5_filepath,
gene_map,
is_fixed_input=True,
**kwargs):
"""Create feature matrix for each feautre in parallel
"""
mp_dict = mp.Manager().dict()
mp_jobs = []
## Define parallel jobs
for feat_tuple in features:
mp_job = mp.Process(target=create_feat_mtx_wrapper,
args=(mp_dict, feat_tuple, h5_filepath, gene_map,
is_fixed_input),
kwargs=kwargs)
mp_jobs.append(mp_job)
## Execute parallel jobs
_ = [p.start() for p in mp_jobs]
_ = [p.join() for p in mp_jobs]
return mp_dict
def execute(self, transformer: Callable, upstream: Iterable) -> Generator:
manager = mp.Manager()
manager.register('None', type(None))
input_queue = manager.Queue(maxsize=self.processes)
output_queue = manager.Queue()
collector = TPE(1, 'multiprocessing-executor-collector')
collector.submit(self._collect, upstream, input_queue)
with mp.Pool(self.processes) as pool:
for _ in range(self.processes):
pool.apply_async(MultiProcessingExecutor._work,
args=(input_queue, transformer, output_queue))
iterator = iterate_until_none(output_queue.get, self.processes)
try:
for output in iterator:
yield output
finally:
close_iterator(iterator)
collector.shutdown()
def __init__(self, time_unit='ms'):
self._processes = {}
self._manager = multiprocess.Manager()
self._namespace = self._manager.Namespace()
self._namespace_dir = set()
self._estimators = {}
self._to_do = self._manager.list()
self._doing = self._manager.dict()
self._done = self._manager.list()
self._processed = []
self._errors = []
self._proceed_worker = self._manager.dict()
self._worker_status = self._manager.dict()
self._projects = {}
self._tasks_by_id = {}
self._time_unit = time_unit
self._worker_id_counter = 0
atexit.register(self.terminate)
self._last_error_task = None
def main(output_fn, **opts):
# openpyxl has trouble with relative paths
# convert to absolute path to avoid this issue
abs_output = pathlib.Path(output_fn)
abs_output = abs_output.resolve()
print("Outputting data to", abs_output)
ncores = opts['ncores']
shp_fns = opts['shp_fns']
rst_fns = opts['rst_fns']
field = opts['field']
stats = opts['stats']
ignore = opts['ignore']
preprocess = opts['preprocess']
if ncores > 1:
import multiprocess as mp
from hazardstat import apply_extract
with mp.Manager() as manager:
# `d` is a DictProxy object shared between all processes
# can be converted to dict for final write out
d = manager.dict()
with manager.Pool(processes=ncores) as pool:
# Map each shapefile to a single raster
# Then call apply_extract for each shp->raster combination
file_combs = itools.product(*[[d], shp_fns, rst_fns, [field],
[stats], [ignore], [preprocess]])
procs = pool.starmap_async(apply_extract, file_combs)
procs.get()
results = dict(d)
else:
results = extract_stats(shp_fns, rst_fns, field, stats, ignore,
preprocess)
print("Writing results...")
write_to_excel(abs_output, results)
print("Finished")
def _set_similarity_matrix(self):
"""Calculate the similarity metric using a molecular descriptor
and a similarity measure. Set this attribute.
"""
n_mols = len(self.molecule_database)
similarity_matrix = np.zeros(shape=(n_mols, n_mols))
# Parallel implementation of similarity calculations.
if self.n_threads > 1:
m = multiprocess.Manager()
q = m.Queue()
# worker thread
def worker(thread_idx, n_mols, start_idx, end_idx,
queue): # pragma: no cover
# make a local copy of the overall similarity matrix
local_similarity_matrix = np.zeros(shape=(n_mols, n_mols))
if self.is_verbose:
print(
"thread",
thread_idx,
"will calculate molecules",
start_idx,
"through",
end_idx,
"(",
end_idx - start_idx,
"total)",
)
# same iteration as serial implementation, but only compute
# source molecules in the specified range
for source_mol_id, molecule in enumerate(
self.molecule_database):
if source_mol_id >= start_idx and source_mol_id < end_idx:
for target_mol_id in range(0, n_mols):
if self.is_verbose:
print(
f"thread {thread_idx} computing similarity "
f"of molecule num "
f"{target_mol_id + 1} "
f"against {source_mol_id + 1}")
# diagonal entry
if target_mol_id == source_mol_id:
local_similarity_matrix[source_mol_id,
target_mol_id] = 1
else: # non-diagonal entries
try:
local_similarity_matrix[
source_mol_id,
target_mol_id] = molecule.get_similarity_to(
self.
molecule_database[target_mol_id],
similarity_measure=self.
similarity_measure,
)
except NotInitializedError as e:
e.message += "Similarity matrix could not be set "
raise e
queue.put(local_similarity_matrix)
return None
# calculate work distribution and spawn threads
remainder = n_mols % (self.n_threads)
bulk = n_mols // (self.n_threads)
threads = []
for i in range(int(self.n_threads)):
# last thread
if i == self.n_threads - 1:
thread = multiprocess.Process(
target=worker,
args=(
i,
n_mols,
i * bulk,
bulk * (i + 1) + remainder,
q,
),
)
threads.append(thread)
thread.start()
else:
thread = multiprocess.Process(
target=worker,
args=(
i,
n_mols,
i * bulk,
bulk * (i + 1),
q,
),
)
threads.append(thread)
thread.start()
# retrieve the result and sum all the matrices together.
for thread in threads:
thread.join()
thread_results = []
for _ in range(int(self.n_threads)):
thread_results.append(q.get())
similarity_matrix = sum(thread_results)
else:
# serial implementation
for source_mol_id, molecule in enumerate(self.molecule_database):
for target_mol_id in range(n_mols):
if self.is_verbose:
print(
"Computing similarity of molecule num "
f"{target_mol_id + 1} against {source_mol_id + 1}")
similarity_matrix[
source_mol_id,
target_mol_id] = molecule.get_similarity_to(
self.molecule_database[target_mol_id],
similarity_measure=self.similarity_measure,
)
self.similarity_matrix = similarity_matrix
def get_GPM_swath(self, GPM_file, band='Ku'):
'''
Simulates a GPM swath
Args:
GPM_file: a GPM-DPR file in the HDF5 format
band: can be either 'Ka', 'Ku', 'Ku_matched', 'Ka_matched'
'Ka' will be at the location of the HS (high sensitivity)
coordinates and at 35.6 GHz
'Ka_matched' will be at the location of the MS (matched scan)
coordinates and at 35.6 GHz
'Ku' will be at the location of the NS (high sensitivity)
coordinates and at 13.6 GHz
'Ku_matched' will be at the location of the MS (matched scan)
coordinates and at 13.6 GHz
Returns:
An instance of the SimulatedGPM class (see gpm_wrapper.py) which
contains the simulated radar observables at the coordinates in
the GPM file
'''
# Check if model file has been loaded
if self.dic_vars == {}:
print('No model file has been loaded! Aborting...')
return
# Assign correct frequencies and 3dB beamwidth, whatever user config
cfg_copy = copy.deepcopy(self.config)
if band == 'Ku' or band == 'Ku_matched':
cfg_copy['radar']['frequency'] = constants.GPM_KU_FREQUENCY
elif band == 'Ka' or band == 'Ka_matched':
cfg_copy['radar']['frequency'] = constants.GPM_KA_FREQUENCY
cfg_copy['radar']['3dB_beamwidth'] = constants.GPM_3DB_BEAMWIDTH
cfg_copy['radar']['sensitivity'] = constants.GPM_SENSITIVITY
cfg_copy['radar']['type'] = 'GPM'
cfg_copy['radar']['radial_resolution'] = \
constants.GPM_RADIAL_RES_KA if band == 'Ka' \
else constants.GPM_RADIAL_RES_KU
self.update_config(cfg_copy, check=False)
# Needs to be done in order to deal with Multiprocessing's annoying limitations
global dic_vars, N, lut_sz, output_variables
dic_vars, N, lut_sz, output_variables = self.define_globals()
az, elev, rang, coords_GPM = get_GPM_angles(GPM_file, band)
# Initialize computing pool
pool = mp.Pool(processes=mp.cpu_count(), maxtasksperchild=1)
m = mp.Manager()
event = m.Event()
def worker(event, params):
try:
if not event.is_set():
azimuth = params[0]
elev = params[1]
""" For some reason modifying self.config instead of
cfg.CONFIG throws in error about not being able to pickle
the pycosmo variables. This is indeed very weird and I
have not been able to figure out why...However since
self.config is just a shallow copy of cfg.CONFIG, it doesn't
really matter...
"""
# Update GPM position and range vector
cfg.CONFIG['radar']['range'] = params[2]
cfg.CONFIG['radar']['coords'] = [
params[3], params[4], params[5]
]
list_subradials = get_interpolated_radial(dic_vars,
azimuth,
elev,
N=N)
output = get_radar_observables(list_subradials, lut_sz)
if output_variables in ['all', 'only_radar']:
output = get_radar_observables(list_subradials, lut_sz)
if output_variables == 'only_model':
output = integrate_radials(list_subradials)
elif output_variables == 'all':
output = combine_subradials(
(output, integrate_radials(list_subradials)))
return output
except:
# Throw signal back
raise
event.set()
dim_swath = az.shape
list_beams = []
for i in range(dim_swath[0]):
print('running slice ' + str(i))
# Update radar position
c0 = np.repeat(coords_GPM[i, 0], len(az[i]))
c1 = np.repeat(coords_GPM[i, 1], len(az[i]))
c2 = np.repeat(coords_GPM[i, 2], len(az[i]))
worker_partial = partial(worker, event)
list_beams.extend(
map(worker_partial, zip(az[i], elev[i], rang[i], c0, c1, c2)))
pool.close()
pool.join()
del dic_vars
del N
del lut_sz
gc.collect()
if not event.is_set():
# Threshold at given sensitivity
if output_variables in ['all', 'only_radar']:
list_beams = cut_at_sensitivity(list_beams)
list_beams_formatted = SimulatedGPM(list_beams, dim_swath, band)
return list_beams_formatted
#midi_fps = glob.glob('./lakh/lakh_clean_midi/*/*.mid*')
midi_fps = glob.glob('test_in/*.mid*')
out_dir = './test_out'
if os.path.isdir(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
def _task(x):
emit_nesmdb_midi_examples(x, out_dir)
drops = mp.Manager().dict({
'largeFile': 0,
'badMidiDecode': 0,
'too long or short': 0,
'negative times': 0,
'not enough instruments': 0,
'not enough instruments after range drop': 0,
'not enough instruments after polyphonic drop': 0,
'not enough instruments after duplicate drop': 0,
'': 0,
'': 0,
'': 0,
'': 0,
'': 0
})
with mp.Pool(8, initargs=(drops, )) as p:
r = list(tqdm(p.imap(_task, midi_fps), total=len(midi_fps)))
