def setupGlobals(): multiprocess.current_process().authkey = b"176778741" manager = multiprocess.Manager() globals_dict = {"OUTPUT_QUEUE": manager.Queue()} return globals_dict
def compute_feature_extraction(extractor, data): manager = mp.Manager() q = manager.Queue(maxsize=8) process_list = [] pre_compute = mp.Process(target=pre_compute_sillhouttes, args=(data, q, extractor)) pre_compute.start() num_processes = 4 for i in range(num_processes): time.sleep(random.random() * 0.1) p = mp.Process( target=listener, args=(extractor, q, i), ) process_list.append(p) p.start() pre_compute.join() for i in range(num_processes + 5): q.put((None, None)) for p in process_list: p.join() return True
def run_parallel_async(graph, nprocs=None, sleep=0.2): if nprocs == 1: return run_async(graph) nprocs = nprocs or mp.cpu_count() // 2 with mp.Manager() as manager: graph = tgraph.create_parallel_compatible_graph(graph, manager) ioq = mp.Queue(len(graph.funcs.keys())) cpuq = mp.Queue(len(graph.funcs.keys())) for _ in range(nprocs): proc = mp.Process(target=run_scheduler, args=(graph, sleep, ioq, cpuq)) proc.start() while not tgraph.all_done(graph): for task in tgraph.get_ready_tasks(graph): graph = tgraph.mark_as_in_progress(graph, task) mlog(graph).info('pid {}: queueing task {}'.format( os.getpid(), task)) if task in graph.io_bound: ioq.put(task) else: cpuq.put(task) time.sleep(sleep) return tgraph.recover_values_from_manager(graph)
def __init__(self, isThread=1): self.threadtool = ThreadTool(isThread) self.isThread = isThread # self.threadtool.add_task(self.task) if isThread == 1: self.lock = Lock() #线程锁 self.q_request = Queue.Queue() #任务所处理的对象队列 self.q_finish = Queue.Queue() #任务所处理的对象完成队列 else: self.lock = multiprocessing.Manager().Lock() self.q_request = multiprocess.Manager().Queue() self.q_finish = multiprocess.Manager().Queue() self.running = 0
def run_parallel(graph, nprocs=None, sleep=0.2, raise_errors=False): nprocs = nprocs or mp.cpu_count() - 1 with mp.Manager() as manager: graph = tgraph.create_parallel_compatible_graph(graph, manager) with mp.Pool(nprocs) as pool: exception_q = mp.Queue(10) def error_callback(exception): exception_q.put_nowait(exception) pool.terminate() while not tgraph.all_done(graph): for task in tgraph.get_ready_tasks(graph, reverse=False): graph = tgraph.mark_as_in_progress(graph, task) mlog(graph).info('pid {}: assigning task {}'.format( os.getpid(), task)) pool.apply_async(run_task, args=(graph, task, raise_errors), error_callback=error_callback) time.sleep(sleep) if not exception_q.empty(): raise exception_q.get() return tgraph.recover_values_from_manager(graph)
def __init__(self, time_unit=TIME_UNIT, max_cpu_count=MAX_CPU_COUNT, sleep_time=SLEEP_TIME, empty_count_limit=EMPTY_COUNT_LIMIT, max_sleep_time=MAX_SLEEP_TIME): super().__init__(time_unit=time_unit) self._manager = multiprocess.Manager() # in parent class: self._to_do_queue = self._manager.Queue() self._done_queue = self._manager.Queue() self._tasks_status = self._manager.dict() self._incomplete_task_ids = self._manager.dict() # not in parent class: self._workers_doing = self._manager.dict( ) # key: worker_id, value: task_id self._worker_reports = self._manager.dict() self._cpu_usage = self._manager.Value('i', 0) self._keep_workers_alive = self._manager.Value('i', 1) self._workers = dict() self._max_cpu_count = max_cpu_count or self.system_cpu_count self._sleep_time = sleep_time self._empty_count_limit = empty_count_limit self._max_sleep_time = max_sleep_time atexit.register(self.terminate)
def __init__(self,isThread=1): self.isThread=isThread self.Threads=[] self.idletask={} # self.running = 0 # def __del__(self): #解构时需等待两个队列完成 # time.sleep(0.5) # if self.isThread==1: # self.q_request.join() # self.q_finish.join() if self.isThread==1: # self.lock = Lock() #线程锁 self.work_queue = Queue.Queue()#任务队列 self.lock = Lock() #线程锁 # self.q_request = Queue() #任务所处理的对象队列 # self.q_finish = Queue() #任务所处理的对象完成队列 else : # self.lock = multiprocessing.Lock() # self.q_request=multiprocessing.Queue() # self.q_finish=multiprocessing.Queue() self.lock = multiprocessing.Manager().Lock() temp=multiprocess.Manager() self.work_queue = temp.Queue()#任务队列
def run_parallel_async(graph, nprocs=None, sleep=0.2, raise_errors=False): if nprocs == 1: return run_async(graph, sleep=sleep, raise_errors=raise_errors) nprocs = nprocs or mp.cpu_count() // 2 with mp.Manager() as manager: graph = tgraph.create_parallel_compatible_graph(graph, manager) ioq = mp.Queue(len(graph.funcs.keys())) cpuq = mp.Queue(len(graph.funcs.keys())) procs = [mp.Process(target=run_scheduler, args=(graph, sleep, ioq, cpuq, raise_errors)) for _ in range(nprocs)] for proc in procs: proc.start() while not tgraph.all_done(graph): for task in tgraph.get_ready_tasks(graph): graph = tgraph.mark_as_in_progress(graph, task) mlog(graph).info( 'pid {}: queueing task {}'.format(os.getpid(), task)) if task in graph.io_bound: ioq.put(task) else: cpuq.put(task) time.sleep(sleep) if raise_errors and sum(not p.exitcode for p in procs): raise RuntimeError('An async task has failed. Please check your logs') return tgraph.recover_values_from_manager(graph)
def options_mocks(mocker): mgr = multiprocess.Manager() q = mgr.Queue() def on_subscribe(self, msg, params, websocket): new_params = copy.deepcopy(params) new_params.update({'context': msg.get('context', {})}) q.put(self) return new_params def on_connect(self, message, websocket): q.put(self) def on_disconnect(self, websocket): q.put(self) def on_unsubscribe(self, websocket): q.put(self) options_mocks = { 'on_subscribe': PickableMock(side_effect=promisify(on_subscribe), name='on_subscribe'), 'on_unsubscribe': PickableMock(side_effect=on_unsubscribe, name='on_unsubscribe'), 'on_connect': PickableMock(return_value={'test': 'test_context'}, side_effect=on_connect, name='on_connect'), 'on_disconnect': PickableMock(side_effect=on_disconnect, name='on_disconnect') } return options_mocks, q
def main(): initialize() game = setup_game() with mp.Manager() as manager: exc = manager.Queue() arg_list = [] for i in range(0, 3): arg_list.append(((i, i + 3), game, False, exc)) arg_list.append(((5, 5), game, True, exc)) proc_list = [] for arg in arg_list: proc_list.append(mp.Process(target=function, args=arg)) proc_list[-1].start() print("Number of active children post start: %d" % len(mp.active_children())) for p in proc_list: p.join() if (not exc.empty()): e = exc.get() print(e.message) print("Number active children post join: %d " % len(mp.active_children())) print(mp.active_children()) print(mp.current_process())
def __init__(self, task_data={}): self.id = str(uuid.uuid4()) self.data = task_data self.process = None manager = multiprocess.Manager() self.status_dict = manager.dict() self.set_status('') self.logs = manager.list() self.logger = utilities.create_logger(self.id)
def run(self): with mb.Manager() as m: results = m.dict() p = mb.Process(target=QLWinSingleTest._run_test, args=(self, results)) p.start() p.join() if "exception" not in results: return results['result'] else: raise RuntimeError(f"\n\nGot an exception during subprocess:\n\n{results['exception']}")
def run(self): with mb.Manager() as m: results = m.dict() p = mb.Process(target=QLWinSingleTest._run_test, args=(self, results)) p.start() p.join() if "exception" not in results: return results['result'] else: raise results['exception']
def test(): manager = processing.Manager() gc.disable() print('\n\t######## testing Queue.Queue\n') test_queuespeed(threading.Thread, Queue.Queue(), threading.Condition()) print('\n\t######## testing processing.Queue\n') test_queuespeed(processing.Process, processing.Queue(), processing.Condition()) print('\n\t######## testing Queue managed by server process\n') test_queuespeed(processing.Process, manager.Queue(), manager.Condition()) print('\n\t######## testing processing.Pipe\n') test_pipespeed() print print('\n\t######## testing list\n') test_seqspeed(range(10)) print('\n\t######## testing list managed by server process\n') test_seqspeed(manager.list(range(10))) print('\n\t######## testing Array("i", ..., lock=False)\n') test_seqspeed(processing.Array('i', range(10), lock=False)) print('\n\t######## testing Array("i", ..., lock=True)\n') test_seqspeed(processing.Array('i', range(10), lock=True)) print() print('\n\t######## testing threading.Lock\n') test_lockspeed(threading.Lock()) print('\n\t######## testing threading.RLock\n') test_lockspeed(threading.RLock()) print('\n\t######## testing processing.Lock\n') test_lockspeed(processing.Lock()) print('\n\t######## testing processing.RLock\n') test_lockspeed(processing.RLock()) print('\n\t######## testing lock managed by server process\n') test_lockspeed(manager.Lock()) print('\n\t######## testing rlock managed by server process\n') test_lockspeed(manager.RLock()) print() print('\n\t######## testing threading.Condition\n') test_conditionspeed(threading.Thread, threading.Condition()) print('\n\t######## testing processing.Condition\n') test_conditionspeed(processing.Process, processing.Condition()) print('\n\t######## testing condition managed by a server process\n') test_conditionspeed(processing.Process, manager.Condition()) gc.enable()
def create_jobs(fns, args): manager = multiprocessing.Manager() return_dict = manager.dict() fn_arg = zip(fns, args) jobs = [] for i, (f, a) in enumerate(fn_arg): arg = (i, f, a, return_dict) job = multiprocessing.Process(target=target, args=arg) jobs.append(job) return jobs, return_dict
def __init__(self, filename, mode): logging.Handler.__init__(self) self.handler = logging.FileHandler(filename, mode) manager = multiprocess.Manager() self.queue = manager.Queue(-1) # self.queue = multiprocess.Queue(-1) self.is_closed = False self.t = threading.Thread(target=self.receive) self.t.daemon = True self.t.start()
def run_parallel(graph, nprocs=None, sleep=0.2): nprocs = nprocs or mp.cpu_count() - 1 with mp.Manager() as manager: graph = tgraph.create_parallel_compatible_graph(graph, manager) with mp.Pool(nprocs) as pool: while not tgraph.all_done(graph): for task in tgraph.get_ready_tasks(graph, reverse=False): graph = tgraph.mark_as_in_progress(graph, task) mlog(graph).info('pid {}: assigning task {}'.format( os.getpid(), task)) pool.apply_async(run_task, args=(graph, task)) time.sleep(sleep) return tgraph.recover_values_from_manager(graph)
def __init__(self, num_workers, eval_function, timeout=None): self.num_workers = num_workers self.eval_function = eval_function self.timeout = timeout self.manager = mp.Manager() self.connection_strings = self.manager.Queue() for i in range(num_workers): # Connect to SITL directly wthout mavproxy, can only do one instance # port = 5760 + i*10 # self.connection_strings.put('tcp:127.0.0.1:' + str(port)) # Connect to mavproxy, uses more rescources, max about 20 instances port = 14550 + i * 10 self.connection_strings.put('127.0.0.1:' + str(port)) self.pool = mp.Pool(num_workers, initializer=self.initializer, initargs=(self.connection_strings, ))
def on_sub_mock(mocker): mgr = multiprocess.Manager() q = mgr.Queue() def on_subscribe(self, msg, params, websocket): new_params = copy.deepcopy(params) new_params.update({'context': msg.get('context', {})}) q.put(self) return new_params on_sub_mock = { 'on_subscribe': PickableMock(side_effect=promisify(on_subscribe), name='on_subscribe') } return on_sub_mock, q
def kline_data(self, pair_list, interval, **kwargs): start_date = kwargs.get('start_date', '') end_date = kwargs.get('end_date', '') storage = kwargs.get('storage', '') output = kwargs.get('output', '') progress_statements = kwargs.get('progress_statements', '') if start_date: start_date = datetime.datetime.strptime(start_date, '%m/%d/%Y') if end_date: end_date = datetime.datetime.strptime(end_date, '%m/%d/%Y') valid_kline_intervals = [ '1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h' ] if interval not in set(valid_kline_intervals): raise ValueError( 'Invalid Interval: Kline interval should be one of the following - {}' .format(','.join(valid_kline_intervals))) output = self.process_kline_output(output) if not storage: storage = ['csv', None] try: storage_method, intended_dir = storage except ValueError: storage_method = storage[0] intended_dir = None if progress_statements: self.progress_statements = progress_statements if storage_method.lower() == 'csv': kline_interval_directory = self.create_csv_directories( pair_list, interval, intended_dir) csv_file_info = mp.Manager().list() pair = [currency_pair for i, currency_pair in enumerate(pair_list)] lock = mp.Lock() pool = mp.Pool(processes=3, initargs=(lock, )) # data = pool.starmap(self.kline_to_csv,zip(pair,re(start_date),re(end_date),re(kline_interval_directory),re(interval),re(titles),re(fields),re(csv_file_info))) data = pool.starmap( self.kline_to_csv, zip(pair, re(start_date), re(end_date), re(kline_interval_directory), re(interval), re(csv_file_info))) pool.close() pool.join() self.concatenate_csvs(set(list(csv_file_info))) else: raise ValueError( 'Invalid Storage Type: Currently only csv storage supported')
def _set_alphas(self): manager = mp.Manager() shared_alphas = manager.dict() shared_test_counts = manager.dict() self.alphas = dict() self.test_counts = dict() test_anno = self.anno.groupby('id')['test_data'].agg(is_test=any) exclude_test_samples = self.exclude_test_samples def load_alphas(entry): data = pd.read_csv(entry[1], sep="\t", skip_blank_lines=False, keep_default_na=False) samp_names = [re.sub('\.[0-9]+', '', x) for x in data.columns] if not samp_names: return entry[0], False is_test_sample = [ test_anno.loc[x, 'is_test'] if x in test_anno.index else False for x in samp_names ] is_test_sample = np.array(is_test_sample) if exclude_test_samples is True and any(is_test_sample): shared_alphas[entry[0]] = data.loc[:, ~is_test_sample] + 1 shared_test_counts[entry[0]] = data.loc[:, is_test_sample] else: try: shared_alphas[entry[0]] = data + 1 except Exception as e: raise Exception(f'Unable to deal with {entry[1]}') return entry[0], True tasks = list(self.expression_tsv.items()) with mp.Pool() as pool: for key, success in tqdm(pool.imap(load_alphas, tasks), total=len(tasks), desc='loading counts'): if success is False: warnings.warn('No samples found for {}.'.format(key), RuntimeWarning) continue self.alphas[key] = shared_alphas[key] if key in shared_test_counts.keys(): self.test_counts[key] = shared_test_counts[key] return self.alphas, self.test_counts
def evaluate_population(self, population, n_cores=0): manager = multiprocess.Manager() cache = manager.dict() eval_fun = lambda ind: self.evaluate(ind, make_copy=True, cache=cache) if n_cores == 1: for ind in population: try: eval_fun(ind) except CADETProcessError: print(ind) else: if n_cores == 0: n_cores = None with pathos.multiprocessing.ProcessPool(ncpus=n_cores) as pool: pool.map(eval_fun, population) return cache
def __init__( self, event: mp.synchronize.Event=mp.Event(), # worker: t.Callable=_worker, **kargs ): super(FormalLoader, self).__init__(**kargs) self.event = event self.manager = mp.Manager() self.queue_in, self.queue_out = ( self.manager.Queue(), self.manager.Queue(self.num_workers * 3) ) for smiles_block_i in self.smiles_blocks: self.queue_in.put(smiles_block_i) # self.worker = worker self.pool = mp.Pool(self.num_workers)
def create_feat_mtx_parallel(features, h5_filepath, gene_map, is_fixed_input=True, **kwargs): """Create feature matrix for each feautre in parallel """ mp_dict = mp.Manager().dict() mp_jobs = [] ## Define parallel jobs for feat_tuple in features: mp_job = mp.Process(target=create_feat_mtx_wrapper, args=(mp_dict, feat_tuple, h5_filepath, gene_map, is_fixed_input), kwargs=kwargs) mp_jobs.append(mp_job) ## Execute parallel jobs _ = [p.start() for p in mp_jobs] _ = [p.join() for p in mp_jobs] return mp_dict
def execute(self, transformer: Callable, upstream: Iterable) -> Generator: manager = mp.Manager() manager.register('None', type(None)) input_queue = manager.Queue(maxsize=self.processes) output_queue = manager.Queue() collector = TPE(1, 'multiprocessing-executor-collector') collector.submit(self._collect, upstream, input_queue) with mp.Pool(self.processes) as pool: for _ in range(self.processes): pool.apply_async(MultiProcessingExecutor._work, args=(input_queue, transformer, output_queue)) iterator = iterate_until_none(output_queue.get, self.processes) try: for output in iterator: yield output finally: close_iterator(iterator) collector.shutdown()
def __init__(self, time_unit='ms'): self._processes = {} self._manager = multiprocess.Manager() self._namespace = self._manager.Namespace() self._namespace_dir = set() self._estimators = {} self._to_do = self._manager.list() self._doing = self._manager.dict() self._done = self._manager.list() self._processed = [] self._errors = [] self._proceed_worker = self._manager.dict() self._worker_status = self._manager.dict() self._projects = {} self._tasks_by_id = {} self._time_unit = time_unit self._worker_id_counter = 0 atexit.register(self.terminate) self._last_error_task = None
def main(output_fn, **opts): # openpyxl has trouble with relative paths # convert to absolute path to avoid this issue abs_output = pathlib.Path(output_fn) abs_output = abs_output.resolve() print("Outputting data to", abs_output) ncores = opts['ncores'] shp_fns = opts['shp_fns'] rst_fns = opts['rst_fns'] field = opts['field'] stats = opts['stats'] ignore = opts['ignore'] preprocess = opts['preprocess'] if ncores > 1: import multiprocess as mp from hazardstat import apply_extract with mp.Manager() as manager: # `d` is a DictProxy object shared between all processes # can be converted to dict for final write out d = manager.dict() with manager.Pool(processes=ncores) as pool: # Map each shapefile to a single raster # Then call apply_extract for each shp->raster combination file_combs = itools.product(*[[d], shp_fns, rst_fns, [field], [stats], [ignore], [preprocess]]) procs = pool.starmap_async(apply_extract, file_combs) procs.get() results = dict(d) else: results = extract_stats(shp_fns, rst_fns, field, stats, ignore, preprocess) print("Writing results...") write_to_excel(abs_output, results) print("Finished")
def _set_similarity_matrix(self): """Calculate the similarity metric using a molecular descriptor and a similarity measure. Set this attribute. """ n_mols = len(self.molecule_database) similarity_matrix = np.zeros(shape=(n_mols, n_mols)) # Parallel implementation of similarity calculations. if self.n_threads > 1: m = multiprocess.Manager() q = m.Queue() # worker thread def worker(thread_idx, n_mols, start_idx, end_idx, queue): # pragma: no cover # make a local copy of the overall similarity matrix local_similarity_matrix = np.zeros(shape=(n_mols, n_mols)) if self.is_verbose: print( "thread", thread_idx, "will calculate molecules", start_idx, "through", end_idx, "(", end_idx - start_idx, "total)", ) # same iteration as serial implementation, but only compute # source molecules in the specified range for source_mol_id, molecule in enumerate( self.molecule_database): if source_mol_id >= start_idx and source_mol_id < end_idx: for target_mol_id in range(0, n_mols): if self.is_verbose: print( f"thread {thread_idx} computing similarity " f"of molecule num " f"{target_mol_id + 1} " f"against {source_mol_id + 1}") # diagonal entry if target_mol_id == source_mol_id: local_similarity_matrix[source_mol_id, target_mol_id] = 1 else: # non-diagonal entries try: local_similarity_matrix[ source_mol_id, target_mol_id] = molecule.get_similarity_to( self. molecule_database[target_mol_id], similarity_measure=self. similarity_measure, ) except NotInitializedError as e: e.message += "Similarity matrix could not be set " raise e queue.put(local_similarity_matrix) return None # calculate work distribution and spawn threads remainder = n_mols % (self.n_threads) bulk = n_mols // (self.n_threads) threads = [] for i in range(int(self.n_threads)): # last thread if i == self.n_threads - 1: thread = multiprocess.Process( target=worker, args=( i, n_mols, i * bulk, bulk * (i + 1) + remainder, q, ), ) threads.append(thread) thread.start() else: thread = multiprocess.Process( target=worker, args=( i, n_mols, i * bulk, bulk * (i + 1), q, ), ) threads.append(thread) thread.start() # retrieve the result and sum all the matrices together. for thread in threads: thread.join() thread_results = [] for _ in range(int(self.n_threads)): thread_results.append(q.get()) similarity_matrix = sum(thread_results) else: # serial implementation for source_mol_id, molecule in enumerate(self.molecule_database): for target_mol_id in range(n_mols): if self.is_verbose: print( "Computing similarity of molecule num " f"{target_mol_id + 1} against {source_mol_id + 1}") similarity_matrix[ source_mol_id, target_mol_id] = molecule.get_similarity_to( self.molecule_database[target_mol_id], similarity_measure=self.similarity_measure, ) self.similarity_matrix = similarity_matrix
def get_GPM_swath(self, GPM_file, band='Ku'): ''' Simulates a GPM swath Args: GPM_file: a GPM-DPR file in the HDF5 format band: can be either 'Ka', 'Ku', 'Ku_matched', 'Ka_matched' 'Ka' will be at the location of the HS (high sensitivity) coordinates and at 35.6 GHz 'Ka_matched' will be at the location of the MS (matched scan) coordinates and at 35.6 GHz 'Ku' will be at the location of the NS (high sensitivity) coordinates and at 13.6 GHz 'Ku_matched' will be at the location of the MS (matched scan) coordinates and at 13.6 GHz Returns: An instance of the SimulatedGPM class (see gpm_wrapper.py) which contains the simulated radar observables at the coordinates in the GPM file ''' # Check if model file has been loaded if self.dic_vars == {}: print('No model file has been loaded! Aborting...') return # Assign correct frequencies and 3dB beamwidth, whatever user config cfg_copy = copy.deepcopy(self.config) if band == 'Ku' or band == 'Ku_matched': cfg_copy['radar']['frequency'] = constants.GPM_KU_FREQUENCY elif band == 'Ka' or band == 'Ka_matched': cfg_copy['radar']['frequency'] = constants.GPM_KA_FREQUENCY cfg_copy['radar']['3dB_beamwidth'] = constants.GPM_3DB_BEAMWIDTH cfg_copy['radar']['sensitivity'] = constants.GPM_SENSITIVITY cfg_copy['radar']['type'] = 'GPM' cfg_copy['radar']['radial_resolution'] = \ constants.GPM_RADIAL_RES_KA if band == 'Ka' \ else constants.GPM_RADIAL_RES_KU self.update_config(cfg_copy, check=False) # Needs to be done in order to deal with Multiprocessing's annoying limitations global dic_vars, N, lut_sz, output_variables dic_vars, N, lut_sz, output_variables = self.define_globals() az, elev, rang, coords_GPM = get_GPM_angles(GPM_file, band) # Initialize computing pool pool = mp.Pool(processes=mp.cpu_count(), maxtasksperchild=1) m = mp.Manager() event = m.Event() def worker(event, params): try: if not event.is_set(): azimuth = params[0] elev = params[1] """ For some reason modifying self.config instead of cfg.CONFIG throws in error about not being able to pickle the pycosmo variables. This is indeed very weird and I have not been able to figure out why...However since self.config is just a shallow copy of cfg.CONFIG, it doesn't really matter... """ # Update GPM position and range vector cfg.CONFIG['radar']['range'] = params[2] cfg.CONFIG['radar']['coords'] = [ params[3], params[4], params[5] ] list_subradials = get_interpolated_radial(dic_vars, azimuth, elev, N=N) output = get_radar_observables(list_subradials, lut_sz) if output_variables in ['all', 'only_radar']: output = get_radar_observables(list_subradials, lut_sz) if output_variables == 'only_model': output = integrate_radials(list_subradials) elif output_variables == 'all': output = combine_subradials( (output, integrate_radials(list_subradials))) return output except: # Throw signal back raise event.set() dim_swath = az.shape list_beams = [] for i in range(dim_swath[0]): print('running slice ' + str(i)) # Update radar position c0 = np.repeat(coords_GPM[i, 0], len(az[i])) c1 = np.repeat(coords_GPM[i, 1], len(az[i])) c2 = np.repeat(coords_GPM[i, 2], len(az[i])) worker_partial = partial(worker, event) list_beams.extend( map(worker_partial, zip(az[i], elev[i], rang[i], c0, c1, c2))) pool.close() pool.join() del dic_vars del N del lut_sz gc.collect() if not event.is_set(): # Threshold at given sensitivity if output_variables in ['all', 'only_radar']: list_beams = cut_at_sensitivity(list_beams) list_beams_formatted = SimulatedGPM(list_beams, dim_swath, band) return list_beams_formatted
#midi_fps = glob.glob('./lakh/lakh_clean_midi/*/*.mid*') midi_fps = glob.glob('test_in/*.mid*') out_dir = './test_out' if os.path.isdir(out_dir): shutil.rmtree(out_dir) os.makedirs(out_dir) def _task(x): emit_nesmdb_midi_examples(x, out_dir) drops = mp.Manager().dict({ 'largeFile': 0, 'badMidiDecode': 0, 'too long or short': 0, 'negative times': 0, 'not enough instruments': 0, 'not enough instruments after range drop': 0, 'not enough instruments after polyphonic drop': 0, 'not enough instruments after duplicate drop': 0, '': 0, '': 0, '': 0, '': 0, '': 0 }) with mp.Pool(8, initargs=(drops, )) as p: r = list(tqdm(p.imap(_task, midi_fps), total=len(midi_fps)))