def main(): base_filename = "../plots/survival/{}.pdf" survival_functions = [(sv.FractionOldNew, 'FractionNew'), (sv.OldNewSurvival, 'OldNewMix'), (sv.OldWaning, 'OldWaning')] p = Pool() p.map(run_survival_function, survival_functions)
def createZips(self): t1 = time() if __name__ == '__main__': self.get_list_of_id() # get set of string id p = Pool() p.map(self.createZip, range(self.count_zips)) p.close() p.join() print('Create .zip files time = ' + str(time() - t1) + 's')
def fmultiprocess(log, function, inputArray, poolSize=False, **kwargs): """multiprocess pool **Key Arguments:** - ``log`` -- logger - ``function`` -- the function to multiprocess - ``inputArray`` -- the array to be iterated over **Return:** - ``resultArray`` -- the array of results **Usage:** .. code-block:: python from fundamentals import multiprocess # DEFINE AN INPUT ARRAY inputArray = range(10000) results = multiprocess(log=log, function=functionName, inputArray=inputArray, otherFunctionKeyword="cheese") """ log.info('starting the ``multiprocess`` function') # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1) # if cpu_count() > 1: # poolSize = cpu_count() - 1 # else: # poolSize = 1 # if len(inputArray) < poolSize: # poolSize = len(inputArray) if poolSize: p = Pool(processes=poolSize) else: p = Pool() # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES try: mapfunc = partial(function, log=log, **kwargs) resultArray = p.map(mapfunc, inputArray) except: try: mapfunc = partial(function, **kwargs) resultArray = p.map(mapfunc, inputArray) except: mapfunc = partial(function, log=log, **kwargs) resultArray = p.map(mapfunc, inputArray) p.close() p.terminate() p.join() log.info('completed the ``multiprocess`` function') return resultArray
def extract_patterns_matching_async(self): startTime = time.time() print "running on {} processors".format(WORKERS) pool = Pool(processes=WORKERS, initargs=(sent_locker, lock, sentence_counter)) pool.map(self.extract_patterns_from_file, self.data_wrapper.ngrams_files) pool.close() pool.join() total_time = time.time() - startTime print "extract_patterns_matching_async running time: {}".format( total_time)
def convertpool(self): if len(self.todo) > 0: if self.type in [".h264", ".mp4", ".avi"]: pool = Pool(min(self.pools, len(self.todo))) try: pool.map(self.conv_single, self.todo) pool.close() lineprint("Done converting all videofiles!") except KeyboardInterrupt: lineprint("User terminated converting pool..") pool.terminate() except Exception as e: excep = "Got exception: %r, terminating pool" % (e, ) lineprint(excep) pool.terminate() finally: pool.join() if self.delete: for filein in self.todo: os.remove(filein) lineprint("Deleted all original videofiles..") elif self.type in [".jpg", ".jpeg", ".png"]: vidname = commonpref(self.todo) lineprint("Start converting " + str(len(self.todo)) + " images") frame_array = [] for filename in self.todo: frame = cv2.imread(filename) frame_array.append(frame) #os.rename(filename, self.outdir+"/"+filename) h, w, _ = frame_array[0].shape if self.outdir != "": vidname = self.outdir + "/" + os.path.basename(vidname) vidout = videowriter(vidname, w, h, self.imgfps, self.resizeval) for i in range(len(frame_array)): vidout.write(frame_array[i]) vidout.release() lineprint("Finished converting " + os.path.basename(vidname)) else: lineprint("No video or image files found..")
def main(): workers = 5 while True: try: # Worklist contains data to be distributed worklist = [] # Launch workers process = Pool(workers) # Map data to worker_main function process.map(worker_main, worklist) # Block until all work completed process.close() process.join() except Exception as ex: print(str(ex))
def calculate_prob(hole_cards, num_iterations, given_board): import itertools #must pip these library from multiprocess import Pool import dill as pickle #creates 4 threads p = Pool(4) deck_cards = prob_functions.generate_deck(hole_cards) possible_card_pairings = tuple(itertools.combinations(deck_cards, 2)) card_combos = map( lambda x: tuple (list(hole_cards) + [x]), possible_card_pairings) s = pickle.dumps(lambda hc: single_prob(hc, num_iterations, given_board)) f = pickle.loads(s) prob_list = p.map( f , card_combos) tie = 0 win = 0 for prob in prob_list: tie += prob[0] win += prob[1] l = len(prob_list) tie = tie / l win = win / l return (tie,win)
def compute_jaccard_list_vs_all(seed_indices): pool = Pool(14) affinities_to_seeds = np.array( pool.map(lambda i: compute_jaccard_i_vs_all(i), seed_indices)) pool.close() pool.join() return affinities_to_seeds
def compute_jaccard_pairwise(indices, square_form=True, parallel=True, return_poses=False): n = len(indices) if parallel: pool = Pool(16) scores_poses_tuples = pool.map( lambda x: compute_jaccard_i_vs_list(x[0], x[1]), [(indices[i], indices[i + 1:]) for i in range(n)]) pool.close() pool.join() else: scores_poses_tuples = [ compute_jaccard_i_vs_list(indices[i], indices[i + 1:]) for i in range(n) ] pairwise_scores = np.array( [scores for scores, poses in scores_poses_tuples]) if square_form: pairwise_scores = squareform(np.concatenate(pairwise_scores)) if return_poses: poses = np.array([poses for scores, poses in scores_poses_tuples]) return pairwise_scores, poses else: return pairwise_scores
def create_csv(self): t1 = time() file1 = open(self.out_csv1, "w") file1.write("id" + ',' + "level" + '\n') file2 = open(self.out_csv2, "w") file2.write("id" + ',' + "object_name" + '\n') file1.close() file2.close() if __name__ == '__main__': i = range(len(self.list_of_zips)) p = Pool() p.map(self.parse_Zip, i) p.close() p.join() print('Create .csv files time = ' + str(time() - t1) + 's')
def main(self): urls = self.generateListOfUrls(self.WEBSITE) pool = Pool(5) p_map = pool.map(self.processPage, urls) self.saveLinks(p_map)
def load_scoremaps_multiple_sections_parallel(sections, stack, structure, downscale, detector_id): pool = Pool(12) scoremaps = pool.map(lambda sec: load_scoremap_worker(stack, sec, structure, downscale, detector_id=detector_id), sections) pool.close() pool.join() return {sec: sm for sec, sm in zip(sections, scoremaps) if sm is not None}
def calculate_prob(hole_cards, num_iterations, given_board): import itertools #must pip these library from multiprocess import Pool import dill as pickle #creates 4 threads p = Pool(4) deck_cards = prob_functions.generate_deck(hole_cards) possible_card_pairings = tuple(itertools.combinations(deck_cards, 2)) card_combos = map(lambda x: tuple(list(hole_cards) + [x]), possible_card_pairings) s = pickle.dumps(lambda hc: single_prob(hc, num_iterations, given_board)) f = pickle.loads(s) prob_list = p.map(f, card_combos) tie = 0 win = 0 for prob in prob_list: tie += prob[0] win += prob[1] l = len(prob_list) tie = tie / l win = win / l return (tie, win)
def parallel_apply(df, func, n_cores, n_jobs): df_split = np.array_split(df, n_jobs) pool = Pool(n_cores) df = pd.concat(pool.map(func, df_split)) pool.close() pool.join() return (df)
def calculate_expected(uri, chroms, maxdis=2000000, balance=True, nproc=1): # B: Block Bias, constant for each copy number pair hic_pool = cooler.Cooler(uri) res = hic_pool.binsize maxdis = maxdis // res args = [] for c in chroms: args.append((hic_pool, c, maxdis, balance)) # Allocate processes if nproc == 1: results = list(map(_expected_core, args)) else: pool = Pool(nproc) results = pool.map(_expected_core, args) pool.close() pool.join() expected = {} for i in range(1, maxdis + 1): nume = 0 denom = 0 for extract in results: if i in extract: nume += extract[i][0] denom += extract[i][1] if nume > 0: expected[i] = nume / denom return expected
def fit(self, frame=None, bootstrap=False, n_iter=200): """ Fit Exponential Model """ if not isinstance(frame, pd.core.frame.DataFrame): frame = self.infections.copy() if bootstrap: p = Pool() bootstrapped_lams = p.map(self.fit, bootstrap_frame(frame, n_iter)) # generate durations and initial guess l1_d, l2_d, durations = self.GetInfectionDurations(frame) lam = np.random.random() # run minimization of negative log likelihood opt = minimize(decay_function, lam, args=(l1_d, l2_d), method='L-BFGS-B', bounds=((1e-6, None), )) self.optimizers.append(opt) self.estimated_lam = opt.x[0] if bootstrap: self.bootstrapped_lams = np.array(bootstrapped_lams) return (self.estimated_lam, self.bootstrapped_lams) else: return self.estimated_lam
def main(): xtcfp, top, sel, outp, T, dt, cf = parse_args() if os.path.isfile(xtcfp) and xtcfp.endswith(".xtc"): #seq = cal_rmsdmatrix(xtcfp, cf, top, sel, dt) assign_(xtcfp, cf, top, sel, dt=dt) else: if not os.path.exists(outp): os.mkdir(outp) cwd0 = os.getcwd() os.chdir(outp) if xtcfp.endswith("pkl3") or xtcfp.endswith("pkl") or xtcfp.endswith( "pkl2"): meta = pd.read_pickle(xtcfp) xtcfs = meta["traj_fn"].to_numpy() top = meta["top_fn"].iloc[0] else: xtcfs = [ os.path.join(xtcfp, _) for _ in os.listdir(xtcfp) if _.endswith(".xtc") ] n_trajs = len(xtcfs) dtrajs = [] if T > 1: pool = Pool(T) args = [(xtcfs[i], cf, top, sel, dt) for i in range(n_trajs)] dtrajs = pool.map(parallel, args) else: for i in range(n_trajs): dtraj = assign_(xtcfs[i], cf, top, sel, dt=dt, outfname=None) dtrajs.append(dtraj) os.chdir(cwd0) pd.to_pickle(dtrajs, "dtrajs.pkl3")
def run_parallel_fep(self, mutant_params, system_idx, mutant_idx, n_steps, n_iterations, windows): logger.debug('Computing FEP for {}...'.format(self.name)) if not self.opt: mutant_systems = mutant_params.build_fep_systems( system_idx, mutant_idx, windows) else: mutant_systems = mutant_params nstates = len(mutant_systems) chunk = math.ceil(nstates / self.num_gpu) groups = grouper(range(nstates), chunk) pool = Pool(processes=self.num_gpu) system = copy.deepcopy(self.wt_system) box_vectors = self.input_pdb.topology.getPeriodicBoxVectors() system.setDefaultPeriodicBoxVectors(*box_vectors) system.addForce( mm.MonteCarloBarostat(1 * unit.atmospheres, self.temperature * unit.kelvin, 25)) ### fep = partial(run_fep, sim=self, system=system, pdb=self.extended_pdb, n_steps=n_steps, n_iterations=n_iterations, all_mutants=mutant_systems) u_kln = pool.map(fep, groups) pool.close() pool.join() pool.terminate() ddg = FSim.gather_dg(self, u_kln, nstates) return ddg
def get_new_tickets(self, from_time=utils.pre_day_to_string(1)): search_conditions = { "skip": 0, "query": { "ctimeGte": "{}T21:00:00.000Z".format(from_time) } } pool_size = multiprocess.cpu_count() pool_volume = 10 * pool_size index = 0 tickets_num = self._get_number_of_tickets(from_time, to_time) req_num = utils.ceil_division(tickets_num, 1000) pool = Pool(pool_size) for req_count in range(req_num): search_tickets = self.search_tickets(search_conditions) while True: tickets = pool.map( self.add_attr_to_ticket, itertools.islice(search_tickets, pool_volume)) if tickets: print('Downloaded {}/{} tickets'.format( index, tickets_num), end='\r') index += pool_volume yield tickets else: break search_conditions['skip'] += 1000
def main(): model = Doc2Vec.load("/usr/local/apsis/queries/title_model") do_shingle = True print(scopus_doc.objects.filter(TI__isnull=True).count()) if do_shingle: unshingled_s_docs = scopus_doc.objects.filter(shingle__exists=False) print(unshingled_s_docs.count()) for sd in unshingled_s_docs: if not hasattr(sd, 'TI'): print(sd) sd.delete() else: #try: sd.shingle = list(shingle(sd.TI, 2)) sd.save() #except: # pass scopus_docs_all = scopus_doc.objects.filter(shingle__exists=True, DO__exists=True, doc2vec_checked=False) s_docs_i = scopus_docs_all.count() #s_docs_i = 10 chunk_size = 3 #similarity.objects.all().delete() for i in range(s_docs_i // chunk_size + 1): print(i) #t0 = time.time() f = i * chunk_size l = (i + 1) * chunk_size if l > s_docs_i: l = s_docs_i - 1 s_docs = scopus_docs_all[f:l] print(s_docs) # initialise an empty list, and append sim items to it in parallel sims = [] pool = Pool(processes=chunk_size) sims.append(pool.map(partial(compare, model=model), s_docs)) pool.terminate() #sims = [item for sublist in sims for item in sublist] #try: # sims = [item for sublist in sims for item in sublist] #except: # pass # Flatten and remove nones #print(sims) sims = flatten(sims) sims = list(filter(None.__ne__, sims)) similarity.objects.insert(sims) s_docs.update(doc2vec_checked=True)
def handle(self, *args, **options): qid = options['qid'] q = Query.objects.get(pk=qid) docs = Doc.objects.filter(query=q, wosarticle__cr__isnull=False, cdo__citation__isnull=True) docs = docs ndocs = docs.count() print(ndocs) # Chunk size, so as to prevent overuse of memory chunk_size = 1000 for i in range(ndocs // chunk_size + 1): cdos = [] f = i * chunk_size print(f) l = (i + 1) * chunk_size if l > ndocs: l = ndocs - 1 chunk_docs = docs[f:l] pool = Pool(processes=4) cdos.append(pool.map(doc_cites, chunk_docs)) pool.terminate() gc.collect() django.db.connections.close_all() cdos = flatten(cdos) CDO.objects.bulk_create(cdos)
def find_day_series(df, day, tol, min_occurrence, is_departure: bool, num_procs=1): df_day = df[df["week day"] == day].copy() date_num = dict( zip(np.sort(df_day.day.unique()), range(len(df_day.day.unique())))) df_day["day_num"] = df_day.day.apply(lambda d: date_num[d]) series = df_day.series.unique() len_tot = series.shape[0] len_slice = len_tot // num_procs split_series = [i * len_slice for i in range(num_procs)] + [len_tot] split_flights = tuple([(series[split_series[i]:split_series[i + 1]], df_day[df_day.series.isin( series[split_series[i]:split_series[i + 1]])], tol, min_occurrence, is_departure) for i in range(num_procs)]) pool = Pool(num_procs) result = pool.map(compute_series, split_flights) final_df = pd.concat(result, ignore_index=True) pool.close() pool.join() return final_df
def _train_batch_parallelize(self, trees, n_incorrect_answers): """Parallelizes training for a list of trees. Uses the number of threads given by multiprocessing.cpu_count() Updates model parameters directly, and returns batch error. """ # Defaults to using cpu_count() threads pool = Pool() def get_subbatch_deltas(_trees): return self._train_batch(_trees, n_incorrect_answers, apply_learning=False) subbatches = utils.split(trees, n_slices=cpu_count()) # result will be a list of tuples (error, deltas) result = pool.map(get_subbatch_deltas, subbatches) # no more processes accepted by this pool pool.close() # Wait until mapping is completed pool.join() error = sum([r[0] for r in result]) deltas = [r[1] for r in result] for (delta_Wv, delta_b, delta_We, delta_Wr) in deltas: self.Wv -= delta_Wv self.b -= delta_b self.We -= delta_We self.Wr -= delta_Wr return error
def main(): #match.objects.all().delete() s_docs_count = scopus_doc.objects.filter(DO__exists=True,shingle__exists=True).count() print(s_docs_count) model = Doc2Vec.load("/queries/title_model") s_docs_count = 10025 chunk_size= 10000 for i in range(s_docs_count//chunk_size+1): f = i*chunk_size l = (i+1)*chunk_size-1 if l > s_docs_count: l = s_docs_count-1 s_docs = scopus_doc.objects.filter(DO__exists=True,shingle__exists=True)[f:l] matches = [] sims = [] pool = Pool(processes=16) #matches.append(pool.map(partial(find_match,),s_docs)) sims.append(pool.map(partial(find_sim,),s_docs)) pool.terminate() #matches = [x for x in matches[0] if x is not None] #matches = list(filter(None.__ne__, matches[0])) sims = list(filter(None.__ne__, sims[0])) #match.objects.insert(matches) similarity.objects.insert
def get_panorama(self, fname, pano_id, zoom_level=3): server_url = 'http://cbk%d.google.com/' % randint(0,3) pano_url = server_url + 'cbk?output=tile&panoid=%s&zoom=%d&x=%d&y=%d' zoom_sizes = {3:(7,4), 4:(13,7), 5:(26,13)} max_x, max_y = zoom_sizes[zoom_level] jobs = [] for y in xrange(max_y): for x in xrange(max_x): tile_url = pano_url % (pano_id, zoom_level, x, y) jobs.append(tile_url) p = Pool(len(jobs)) tiles = p.map(self.get_tile, jobs) p.close() if all(x.size for x in tiles): tiles = np.array(tiles) strips = [] for y in xrange(max_y): strips.append(np.hstack(tiles[y*max_x:(y+1)*max_x,:,:,:])) pano = np.vstack(strips) pano = pano[0:1664, 0:3328] else: pano = np.array([]) return pano
def pcall_mp(fun, args, cores=cores): """Calls a function for every input in args""" mainpool = Pool(cores) # create pool out = mainpool.map(fun, args) # return list mainpool.terminate() del mainpool # delete pool return out
def run(self): """ This functions reads the feature extraction filelist and creates a pool of processes to extract features from distinct files in parallel. It outputs one pymir3 FeatureTrack file per input file. Output is buffered to save memory and defer disk access. .. note:: These keys are expected to be set in the experiment file: * ['general']['feature_extraction_filelist'] * ['general']['scratch_directory'] * ['feature_extraction']['output_buffer_size'] * ['feature_extraction']['worker_extractors'] """ print("Running feature extraction behavior: %s" % self.name) # todo: use metadata file to add labels to track metadata (if available) # deve garantir a label no metadados pra facilitar a vida, ao invés de usar o nome do arquivo (acho que não precisa) with open(self.params['general']['feature_extraction_filelist']) as f: files = f.read().splitlines() # todo: usar um multiprocessing.manager pra realizar o compatilhamento do buffer (ao invés de fazer por chunks, como abaixo) metas = copy.copy(files) files = [] for i in metas: files.append(i.split("\t")[0]) metas = [] num_files = len(files) output_buffer_size = self.params['feature_extraction']['output_buffer_size'] pool = Pool(processes=self.params['feature_extraction']['worker_extractors']) for i in range(0, num_files, output_buffer_size): print "processing files %d through %d of %d" % (i + 1, min(i + output_buffer_size, num_files), num_files) result = pool.map(self.extract, files[i:min(i + output_buffer_size, num_files)]) T0 = time.time() for track in result: filename = acf_utils.extract_filename(track.metadata.filename, "wav") + ".features" filename = self.params['general']['scratch_directory'] + "/" + filename print "writing features to file %s..." % (filename) feature_file = open(filename, "w") track.save(feature_file) feature_file.close() del track T1 = time.time() print "writing feature files to disk took %f seconds" % (T1 - T0) del result gc.collect() pool.close() pool.join() print ('Feature extraction done!')
def pcall_mp(fun, args, cores=cores): """Calls a function for every input in args""" mainpool = Pool(cores) # create pool # print("Using",cores,"cores") out = mainpool.map(fun, args) # return list mainpool.terminate() # clear the pool del mainpool # delete pool return out
def ospf_check(): clear_log() devices = [x.split(',')[0] for x in open(devicesFile)] pool = Pool(processor) lock = Manager().Lock() list(pool.map(partial(_inf_ospf_check, lock), devices)) pool.close() pool.join()
def pcall_mp(fun,args,cores=cores): """Calls a function for every input in args""" mainpool = Pool(cores) # create pool # print("Using",cores,"cores") out = mainpool.map(fun,args) # return list mainpool.terminate() del mainpool # delete pool return out
def Multiprocessed_OCRPDF( source="", targetPath=None, processes=4, nice=5, verbose=False, tesseract_config='--oem 1 -l best/eng -c preserve_interword_spaces=1 textonly_pdf=1', logger=None): if isinstance(source, str): if verbose: ( logger.info if logger else print )("You passed a string in as source. Trying this as source pdf file path." ) page_count = PyPDF2.PdfFileReader(source).getNumPages() else: if verbose: (logger.info if logger else print)("OCRUSREX - Try extracting Images from bytes object") page_count = PyPDF2.PdfFileReader(io.BytesIO(source)).getNumPages() output = PyPDF2.PdfFileWriter() # set up a multiprocess pool with the specified number of processes. Then call the single-threaded OCRPDF pethod # on each page p = Pool(processes) for ocred_page in p.map( lambda p: OCRPDF(source=source, verbose=verbose, nice=nice, page=p + 1, tesseract_config=tesseract_config, logger=logger), range(0, page_count)): output.addPage(PyPDF2.PdfFileReader(io.BytesIO(ocred_page)).getPage(0)) if verbose: (logger.info if logger else print)("Multithreaded Execution Complete!") # If targetPath was provided, assume that it's a string and valid path. Try to write. if targetPath: outputStream = open(targetPath, "wb") output.write(outputStream) outputStream.close() # upon success, return truthy values (in this case, True) return True # otherwise, return results as bytes obj else: output_file_obj = io.BytesIO() output.write(output_file_obj) return output_file_obj.getvalue() if verbose: (logger.info if logger else print)( "Complete! Elapsed time: {0}".format(end - start))
def get_gameplays(): PlayTypeDict = {} PlayTypeStrings = { 'Pass': ['pass incomplete', 'pass complete', 'sacked'], 'Admin': ['spiked the ball', 'Timeout', 'Penalty', 'aborted'], 'Kneel': ['knee', 'knelt'], 'Punt': ['Punts'], 'Field Goal': ['field goal', 'no good'], 'Special Teams': ['kicks off', 'kicks onside', 'extra point', 'two point'], 'Run': [ 'left end', 'right end', ' for ', 'up the middle', 'middle for', 'left tackle', 'left guard', 'right guard', 'right tackle' ], } YearStart = 1998 YearsToGo = 20 for Year in range(YearStart, YearStart + YearsToGo): PlayTypeCounts = { 'Pass': 0, 'Run': 0, 'Punt': 0, 'Field Goal': 0, 'Admin': 0, 'Kneel': 0, 'Special Teams': 0 } for GameNumber in range(1, 17): print('Game', GameNumber, 'in', Year, 'Time: ', datetime.now()) PlayTypeDict = {} PathList = [] for Team in TeamLookup: for GameLocation in ['H', 'A']: path = 'https://widgets.sports-reference.com/wg.fcgi?css=1&site=pfr&url=%2Fplay-index%2Fplay_finder.cgi%3Frequest%3D1%26match%3Dall%26year_min%3D{YEAR}%26year_max%3D{YEAR}%26game_type%3DR%26game_num_min%3D{GameNumber}%26game_num_max%3D{GameNumber}%26week_num_min%3D0%26week_num_max%3D99%26game_location%3D{GameLocation}%26minutes_max%3D15%26seconds_max%3D0%26minutes_min%3D0%26seconds_min%3D0%26team_id%3D{TEAM}%26field_pos_min_field%3Dteam%26field_pos_max_field%3Dteam%26end_field_pos_min_field%3Dteam%26end_field_pos_max_field%3Dteam%26type%255B%255D%3DPASS%26type%255B%255D%3DRUSH%26type%255B%255D%3DPUNT%26type%255B%255D%3DKOFF%26type%255B%255D%3DONSD%26type%255B%255D%3DFG%26type%255B%255D%3DXP%26type%255B%255D%3D2PC%26no_play%3DN%26turnover_type%255B%255D%3Dinterception%26turnover_type%255B%255D%3Dfumble%26score_type%255B%255D%3Dtouchdown%26score_type%255B%255D%3Dfield_goal%26score_type%255B%255D%3Dsafety%26order_by%3Dyds_to_go&div=div_all_plays&del_col=1,11,12,13,14'.format( YEAR=Year, GameNumber=GameNumber, TEAM=Team, GameLocation=GameLocation) PathList.append(path) #req = get(path) p = Pool(8) # Pool tells how many at a time records = p.map(GetAndParsePath, PathList) p.terminate() p.join() with open( 'output/PlayTypeCounts-Year-' + str(Year) + '-Game-' + str(GameNumber) + '.json', 'w') as outfile: json.dump(PlayTypeDict, outfile)
def zte_gpon_svlan_check(): clear_log() nodes = graph.cypher.execute( "match(n:Olt)--(c:Card) where c.name='GTGO' return n.ip,collect(c.slot)") olts = ((x[0], x[1]) for x in nodes) lzte_gpon_svlan = lambda x: zte_gpon_svlan(ip=x[0], slots=x[1]) pool = Pool(8) lock = Manager().Lock() func = partial(svlan_entry, lock) list(pool.map(compose(func, lzte_gpon_svlan), olts)) pool.close() pool.join()
def interface_check_m(): clear_log() # cmd = "match(s: Switch) where s.model in ['S8505','S8508'] return s.ip, s.model" cmd = "match(s: Switch) return s.ip, s.model" # cmd = "match(s:Switch) where s.model='S9306' or s.model='s9303' return s.ip,s.model limit 2" nodes = graph.cypher.execute(cmd) switchs = [(x[0], x[1]) for x in nodes] pool = Pool(16) lock = Manager().Lock() out_inf = partial(output_interface_m, lock) list(pool.map(compose(out_inf, get_interface), switchs)) pool.close() pool.join()
def svlan_check(): clear_log() # nodes = graph.find('Olt', property_key='ip', property_value='9.192.96.246') nodes = graph.find('Olt') # nodes = graph.find('Olt', property_key='company', property_value='zte') olts = [(x['ip'], x['company'], x['area']) for x in nodes] # list(map(compose(card_entry, get_card), olts)) pool = Pool(16) lock = Manager().Lock() func = partial(svlan_entry, lock) list(pool.map(compose(func, get_svlan), olts)) pool.close() pool.join()
def add_infs(): funcs = {'zte': Zte.get_infs, 'hw': Huawei.get_infs} get_infs = partial(_company, funcs) clear_log() nodes = graph.cypher.execute( 'match (n:Olt) return n.ip as ip,n.company as company') olts = [dict(ip=x['ip'], company=x['company']) for x in nodes] pool = Pool(128) lock = Manager().Lock() _add_infs_p = partial(_add_infs, lock) list(pool.map(compose(_add_infs_p, get_infs), olts)) pool.close() pool.join()
def main(args): filedate = args.filedate database = args.database slablist = ['alu','cal','cam','car','cas','cot','hal','hel','him','hin','izu','jap','ker','kur','mak','man','mue','pam','png','phi','puy','ryu','sam','sco','sol','sul','sum','van'] indices = range(len(slablist)) pool1 = Pool(args.nCores) partial_loop1 = partial(calls2d, database, filedate, slablist) pts = pool1.map(partial_loop1, indices) pool1.close() pool1.join()
def hostname_check(): clear_log() nodes = graph.find('Olt') # nodes = graph.find('Olt', property_key='ip', property_value='172.18.0.46') olts = [(x['ip'], x['company']) for x in nodes] pool = Pool(16) lock = Manager().Lock() func = partial(hostname_entry, lock) list(pool.map(compose(func, get_hostname), olts)) pool.close() pool.join() ip_hostname = (x.split(',') for x in open(result_file)) cmd = "match (n:Olt) where n.ip={ip} set n.hostname={hostname}" list(map(lambda x: graph.cypher.execute( cmd, ip=x[0], hostname=x[1]), ip_hostname))
def get_vlan_usersP(bras): def _get_vlan_users(bas): funcs = {'m6k': M6k.get_vlan_users, 'me60': ME60.get_vlan_users} _gvu = partial(_model, funcs) return _gvu(bas) bras = [dict(ip=x[0], model=x[1], inf=x[2]) for x in bras] pool = Pool(len(bras)) temp = pool.map(_get_vlan_users, bras) pool.close() pool.join() temp = [x[1] for x in temp if x[1]] rslt = reduce(lambda x, y: merge_with(sum, x, y), temp) return rslt
def calculate(self, data): t1 = dt.datetime.utcnow() LOGGER.info('Starting calculation...') self._data = deepcopy(data) self._check_inputs(data) dep = self._dependencies() sorted_dep = topological_sort(dep) for items in sorted_dep: # loading node with inputs for item in items: node = self._get_node(item) args = [i_name for i_name in node.input_names if i_name not in node.kwargs] data_to_pass = [] for arg in args: data_to_pass.append(self._data[arg]) kwargs_to_pass = {} for kwarg in node.kwargs: kwargs_to_pass[kwarg] = self._data[kwarg] node.load_inputs(data_to_pass, kwargs_to_pass) # running nodes if self._parallel: pool = Pool(self._pool_size) results = pool.map( Graph.run_node, [self._get_node(i) for i in items] ) pool.close() pool.join() results = {k: v for k, v in results} else: results = {} for item in items: node = self._get_node(item) res = node.run_with_loaded_inputs() results[node.id] = res # save results for item in items: node = self._get_node(item) res = results[node.id] if len(node.output_names) == 1: self._data[node.output_names[0]] = res else: for i, out in enumerate(node.output_names): self._data[out] = res[i] t2 = dt.datetime.utcnow() LOGGER.info('Calculation finished in {}'.format(t2-t1)) return res
def zhongji_check(): clear_log() nodes = graph.find('Olt') # nodes = graph.find('Olt', property_key='ip', property_value='172.18.0.46') olts = [(x['ip'], x['company']) for x in nodes] pool = Pool(16) lock = Manager().Lock() func = partial(zhongji_entry, lock) list(pool.map(compose(func, get_zhongji), olts)) pool.close() pool.join() ports = (x.split(',') for x in open(result_file)) cmd = """match(n: Olt) where n.ip = {ip} merge(n) - [:HAS]->(m: Etrunk{name: {sm}}) merge(m) - [:Include]->(p: Port{name: {interface}})""" list(map(lambda x: graph.cypher.execute( cmd, ip=x[0], sm=x[1], interface=x[2]), ports))
def parallel_cdist(data1, data2, n_rows_per_job=100): from scipy.spatial.distance import cdist data1 = np.array(data1) data2 = np.array(data2) pool = Pool(12) start_indices = np.arange(0, data1.shape[0], n_rows_per_job) end_indices = start_indices + n_rows_per_job - 1 partial_distance_matrices = pool.map(lambda (si, ei): cdist(data1[si:ei+1].copy(), data2), zip(start_indices, end_indices)) pool.close() pool.join() distance_matrix = np.concatenate(partial_distance_matrices) return distance_matrix
def eval_EFG(self,x,num_procs=None,info=False): from multiprocess import Pool,cpu_count if not num_procs: num_procs = cpu_count() num_samples = self.parameters['num_samples'] pool = Pool(num_procs) num = int(np.ceil(float(num_samples)/float(num_procs))) results = list(zip(*pool.map(lambda i: self.eval_EFG_sequential(x,num,i,info),range(num_procs),chunksize=1))) pool.terminate() pool.join() if not info: assert(len(results) == 4) else: assert(len(results) == 5) assert(all([len(vals) == num_procs for vals in results])) return [sum(vals)/float(num_procs) for vals in results]
def add_power_info(): funcs = {'S8508': S85.get_power_info, 'S8505': S85.get_power_info, 'T64G': T64.get_power_info, 'S8905': S89.get_power_info, 'S8905E': S8905E.get_power_info, 'S9306': S93.get_power_info, 'S9303': S93.get_power_info} get_power_info = partial(_model, funcs) # clear_log() nodes = graph.cypher.execute( "match (s:Switch) where s.snmpState='normal' return s.ip as ip,s.model as model") switches = [dict(ip=x['ip'], model=x['model']) for x in nodes] pool = Pool(processor) lock = Manager().Lock() _ff = partial(_add_power_info, lock) list(pool.map(compose(_ff, get_power_info), switches)) pool.close() pool.join()
def add_traffics(): funcs = {'S8508': S85.get_traffics, 'S8505': S85.get_traffics, 'T64G': T64.get_traffics, 'S8905': S89.get_traffics, 'S8905E': S8905E.get_traffics, 'S9306': S93.get_traffics, 'S9303': S93.get_traffics} get_traffics = partial(_model, funcs) # clear_log() nodes = graph.cypher.execute( "match (s:Switch)--(i:Inf) where s.snmpState='normal' return s.ip as ip,collect(i.name) as infs,s.model as model") switchs = [dict(ip=x['ip'], infs=x['infs'], model=x['model']) for x in nodes] pool = Pool(processor) lock = Manager().Lock() _ff = partial(_add_traffics, lock) list(pool.map(compose(_ff, get_traffics), switchs)) pool.close() pool.join()
def compute_jaccard_pairwise(indices, square_form=True, parallel=True, return_poses=False): n = len(indices) if parallel: pool = Pool(16) scores_poses_tuples = pool.map(lambda x: compute_jaccard_i_vs_list(x[0],x[1]), [(indices[i], indices[i+1:]) for i in range(n)]) pool.close() pool.join() else: scores_poses_tuples = [compute_jaccard_i_vs_list(indices[i], indices[i+1:]) for i in range(n)] pairwise_scores = np.array([scores for scores, poses in scores_poses_tuples]) if square_form: pairwise_scores = squareform(np.concatenate(pairwise_scores)) if return_poses: poses = np.array([poses for scores, poses in scores_poses_tuples]) return pairwise_scores, poses else: return pairwise_scores
def eval_EQ(self,p,num_procs=None,quiet=True): """ Evaluates E[Q(p,r)] and its gradient in parallel. Parameters ---------- p : generator powers num_procs : number of parallel processes quiet : flag """ from multiprocess import Pool,cpu_count if not num_procs: num_procs = cpu_count() num_samples = self.parameters['num_samples'] pool = Pool(num_procs) num = int(np.ceil(float(num_samples)/float(num_procs))) results = list(zip(*pool.map(lambda i: self.eval_EQ_sequential(p,num,i,quiet),range(num_procs),chunksize=1))) pool.terminate() pool.join() assert(len(results) == 2) assert(all([len(vals) == num_procs for vals in results])) return [sum(vals)/float(num_procs) for vals in results]
def get_new_tickets(self, from_time=utils.pre_day_to_string(1)): search_conditions = { "skip": 0, "query": { "ctimeGte": "{}T21:00:00.000Z".format(from_time) } } pool_size = multiprocess.cpu_count() pool_volume = 10 * pool_size index = 0 tickets_num = self._get_number_of_tickets(from_time, to_time) req_num = utils.ceil_division(tickets_num, 1000) pool = Pool(pool_size) for req_count in range(req_num): search_tickets = self.search_tickets(search_conditions) while True: tickets = pool.map(self.add_attr_to_ticket, itertools.islice(search_tickets, pool_volume)) if tickets: print('Downloaded {}/{} tickets'.format(index, tickets_num), end='\r') index += pool_volume yield tickets else: break search_conditions['skip'] += 1000
else: raise # input_dir = DataManager.get_image_dir_v2(stack=stack, prep_id=5, version=version, resol='raw') out_dir = DataManager.get_image_dir_v2(stack=stack, prep_id=2, resol=resol, version=version) print 'out_dir:', out_dir # script = os.path.join(REPO_DIR, 'preprocess', 'warp_crop_IM_v3.py') # ! rm -rf {out_dir} create_if_not_exists(out_dir) t = time.time() pool = Pool(8) _ = pool.map(lambda img_name: crop(stack=stack, img_name=img_name, version=version, resol=resol, x=x, y=y, w=w, h=h), metadata_cache['valid_filenames'][stack]) pool.close() pool.join() # for img_name in metadata_cache['valid_filenames'][stack]: # f(stack=stack, img_name=img_name, version=version, resol=resol, # x=x, y=y, w=w, h=h) # run_distributed('convert \"%%(input_fp)s\" -crop %(w)dx%(h)d+%(x)d+%(y)d \"%%(output_fp)s\"' % \ # {'w':w_raw, 'h':h_raw, 'x':x_raw, 'y':y_raw}, # kwargs_list=[{'input_fp': DataManager.get_image_filepath_v2(stack=stack, prep_id=5, resol='raw', version=version, fn=img_name), # 'output_fp': DataManager.get_image_filepath_v2(stack=stack, fn=img_name, prep_id=2, version=version, resol='raw')} # for img_name in metadata_cache['valid_filenames'][stack]], # # for img_name in ['CHATM3_slide35_2018_02_17-S1']], # argument_type='single',
ntb_to_nissl[ntb_v] = np.unique(a)[0] ntb_values = np.arange(0, 5000) ntb_matched_values = np.interp(ntb_values, [ntb_v for ntb_v, nissl_v in sorted(ntb_to_nissl.items())], [nissl_v for ntb_v, nissl_v in sorted(ntb_to_nissl.items())]) sys.stderr.write('Compute matching: %.2f seconds.\n' % (time.time()-t)) return ntb_matched_values, (region1_x, region1_y, region1_w, region1_h) n_regions = 8 pool = Pool(4) res = pool.map(f, range(n_regions)) ntb_matched_values_all_examples_one_section, region_bboxes_all_examples_one_section = zip(*res) pool.close() pool.join() # for region_id in range(10): # while True: # region1_x = np.random.randint(0, w-10000, 1)[0] # region1_y = np.random.randint(0, h-10000, 1)[0] # region1_w = 5000 # region1_h = 5000 # print region1_x, region1_y, region1_w, region1_h # tb_region1_xmin = region1_x / 32 # tb_region1_xmax = (region1_x + region1_w) / 32
ntb_blue_inv_bins = np.arange(5001) ntb_inv_to_nissl_mapping = np.interp(ntb_blue_inv_bins, ntb_inv_vals, nissl_vals) ntb_to_nissl_mapping = ntb_inv_to_nissl_mapping[5000 - ntb_blue_bins] ntb_to_nissl_mapping = np.round(ntb_to_nissl_mapping).astype(np.uint8) ntb_matched_values_all_examples_one_section.append(ntb_to_nissl_mapping) region_bboxes_all_examples_one_section.append((region1_x, region1_y, region1_w, region1_h)) sys.stderr.write('Compute matching: %.2f seconds.\n' % (time.time()-t)) return ntb_to_nissl_mapping, (region1_x, region1_y, region1_w, region1_h) pool = Pool(4) res = pool.map(match_intensity_histogram_one_region, regions) ntb_matched_values_all_examples_one_section, region_bboxes_all_examples_one_section = zip(*res) pool.close() pool.join() fp = os.path.join(DATA_DIR, stack, stack + '_intensity_mapping', '%s_to_%s_intensity_mapping_all_regions.npy' % (ntb_fn, nissl_fn)) create_parent_dir_if_not_exists(fp) np.save(fp, np.asarray(ntb_matched_values_all_examples_one_section)) upload_to_s3(fp) fp = os.path.join(DATA_DIR, stack, stack + '_intensity_mapping', '%s_to_%s_region_bboxes.npy' % (ntb_fn, nissl_fn)) np.save(fp, np.asarray(region_bboxes_all_examples_one_section)) upload_to_s3(fp) median_mapping_one_section = np.median(ntb_matched_values_all_examples_one_section, axis=0)
def compute_jaccard_list_vs_all(seed_indices): pool = Pool(14) affinities_to_seeds = np.array(pool.map(lambda i: compute_jaccard_i_vs_all(i), seed_indices)) pool.close() pool.join() return affinities_to_seeds
def compute_spm_histograms(labelmap, sample_locs, patch_size, M): """ Args: labelmap (2d-ndarray of int): sample_locs (2d-ndarray): List of (x,y) locations at which to sample the SPM histograms M (int): number of unique SIFT descriptor words, aka. size of vocabulary Returns: hists_arr0 ((1,M)-array of int) hists_arr1 ((4,M)-array of int) hists_arr2 ((16,M)-array of int) """ global labelmap_global labelmap_global = labelmap # compute level-2 histograms l = 2 grid_size = patch_size / 2**l if l == 2: rx = [-2, -1, 0, 1] ry = [-2, -1, 0, 1] elif l == 1: rx = [-1, 0] ry = [-1, 0] elif l == 0: rx = [-.5] ry = [-.5] rxs, rys = np.meshgrid(rx, ry) patch_coords_allGrid = [] for grid_i, (rx, ry) in enumerate(np.c_[rxs.flat, rys.flat]): patch_xmin = sample_locs[:,0] + rx * grid_size patch_ymin = sample_locs[:,1] + ry * grid_size patch_xmax = sample_locs[:,0] + (rx + 1) * grid_size patch_ymax = sample_locs[:,1] + (ry + 1) * grid_size patch_coords_allGrid.append([patch_xmin, patch_ymin, patch_xmax, patch_ymax]) all_coords = np.hstack(patch_coords_allGrid) patch_xmin = all_coords[0] patch_ymin = all_coords[1] patch_xmax = all_coords[2] patch_ymax = all_coords[3] def compute_histogram_particular_label(i): m = (labelmap_global == i).astype(np.uint8) mi = cv2.integral(m) ci = mi[patch_ymin, patch_xmin] + mi[patch_ymax, patch_xmax] - mi[patch_ymax, patch_xmin] - mi[patch_ymin, patch_xmax] return ci t = time.time() # hists = Parallel(n_jobs=16)(delayed(compute_histogram_particular_label)(i) for i in range(1, M+1)) # hists = Parallel(n_jobs=8)(delayed(compute_histogram_particular_label)(i) for i in range(1, M+1)) pool = Pool(8) hists = pool.map(compute_histogram_particular_label, range(1, M+1)) # pool.terminate() pool.close() pool.join() # del pool sys.stderr.write('done in %f seconds\n' % (time.time() - t)) # ~ 13 seconds n_grid = (2**l)**2 hists_arr2 = np.transpose(np.reshape(hists, (M, n_grid, -1))) print hists_arr2.shape # compute level-1 histograms based on level-2 histograms hists_arr1 = np.transpose([hists_arr2[:, [0,1,4,5], :].sum(axis=1), hists_arr2[:, [2,3,6,7], :].sum(axis=1), hists_arr2[:, [8,9,12,13], :].sum(axis=1), hists_arr2[:, [10,11,14,15], :].sum(axis=1)], [1,0,2]) print hists_arr1.shape # compute level-0 histograms based on level-1 histograms hists_arr0 = hists_arr1.sum(axis=1) print hists_arr0.shape return hists_arr0, hists_arr1, hists_arr2
def save_scoremap(structure): viz_fp = DataManager.get_scoremap_viz_filepath(stack=stack, downscale=downscale, fn=fn, structure=structure, detector_id=detector_id) create_parent_dir_if_not_exists(viz_fp) try: if add_label_text: label_text = str(structure) else: label_text = None viz = scoremap_overlay_on(bg='original', stack=stack, fn=fn, structure=structure, out_downscale=downscale, label_text=label_text, detector_id=detector_id, cmap_name=cmap_name, image_version=bg_image_version) imsave(viz_fp, img_as_ubyte(viz)) upload_to_s3(viz_fp) except Exception as e: # raise e sys.stderr.write('%s\n' % e.message) return # for s in all_known_structures: # save_scoremap(s) pool = Pool(NUM_CORES/2) pool.map(save_scoremap, all_known_structures) pool.close() pool.join() sys.stderr.write('Visualize scoremaps: %.2f seconds.\n' % (time.time() - t)) # 7s for one structure, one section, single process # 20s for all structures, one section, 8 processes
if hasattr(args, "rescale_factor"): rescale_factor = args.rescale_factor else: w = args.width h = args.height n_jobs = args.jobs def worker(img_name): input_fp = input_fp_map[img_name] output_fp = output_fp_map[img_name] create_parent_dir_if_not_exists(output_fp) img = imread(input_fp) save_data(img[::1/rescale_factor, ::1/rescale_factor], output_fp) pool = Pool(n_jobs) _ = pool.map(worker, in_image_names) pool.close() pool.join() # run_distributed('convert \"%%(input_fp)s\" -crop %(w)dx%(h)d+%(x)d+%(y)d \"%%(output_fp)s\"' % \ # {'w':w_raw, 'h':h_raw, 'x':x_raw, 'y':y_raw}, # kwargs_list=[{'input_fp': , # 'output_fp': output_fp_map[img_name]} # for img_name in metadata_cache['valid_filenames'][stack]], # argument_type='single', # jobs_per_node=1, # local_only=True)
structure_colors = {n: np.random.randint(0, 255, (3,)) for n in all_known_structures} def generate_annotation_viz_one_section(stack, fn, structure_colors=structure_colors, downsample_factor=downsample_factor): global contours if is_invalid(fn): return img_fp = DataManager.get_image_filepath(stack=stack, fn=fn, resol='lossless', version='compressed') download_from_s3(img_fp) img = imread(img_fp) viz = img[::downsample_factor, ::downsample_factor].copy() for name_u, color in structure_colors.iteritems(): matched_contours = contours[(contours['name'] == name_u) & (contours['filename'] == fn)] for cnt_id, cnt_props in matched_contours.iterrows(): cv2.polylines(viz, [(cnt_props['vertices']/downsample_factor).astype(np.int)], True, color, 2) viz_fp = DataManager.get_annotation_viz_filepath(stack=stack, fn=fn) create_parent_dir_if_not_exists(viz_fp) imsave(viz_fp, viz) upload_to_s3(viz_fp) # for fn in filenames: # generate_annotation_viz_one_section(fn=fn) pool = Pool(NUM_CORES/2) pool.map(lambda fn: generate_annotation_viz_one_section(stack=stack, fn=fn, structure_colors=structure_colors, downsample_factor=downsample_factor), filenames) pool.close() pool.join()
# sys.stderr.write('Compute saturation: %.2f seconds\n' % (time.time() - t1)) # skimage 6.5s; opencv 5s def generate_versions(fn, which): input_fn=os.path.join(input_dir, fn) basename = os.path.splitext(os.path.basename(fn))[0] if 'compressed' in which: output_compressed_fn = os.path.join(output_compressed_dir, basename + '_compressed.jpg') if 'compressed' in which: if os.path.exists(output_compressed_fn): sys.stderr.write('File exists: %s.\n' % output_compressed_fn) else: os.system("convert %(input_fn)s -format jpg %(output_compressed_fn)s" % \ dict(input_fn=input_fn, output_compressed_fn=output_compressed_fn)) if 'saturation' in which: # output_saturation_fn = os.path.join(output_saturation_dir, basename + '_saturation.jpg') # why jpg? output_saturation_fn = os.path.join(output_saturation_dir, basename + '_saturation.tif') if os.path.exists(output_saturation_fn): sys.stderr.write('File exists: %s.\n' % output_saturation_fn) else: convert_to_saturation(input_fn, output_saturation_fn, rescale=True) #Parallel(n_jobs=4)(delayed(generate_versions)(fn, which) for fn in filenames) pool = Pool(4) pool.map(lambda fn: generate_versions(fn, which), filenames) pool.close() pool.join()
for iy, y0 in enumerate(np.arange(0, img_h, 5000)): for ix, x0 in enumerate(np.arange(0, img_w, 5000)): origins.append((x0, y0)) alg = 'cellprofiler' big_labelmap = np.zeros((img_h, img_w), dtype=np.int64) n = 0 for i, input_fp in enumerate(input_fps): prefix = os.path.splitext(input_fp)[0] labelmap = labelmap_alltiles[i].astype(np.int64) # astype(np.int64) is important, otherwise results in negative label values. x0, y0 = origins[i] big_labelmap[y0:y0+5000, x0:x0+5000][labelmap != 0] = labelmap[labelmap != 0] + n n += labelmap.max() labelmap_fp = os.path.splitext(input_img_fp)[0] + '_labelmap_%(alg)s.bp' % dict(alg=alg) bp.pack_ndarray_file(big_labelmap, labelmap_fp) upload_to_s3(labelmap_fp) for fp in input_fps: execute_command('rm ' + fp) t = time.time() pool = Pool(NUM_CORES/2) pool.map(detect_cells, filenames) pool.close() pool.join() sys.stderr.write('Overall time: %.2f seconds.\n' % (time.time()-t))
big_labelmap[y0:y0+5000, x0:x0+5000][labelmap != 0] = labelmap[labelmap != 0] + n n += labelmap.max() labelmap_fp = os.path.splitext(input_img_fp)[0] + '_labelmap_%(alg)s.bp' % dict(alg=alg) bp.pack_ndarray_file(big_labelmap, labelmap_fp) # for tile_i in range(12): # execute_command('rm %(DETECTED_CELLS_DIR)s/%(stack)s/%(img_fn)s/%(img_fn)s_image_inverted_%(tile_i)02d.tif' % \ # dict(DETECTED_CELLS_DIR=DETECTED_CELLS_DIR, stack=stack, img_fn=img_fn, tile_i=tile_i)) # execute_command('rm %(DETECTED_CELLS_DIR)s/%(stack)s/%(img_fn)s/%(img_fn)s_image_inverted_%(tile_i)02d_labelmap_cellprofiler.bp' % \ # dict(DETECTED_CELLS_DIR=DETECTED_CELLS_DIR, stack=stack, img_fn=img_fn, tile_i=tile_i)) # Generate labelmap viz t = time.time() viz = img_as_ubyte(label2rgb(big_labelmap, bg_label=0, bg_color=(0, 0, 0))) cv2.imwrite(os.path.splitext(input_img_fp)[0] + '_labelmap_%(alg)s.png' % dict(alg=alg), viz); sys.stderr.write('Generate labelmap viz: %.2f seconds.\n' % (time.time()-t)) # 60s t = time.time() pool = Pool(12) pool.map(detect_cells, range(first_sec, last_sec+1)) pool.close() pool.join() sys.stderr.write('Overall time: %.2f seconds.\n' % (time.time()-t))
dense_score_map[dense_score_map < 1e-1] = 0 dense_score_map[dense_score_map > 1.] = 1. # sys.stderr.write('threshold: %.2f seconds\n' % (time.time() - t)) if np.count_nonzero(dense_score_map) < 1e5: sys.stderr.write('No %s is detected on section %d\n' % (structure, sec)) return None t1 = time.time() scoremap_bp_filepath, scoremap_interpBox_filepath = \ DataManager.get_scoremap_filepath(stack=stack, fn=fn, anchor_fn=anchor_fn, structure=structure, return_bbox_fp=True, setting=actual_setting) save_hdf(dense_score_map.astype(np.float16), scoremap_bp_filepath, complevel=5) np.savetxt(scoremap_interpBox_filepath, np.array((interpolation_xmin, interpolation_xmax, interpolation_ymin, interpolation_ymax))[None], fmt='%d') sys.stderr.write('save: %.2f seconds\n' % (time.time() - t1)) # 4s, very high penalty when multiprocessing t = time.time() pool = Pool(4) # 8 causes contention, resuls in high upscaling and dumping to disk time. _ = pool.map(generate_score_map, structures) pool.close() pool.join() sys.stderr.write('interpolate: %.2f seconds\n' % (time.time() - t)) # ~ 30 seconds / section