def run(self): from production import util # copy the script to the temp folder and replace the shebang file_dir = os.path.dirname(os.path.abspath(__file__)) util.copy_and_replace( os.path.join(file_dir, 'initial_features.py'), os.path.join(self.tmp_folder, 'initial_features.py')) with open(self.config_path) as f: config = json.load(f) block_shape = config['block_shape'] offsets = config['affinity_offsets'] roi = config.get('roi', None) # hardcoded keys graph_key = 'graph' out_key = 'features' # create the outpuy files f_graph = z5py.File(self.graph_path, use_zarr_format=False) shape = f_graph.attrs['shape'] ds_graph = f_graph[graph_key] n_edges = ds_graph.attrs['numberOfEdges'] f_out = z5py.File(self.out_path, use_zarr_format=False) f_out.require_group('blocks') # chunk size = 64**3 chunk_size = min(262144, n_edges) f_out.require_dataset(out_key, dtype='float64', shape=(n_edges, 10), chunks=(chunk_size, 1), compression='gzip') # get number of blocks blocking = nifty.tools.blocking([0, 0, 0], shape, block_shape) # check if we have a roi and adjuse the block list if we do if roi is None: n_blocks = blocking.numberOfBlocks block_list = list(range(n_blocks)) else: block_list = blocking.getBlockIdsOverlappingBoundingBox( roi[0], roi[1], [0, 0, 0]).tolist() n_blocks = len(block_list) # find the actual number of jobs and prepare job configs n_jobs = min(n_blocks, self.max_jobs) self._prepare_jobs(n_jobs, block_list, offsets) # submit the jobs if self.run_local: # this only works in python 3 ?! with futures.ProcessPoolExecutor(n_jobs) as tp: tasks = [ tp.submit(self._submit_job, job_id) for job_id in range(n_jobs) ] [t.result() for t in tasks] else: for job_id in range(n_jobs): self._submit_job(job_id) # wait till all jobs are finished if not self.run_local: util.wait_for_jobs('papec') # check the job outputs processed_jobs, times = self._collect_outputs(n_jobs) assert len(processed_jobs) == len(times) success = len(processed_jobs) == n_jobs # write output file if we succeed, otherwise write partial # success to different file and raise exception if success: out = self.output() # TODO does 'out' support with job? fres = out.open('w') json.dump({'times': times}, fres) fres.close() else: log_path = os.path.join(self.tmp_folder, 'initial_features_partial.json') with open(log_path, 'w') as out: json.dump({ 'times': times, 'processed_jobs': processed_jobs }, out) raise RuntimeError( "InitialFeatureTask failed, %i / %i jobs processed," % (len(processed_jobs), n_jobs) + "serialized partial results to %s" % log_path)
parser = argparse.ArgumentParser() parser.add_argument('-r', type=float, default=0.0, help='ratio of missing items') args = parser.parse_args() RATIO = 1 - args.r with open('../ml-1m/ratining.csv') as fin: fin.readline() ALL_ITEMS = {line.strip().split(',')[1] for line in fin} with open('item_index2entity_id_ratio_{:.2f}.txt'.format(RATIO), 'wt') as fout: future_to_movie = {} with cf.ProcessPoolExecutor(max_workers=50) as executor: with open('./MappingMovielens2DBpedia-1.2.tsv') as fin: fin.readline() for line in fin: movie_id, _, dbpedia_id = line.strip().split('\t') if movie_id in ALL_ITEMS: future_to_movie[executor.submit(get_mapping, dbpedia_id)] = movie_id output: list = [] for future in cf.as_completed(future_to_movie): movie_id = future_to_movie[future] kg_id = future.result() if kg_id: output.append('{}\t{}'.format(movie_id, kg_id)) random.shuffle(output) fout.write("\n".join(output[:int(len(output) * (RATIO))]))
def estimate_pi(num_steps=20000, step_size=0.1): _, s = markov_chain(num_steps, step_size) return np.mean(s) NUM_STEPS = 20000 # Default step size chain, samples = markov_chain(NUM_STEPS, 0.1, show_plot=True, name="default") # Optimal step size STEP_SIZE = 1.1812 markov_chain(NUM_STEPS, STEP_SIZE, show_plot=True, name="best_step_size") with futures.ProcessPoolExecutor() as executor: fs = [ executor.submit(estimate_pi, NUM_STEPS, STEP_SIZE) for _ in range(100) ] m = np.array([f.result() for f in fs]) print("Empirical variance: {}".format(np.sqrt(np.var(m)))) # batching procedure to estimate the variance def batching(samples, variance): if len(samples) <= 1: return variance if len(samples) % 2 != 0:
import os import time import sys from concurrent import futures def to_do(info): st = time.time() for i in range(10000000): pass print(f"{info[0]} and {info[1]} took {time.time()-st} seconds.") return info[0] start_time = time.time() MAX_WORKERS = 15 param_list = [] for i in range(15): param_list.append(('text%s' % i, 'info%s' % i)) workers = min(MAX_WORKERS, len(param_list)) # with 默认会等所有任务都完成才返回,所以这里会阻塞 with futures.ProcessPoolExecutor(workers) as executor: results = executor.map(to_do, sorted(param_list)) # 打印所有 for index, result in enumerate(results, start=1): print(f"{index} result is: {result}") print(time.time() - start_time) # 耗时0.3704512119293213s, 而线程版本需要14.935384511947632s
def run_tests(all_tests, log_name_base, extra_args): global stop, executor, futures txtname = log_name_base + '.txt' xmlname = log_name_base + '.xml' logfile = open(txtname, 'w', encoding="utf_8") junit_root = ET.Element('testsuites') conf_time = 0 build_time = 0 test_time = 0 passing_tests = 0 failing_tests = 0 skipped_tests = 0 try: # This fails in some CI environments for unknown reasons. num_workers = multiprocessing.cpu_count() except Exception as e: print( 'Could not determine number of CPUs due to the following reason:' + str(e)) print('Defaulting to using only one process') num_workers = 1 executor = conc.ProcessPoolExecutor(max_workers=num_workers) for name, test_cases, skipped in all_tests: current_suite = ET.SubElement(junit_root, 'testsuite', { 'name': name, 'tests': str(len(test_cases)) }) if skipped: print('\nNot running %s tests.\n' % name) else: print('\nRunning %s tests.\n' % name) futures = [] for t in test_cases: # Jenkins screws us over by automatically sorting test cases by name # and getting it wrong by not doing logical number sorting. (testnum, testbase) = os.path.split(t)[-1].split(' ', 1) testname = '%.3d %s' % (int(testnum), testbase) should_fail = False if name.startswith('failing'): should_fail = name.split('failing-')[1] result = executor.submit(run_test, skipped, t, extra_args, unity_flags + backend_flags, compile_commands, should_fail) futures.append((testname, t, result)) for (testname, t, result) in futures: result = result.result() if result is None or 'MESON_SKIP_TEST' in result.stdo: print('Skipping:', t) current_test = ET.SubElement(current_suite, 'testcase', { 'name': testname, 'classname': name }) ET.SubElement(current_test, 'skipped', {}) skipped_tests += 1 else: without_install = "" if len( install_commands) > 0 else " (without install)" if result.msg != '': print('Failed test%s: %s' % (without_install, t)) print('Reason:', result.msg) failing_tests += 1 failing_logs.append(result.stdo) failing_logs.append(result.stde) else: print('Succeeded test%s: %s' % (without_install, t)) passing_tests += 1 conf_time += result.conftime build_time += result.buildtime test_time += result.testtime total_time = conf_time + build_time + test_time log_text_file(logfile, t, result.stdo, result.stde) current_test = ET.SubElement( current_suite, 'testcase', { 'name': testname, 'classname': name, 'time': '%.3f' % total_time }) if result.msg != '': ET.SubElement(current_test, 'failure', {'message': result.msg}) stdoel = ET.SubElement(current_test, 'system-out') stdoel.text = result.stdo stdeel = ET.SubElement(current_test, 'system-err') stdeel.text = result.stde print("\nTotal configuration time: %.2fs" % conf_time) print("Total build time: %.2fs" % build_time) print("Total test time: %.2fs" % test_time) ET.ElementTree(element=junit_root).write(xmlname, xml_declaration=True, encoding='UTF-8') return (passing_tests, failing_tests, skipped_tests)
def calc_csr_kick( beam, charges, Np=None, gamma=None, rho=None, Nz=100, sigma_z=1e-3, Nx=100, sigma_x=1e-3, reuse_psi_grids=False, psi_s_grid_old=None, psi_x_grid_old=None, verbose=True, ): """ """ (x_b, xp_b, y_b, yp_b, z_b, zp_b) = beam zx_positions = np.stack((z_b, x_b)).T # Fix the grid here # The grid needs to enclose most particles in z-x space mins = np.array([-6 * sigma_z, -6 * sigma_x]) # Lower bounds of the grid maxs = np.array([6 * sigma_z, 6 * sigma_x]) # Upper bounds of the grid sizes = np.array([Nz, Nx]) (zmin, xmin) = mins (zmax, xmax) = maxs (Nz, Nx) = sizes (dz, dx) = (maxs - mins) / (sizes - 1) # grid steps # indexes, contrib = split_particles(zx_positions, charges, mins, maxs, sizes) # t1 = time.time(); # charge_grid = deposit_particles(Np, sizes, indexes, contrib) # t2 = time.time(); # Remi's fast code t1 = time.time() charge_grid = histogram_cic_2d(z_b, x_b, charges, Nz, zmin, zmax, Nx, xmin, xmax) t2 = time.time() # Normalize the grid so its integral is unity norm = np.sum(charge_grid) * dz * dx lambda_grid = charge_grid / norm # Apply savgol filter lambda_grid_filtered = np.array( [savgol_filter(lambda_grid[:, i], 13, 2) for i in np.arange(Nx)] ).T # Differentiation in z lambda_grid_filtered_prime = central_difference_z( lambda_grid_filtered, Nz, Nx, dz, order=1 ) zvec = np.linspace(zmin, zmax, Nz) xvec = np.linspace(xmin, xmax, Nx) beta = (1 - 1 / gamma ** 2) ** (1 / 2) t3 = time.time() if reuse_psi_grids == True: psi_s_grid = psi_s_grid_old psi_x_grid = psi_x_grid_old else: # Creating the potential grids zvec2 = np.linspace(2 * zmin, 2 * zmax, 2 * Nz) xvec2 = np.linspace(2 * xmin, 2 * xmax, 2 * Nx) zm2, xm2 = np.meshgrid(zvec2, xvec2, indexing="ij") # psi_s_grid = psi_s(zm2,xm2,beta) beta_grid = beta * np.ones(zm2.shape) with cf.ProcessPoolExecutor(max_workers=12) as executor: temp = executor.map(psi_s, zm2 / 2 / rho, xm2, beta_grid) psi_s_grid = np.array(list(temp)) temp2 = executor.map(psi_x, zm2 / 2 / rho, xm2, beta_grid) psi_x_grid = np.array(list(temp2)) t4 = time.time() if verbose: print("Depositting particles takes:", t2 - t1, "s") print("Computing potential grids take:", t4 - t3, "s") # Compute the wake via 2d convolution conv_s = oaconvolve(lambda_grid_filtered_prime, psi_s_grid, mode="same") conv_x = oaconvolve(lambda_grid_filtered_prime, psi_x_grid, mode="same") Ws_grid = (beta ** 2 / rho) * (conv_s) * (dz * dx) Wx_grid = (beta ** 2 / rho) * (conv_x) * (dz * dx) # Interpolate Ws and Wx everywhere within the grid Ws_interp = RectBivariateSpline(zvec, xvec, Ws_grid) Wx_interp = RectBivariateSpline(zvec, xvec, Wx_grid) r_e = 2.8179403227e-15 q_e = 1.602176634e-19 Nb = np.sum(charge_grid) / q_e kick_factor = r_e * Nb / gamma # Calculate the kicks at the paritcle locations delta_kick = kick_factor * Ws_interp.ev(z_b, x_b) xp_kick = kick_factor * Wx_interp.ev(z_b, x_b) return { "zvec": zvec, "xvec": xvec, "delta_kick": delta_kick, "xp_kick": xp_kick, "Ws_grid": Ws_grid, "Wx_grid": Wx_grid, "psi_s_grid": psi_s_grid, "psi_x_grid": psi_x_grid, "charge_grid": charge_grid, }
def spread(start_locations, frictiondist, friction): """ Total friction of the shortest accumulated friction path over a map with friction values from a source cell to cell under consideration :param start_locations: starting locations :type start_locations: Property :param frictiondist: initial friction :type frictiondist: Property :param friction: friction per cell :type friction: Property :returns: a property with total friction values :rtype: Property For concepts of this operation calculated on each agent see https://pcraster.geo.uu.nl/pcraster/4.3.0/documentation/pcraster_manual/sphinx/op_spread.html """ result_prop = Property('emptyspreadname', start_locations.pset_uuid, start_locations.space_domain, start_locations.shapes) todo = [] for idx in start_locations.values().values.keys(): start_locations_values = start_locations.values().values[idx] frictiondist_values = frictiondist.values().values[idx] friction_values = friction.values().values[idx] west = start_locations.space_domain.p1.xcoord[idx] north = start_locations.space_domain.p1.ycoord[idx] rows = int(start_locations.space_domain.row_discr[idx]) cols = int(start_locations.space_domain.col_discr[idx]) cellsize = (start_locations.space_domain.p2.xcoord[idx] - west) / cols clone = (rows, cols, cellsize, west, north) item = (idx, start_locations_values, frictiondist_values, friction_values, clone) todo.append(item) cpus = multiprocessing.cpu_count() tasks = len(todo) chunks = tasks // cpus with futures.ProcessPoolExecutor(max_workers=cpus) as ex: results = ex.map(_pspread, todo, chunksize=chunks) for result in results: result_prop.values().values[result[0]] = result[1] return result_prop # sequential for idx in start_locations.values().values.keys(): values = start_locations.values().values[idx] _set_current_clone(start_locations, idx) frictiondistvalues = frictiondist.values().values[idx] frictionvalues = friction.values().values[idx] arg1_raster = pcraster.numpy2pcr(pcraster.Nominal, values, -99999) #numpy.nan) frictiondist_raster = pcraster.numpy2pcr(pcraster.Scalar, frictiondistvalues, numpy.nan) friction_raster = pcraster.numpy2pcr(pcraster.Scalar, frictionvalues, numpy.nan) result_raster = pcraster.spread(arg1_raster, frictiondist_raster, friction_raster) result_item = pcraster.pcr2numpy(result_raster, numpy.nan) result_prop.values().values[idx] = result_item return result_prop
def get_dea_strain(self, imgs, output_result_image=False, check_visual_state=False, title="", parallel=True): """ Detect strain for the given set of images. :param imgs: A set of images (1 image per DEA) :param output_result_image: if True, the function returns a visualisation of the measured strain :param check_visual_state: Checks if any image has deviated too much from its reference :param title: A title to print on the result image. Can be a list of titles, one for each result image. :param parallel: Indicate if multiprocessing should be used to process images in parallel :return: four object containing the strain, center shift, result images, and visual_state for each of the n input images. 'strain' is a n-by-4 array with the following columns: [area strain, radial strain X, radial strain Y, average radial strain (sqrt of area strain)] 'center shift' is a n-by-2 array containing the shift in X and Y direction, in pixels (reference: top left corner). 'result images' is a n-element list of images with a visualization of the measured strain. 'visual' state is a n-element list of ints indicating 1 if an image is OK, or 0 if an image shows great deviation from its reference image. """ n_img = len(imgs) # TODO: keep old masks if new masks are rubbish # TODO: make sure the algorithm is robust to Nones if self._reference_images is not None and n_img != len( self._reference_images): raise Exception( "Need to specify one image for each reference image! Set elements to None where no image is" "available for a reference") # indication if any of the samples must be considered an outlier outlier = np.array([False] * len(imgs)) # check if image deviation is too large if check_visual_state: if self._reference_images is not None: img_dev = self.get_deviation_from_reference(imgs) self.logging.debug("Image deviations: {}".format(img_dev)) outlier = np.bitwise_or( outlier, np.array(img_dev) > self.image_deviation_threshold) # initialize lists to store results ellipses = [] if self._exclude_masks is None: self._exclude_masks = [None] * n_img masks = [] # get fit for each DEA if parallel: # Use executor in with statement to ensure processes are cleaned up promptly with futures.ProcessPoolExecutor(max_workers=n_img) as executor: # Start the load operations and mark each future with its index fits_results = executor.map(self.dea_fit_ellipse, imgs, self._exclude_masks) self.logging.debug( "Ellipse fitting in parallel. Submitted tasks to worker processes." ) for res in fits_results: ellipse, mask = res ellipses.append(ellipse) masks.append(mask) else: # don't use multi-processing self.logging.debug( "Sequential ellipse fitting (no multi-processing)") for i in range(n_img): ellipse, mask = self.dea_fit_ellipse(imgs[i], self._exclude_masks[i]) ellipses.append(ellipse) masks.append(mask) for i in range(n_img): # mark as outlier if no ellipse was found if ellipses[i] is None: outlier[i] = True # calculate strain and center shift ellipses_np = ellipses_to_numpy_array(ellipses) # get as numpy array xy_radii = ellipse_radius_at_angle(ellipses_np, self.query_angles) centers = ellipses_np[:, 0:2] # pseudo because we don't bother multiplying by pi; divide by 4 because ellipse uses diameter not radius pseudo_areas = np.prod(ellipses_np[:, 2:4], axis=1) / 4 # if there is not yet any reference for strain measurements, set these ellipses as the reference if self._reference_ellipses is None: if np.any(ellipses_np == np.nan) or any(outlier): self.logging.critical( "Invalid reference! Cannot proceed without a valid reference!" ) raise Exception("The given reference is not valid") # if not explicitly calling set_reference, print a warning that a new reference is being set if self.setting_reference: self.logging.info("Setting new strain reference.") else: self.logging.warning( "No strain reference has been set yet. The given images will be set as reference." ) self._reference_images = imgs self._reference_ellipses = ellipses self._reference_radii = xy_radii self._reference_centers = centers self._reference_pseudo_areas = pseudo_areas title = "Reference" # to be displayed on the result image # calculate strain and shift strain = xy_radii / self._reference_radii strain_area = pseudo_areas / self._reference_pseudo_areas strain_avg = np.sqrt(strain_area) strain_all = np.concatenate( (np.reshape(strain_area, (-1, 1)), strain, np.reshape(strain_avg, (-1, 1))), axis=1) strain_all = strain_all * 100 - 100 # convert to engineering strain center_shift = centers - self._reference_centers # check if strain is too large if check_visual_state: strain = strain * 100 - 100 # convert to engineering strain strain_out = np.bitwise_or( strain > self.outlier_strain_threshold, strain < self.outlier_negative_strain_threshold) strain_out = np.any(strain_out, axis=1) outlier = np.bitwise_or(outlier, strain_out) # TODO: make it so this doesn't crash if value are None or nan shift_out = np.abs(center_shift) > self.center_shift_threshold shift_out = np.any(shift_out, axis=1) outlier = np.bitwise_or(outlier, shift_out) visual_state = list(np.invert(outlier).astype(np.uint8)) else: visual_state = [None] * n_img # update exclude masks for i in range(n_img): # keep from saving this mask if it belongs to an outlier if masks[i] is not None and not outlier[i]: self._exclude_masks[i] = masks[i] if isinstance(title, str): title = [ title ] * n_img # make into a list with the same title for each image elif len(title) == 1 and n_img > 1: title = title * n_img # if single string in list, duplicate to match number of images res_imgs = None if output_result_image: # create a list of flags indicating the DEA state (as determined visually) or non to disable the indicator res_imgs = [ visualize_result(imgs[i], ellipses[i], tuple(xy_radii[i, :]), tuple(self.query_angles), self._reference_ellipses[i], strain_all[i, -1], title=title[i]) for i in range(n_img) ] return strain_all, center_shift, res_imgs, visual_state
def main(wxApp=False): global PRINT_LOCK, FPS, client, img_counter, start, BLUR_THRESHOLD PRINT_LOCK = threading.Lock() BLUR_THRESHOLD = Settings.blur_threshold FPS = Settings.frames_per_second CREDNTIALS = Settings.json_path client = vision.ImageAnnotatorClient.from_service_account_json(CREDNTIALS) img_counter = 0 start = 0 if not wxApp: wx_app = wx.App(0) wx_app.MainLoop() pic_message = wx.BusyInfo("Taking pictures...") queue = Queue() img_dict = dict() try: shutil.rmtree(Settings.save_path) except FileNotFoundError: pass finally: os.makedirs(Settings.save_path) captureImages_thread = threading.Thread(target=captureImages, args=(queue, 0, Settings.seconds_to_run)) captureImages_thread.daemon = True captureImages_thread.start() for process in range(Settings.number_of_processes): thread = threading.Thread(target=threader, args=(queue, img_dict)) thread.daemon = True thread.start() queue.join() captureImages_thread.join() del pic_message processing_message = wx.BusyInfo("Processing pictures...") img_dict = OrderedDict( sorted(img_dict.items(), reverse=True)[:Settings.max_pics_saved]) with cf.ProcessPoolExecutor(Settings.number_of_processes) as ex: ex.map(finalize_image, img_dict.items()) if len(img_dict.keys()): max_score = max(img_dict.keys()) final_image = Image.open(f"{Settings.save_path}/image_{max_score}.jpg") write_to_image(max_score, final_image, 0) for score_status, score_range in Settings.score_ranges.items(): if int(max_score * 1000) in score_range: write_to_image(score_status, final_image, 100) break del processing_message final_image.show() if not wxApp: del wx_app # if __name__ == '__main__': # global PRINT_LOCK, FPS, client, img_counter, start, BLUR_THRESHOLD # PRINT_LOCK = threading.Lock() # BLUR_THRESHOLD = Settings.blur_threshold # FPS = Settings.frames_per_second # CREDNTIALS = Settings.json_path # client = vision.ImageAnnotatorClient.from_service_account_json(CREDNTIALS) # img_counter = 0 # start = 0 # main()
def process_way(): """使用多进程""" workers = 10 with futures.ProcessPoolExecutor(workers) as executor: futs = {executor.submit(blocking_way) for _ in range(10)} return len([fut.result() for fut in futs])
def setUp(self): self.pool = futures.ProcessPoolExecutor(4) self.sc = pysparkling.Context(pool=self.pool, serializer=cloudpickle.dumps, deserializer=pickle.loads)
def download_many(cc_list): with futures.ProcessPoolExecutor() as executor: #<-스레드가 아니라 프로세스. 프로세스 수는 기본적으로 os.cpu_count()의 반환값을 사용하므로, 보통은 별도로 설정하지 않아도 된다. res=executor.map(download_one, sorted(cc_list)) return len(list(res))
def geraComando(i, j, k): cont = i * 25 + j * 5 + k #comando sem l2 # comand = "./dineroIV " + "-l1-usize " + str(capTot[i]) + "K" +" -l1-uassoc " + str(assoc[j]) + " -l1-ubsize " + str(tamLin[k])+ " -l1-uwalloc a " + "-l1-uwback n " + " -l1-uccc " + "-l1-urepl l " + "-maxtrace 101" + " " +"-trname mcf_f2b "+ "-informat s " + ">" +str(cont)+ "_" + str(capTot[i]) + "_" + str(assoc[j]) + "_" + str(tamLin[k]) + ".txt" #comando com l2 comand = "./dineroIV " + "-l1-usize " + str( capTot[i] ) + "K" + " -l1-uassoc " + str(assoc[j]) + " -l1-ubsize " + str( tamLin[k] ) + " -l1-uwalloc a " + "-l1-uwback n " + " -l1-uccc " + "-l1-urepl l " + "-l2-usize " + "256K" + " -l2-uassoc " + str( assoc[j] ) + " -l2-ubsize " + str( tamLin[k] ) + " -l2-uwalloc a " + "-l2-uwback n " + " -l2-uccc " + "-l2-urepl l " + "-maxtrace 101" + " " + "-trname mcf_f2b " + "-informat s " + ">" + str( cont) + "_" + str(capTot[i]) + "_" + str(assoc[j]) + "_" + str( tamLin[k]) + ".txt" print(comand) os.system(comand) executor = futures.ProcessPoolExecutor() for i in range(len(capTot)): for j in range(len(assoc)): for k in range(len(tamLin)): executor.submit(geraComando, i, j, k) #./dineroIV -l1-dsize 2K -l1-isize 2K -l1-ibsize 16 -l1-dbsize 8 -l2-usize 1m -l2-ubsize 64 -l2-usbsize 16 -l2-uassoc 4 -l2-urepl l -l2-ufetch s -l2-uwalloc a -l2-uwback a -informat s -maxtrace 30 -trname ../dineroData/mcf_f2b #./dineroIV -l1-usize "$i"K -l1-ubsize $k -l1-uassoc $j -l1-uwalloc a -l1-uwback a -l1-uccc -l2-usize 256K -l2-ubsize $k -l2-usbsize 8 -l2-uassoc 4 - -l2-ufetch s -l2-uwalloc a -l2-uwback a -l2-uccc -maxtrace 101 -trname gcc_f2b
theta[i] = fit_spot(spot) return theta def fit_spots_parallel(spots, async=False): n_workers = max(1, int(0.75 * _multiprocessing.cpu_count())) n_spots = len(spots) n_tasks = 100 * n_workers spots_per_task = [ int(n_spots / n_tasks + 1) if _ < n_spots % n_tasks else int(n_spots / n_tasks) for _ in range(n_tasks) ] start_indices = _np.cumsum([0] + spots_per_task[:-1]) fs = [] executor = _futures.ProcessPoolExecutor(n_workers) for i, n_spots_task in zip(start_indices, spots_per_task): fs.append(executor.submit(fit_spots, spots[i:i + n_spots_task])) if async: return fs with _tqdm(total=n_tasks, unit='task') as progress_bar: for f in _futures.as_completed(fs): progress_bar.update() return fits_from_futures(fs) def fit_spots_gpufit(spots): size = spots.shape[1] initial_parameters = initial_parameters_gpufit(spots, size) spots.shape = (len(spots), (size * size)) model_id = gf.ModelID.GAUSS_2D_ELLIPTIC
from concurrent import futures import os def task(n): return (n, os.getpid()) ex = futures.ProcessPoolExecutor(max_workers=2) results = ex.map(task, range(5, 0, -1)) for n, pid in results: print('ran task {} in process {}'.format(n, pid))
record = eval(record[1]) print('{} {} {} {} {} {} {} {}'.format(e, record[0], record[1], record[2], record[3], record[4], record[5], record[6])) JFm.append(record[0]) Jm.append(record[1]) Jr.append(record[2]) Jd.append(record[3]) Fm.append(record[4]) Fr.append(record[5]) Fd.append(record[6]) print('=========> sort with J&F: <===========') argidx = np.argmax(np.array(JFm)) print('{} {} {} {} {} {} {} {}'.format(epochs[argidx], JFm[argidx], Jm[argidx], Jr[argidx], Jd[argidx], Fm[argidx], Fr[argidx], Fd[argidx])) print('=========> sort with Jm: <===========') argidx = np.argmax(np.array(Jm)) print('{} {} {} {} {} {} {} {}'.format(epochs[argidx], JFm[argidx], Jm[argidx], Jr[argidx], Jd[argidx], Fm[argidx], Fr[argidx], Fd[argidx])) base_path = join('result', args.dataset) epochs = listdir(base_path) print('total {} epochs'.format(len(epochs))) # multi-process evaluation if args.dataset in ['DAVIS2016', 'DAVIS2017']: with futures.ProcessPoolExecutor(max_workers=args.num_threads) as executor: fs = [executor.submit(eval_davis, e) for e in epochs] print('done') extract_davis(epochs) else: raise ValueError('not supported data')
class LocalJob(BaseJob): """Local QISKit SDK Job class. Attributes: _executor (futures.Executor): executor to handle asynchronous jobs """ if sys.platform == 'darwin': _executor = futures.ThreadPoolExecutor() else: _executor = futures.ProcessPoolExecutor() def __init__(self, fn, q_job): super().__init__() self._q_job = q_job self._backend_name = q_job.backend.name self._future = self._executor.submit(fn, q_job) def result(self, timeout=None): # pylint: disable=arguments-differ """ Get job result. The behavior is the same as the underlying concurrent Future objects, https://docs.python.org/3/library/concurrent.futures.html#future-objects Args: timeout (float): number of seconds to wait for results. Returns: Result: Result object Raises: concurrent.futures.TimeoutError: if timeout occured. concurrent.futures.CancelledError: if job cancelled before completed. """ return self._future.result(timeout=timeout) def cancel(self): return self._future.cancel() @property def status(self): _status_msg = None # order is important here if self.running: _status = JobStatus.RUNNING elif not self.done: _status = JobStatus.QUEUED elif self.cancelled: _status = JobStatus.CANCELLED elif self.done: _status = JobStatus.DONE elif self.exception: _status = JobStatus.ERROR _status_msg = str(self.exception) else: raise LocalJobError('Unexpected behavior of {0}'.format( self.__class__.__name__)) return {'status': _status, 'status_msg': _status_msg} @property def running(self): return self._future.running() @property def done(self): """ Returns True if job successfully finished running. Note behavior is slightly different than Future objects which would also return true if successfully cancelled. """ return self._future.done() and not self._future.cancelled() @property def cancelled(self): return self._future.cancelled() @property def exception(self): """ Return Exception object if exception occured else None. Returns: Exception: exception raised by attempting to run job. """ return self._future.exception(timeout=0) @property def backend_name(self): """ Return backend name used for this job """ return self._backend_name
def from_PTB_basic_FS( cls, filesys: "fs.base.FS", name: typing.Optional[str] = None, glob_str: str = "**/*.psd", disambiguate_IDs_by_path: bool = False, version: Version = Version("0.0.0"), container_version: Version = Version("0.0.0"), process_num: int = typing.cast(int, abctk.config.CONF_DEFAULT["max_process_num"]), tqdm_buffer: typing.Optional[typing.TextIO] = None, uniformly_with_ID: bool = True, ): # ------------ # 1. Collect files and their info # ------------ glob_matches = filesys.glob(glob_str) file_paths = tuple( match.path for match in glob_matches ) file_sizes = tuple( filesys.getinfo(path, ("details", )).size for path in file_paths ) file_num = len(file_paths) file_size_sum = sum(file_sizes) logger.info( f"# of files to be loaded: {file_num}, " f"the total size: {humanize.naturalsize(file_size_sum)}" ) # ------------ # 2. Making a file reading iterator # ------------ def _read_file(filesys: "fs.base.FS", path: str): with filesys.open(path, "r") as f: return f.read() # === END === file_contents = ( _read_file(filesys, path) for path in file_paths ) # As iterator, actual IO made delayed # ------------ # 3. Launch a multiprocessing context # ----------- result_tree_iters = list() with cf.ProcessPoolExecutor( max_workers = process_num ) as executor: logger.info(f"Multiprocessing pool created, number of processes: {process_num}") with tqdm.tqdm( total = file_size_sum, desc = "Reading & parsing treebank files", unit = "B", unit_scale = True, file = tqdm_buffer, disable = tqdm_buffer is None, ) as bar: for _, size, _, trees in executor.map( functools.partial( _load_file, disambiguate_IDs_by_path = disambiguate_IDs_by_path, uniformly_with_ID = uniformly_with_ID, ), zip( file_paths, file_sizes, file_contents, ) ): bar.update(size) result_tree_iters.append(trees) return cls( name if name is not None else str(filesys), version, container_version, index = dict(itertools.chain.from_iterable(result_tree_iters)) )
return matrix def filter_cluser(DS): DS = DS_path + DS index_path = DS + '/giggle_index' results = '' ith_file = 0 files = [] for file in sorted(os.listdir(DS)): if not file.endswith('.gz'): continue ith_file += 1 files += [file] file_path = os.path.join(DS, file) cmdline = f'giggle search -i {index_path} -q {file_path} -s' result = subprocess.check_output(cmdline, shell=True).decode('utf-8').replace( '\t\n', '\n') results = results + file + '\t' + (f'\n{file}\t').join( result.split("\n")[1:-1]) + "\n" matrix = pd.DataFrame([], index=files) if ith_file >= 3: matrix = inconsistent_filter(results, files) return matrix, DS.replace(DS_path, '') with futures.ProcessPoolExecutor(max_workers=4) as executor: for future in executor.map(filter_cluser): future.to_csv(args.output, index=False)
im = im.resize((100, 100)) except Exception as e: print(e) is_invalid = True if is_invalid: shutil.move(file_path, dst_out_path) if __name__=='__main__': cid3_paths = os.listdir(file_src) path_num = len(cid3_paths) thread_num = 8 with futures.ProcessPoolExecutor(max_workers=thread_num) as executor: future_list = list() for i in range(thread_num): begin = int(path_num / thread_num * i) end = int(path_num / thread_num * (i + 1)) thread_paths=cid3_paths[begin:end] future_list.append(executor.submit(split_similarpics, thread_paths)) # future_list.append(executor.submit(split_invalid_pic, thread_paths)) # future_list.append(executor.submit(split_over_num, thread_paths)) for future in futures.as_completed(future_list): if future.exception(): print(future.exception())
def execute(self): """ Raises: NotADirectoryError: When the event data directory does not exist. FileNotFoundError: When the the shake_result HDF file does not exist. """ install_path, data_path = get_config_paths() datadir = os.path.join(data_path, self._eventid, 'current', 'products') if not os.path.isdir(datadir): raise NotADirectoryError('%s is not a valid directory.' % datadir) datafile = os.path.join(datadir, 'shake_result.hdf') if not os.path.isfile(datafile): raise FileNotFoundError('%s does not exist.' % datafile) # Open the ShakeMapOutputContainer and extract the data container = ShakeMapOutputContainer.load(datafile) if container.getDataType() != 'grid': raise NotImplementedError('mapping module can only operate on ' 'gridded data, not sets of points') # get the path to the products.conf file, load the config config_file = os.path.join(install_path, 'config', 'products.conf') spec_file = get_configspec('products') validator = get_custom_validator() config = ConfigObj(config_file, configspec=spec_file) results = config.validate(validator) check_extra_values(config, self.logger) if not isinstance(results, bool) or not results: config_error(config, results) # create contour files self.logger.debug('Mapping...') # get the filter size from the products.conf filter_size = config['products']['contour']['filter_size'] # get the operator setting from config operator = config['products']['mapping']['operator'] # get all of the pieces needed for the mapping functions layers = config['products']['mapping']['layers'] if 'topography' in layers and layers['topography'] != '': topofile = layers['topography'] else: topofile = None if 'roads' in layers and layers['roads'] != '': roadfile = layers['roads'] else: roadfile = None if 'faults' in layers and layers['faults'] != '': faultfile = layers['faults'] else: faultfile = None if 'countries' in layers and layers['countries'] != '': countries_file = layers['countries'] else: countries_file = None if 'states_provs' in layers and layers['states_provs'] != '': states_provs_file = layers['states_provs'] else: states_provs_file = None if 'oceans' in layers and layers['oceans'] != '': oceans_file = layers['oceans'] else: oceans_file = None if 'lakes' in layers and layers['lakes'] != '': lakes_file = layers['lakes'] else: lakes_file = None # Get the number of parallel workers max_workers = config['products']['mapping']['max_workers'] # Reading HDF5 files currently takes a long time, due to poor # programming in MapIO. To save us some time until that issue is # resolved, we'll coarsely subset the topo grid once here and pass # it into both mapping functions # get the bounds of the map info = container.getMetadata() xmin = info['output']['map_information']['min']['longitude'] xmax = info['output']['map_information']['max']['longitude'] ymin = info['output']['map_information']['min']['latitude'] ymax = info['output']['map_information']['max']['latitude'] dy = float( info['output']['map_information']['grid_spacing']['latitude']) dx = float( info['output']['map_information']['grid_spacing']['longitude']) padx = 5 * dx pady = 5 * dy sxmin = float(xmin) - padx sxmax = float(xmax) + padx symin = float(ymin) - pady symax = float(ymax) + pady sampledict = GeoDict.createDictFromBox(sxmin, sxmax, symin, symax, dx, dy) if topofile: topogrid = read(topofile, samplegeodict=sampledict, resample=False, doPadding=True, padValue=0.0) else: tdata = np.full([sampledict.ny, sampledict.nx], 0.0) topogrid = Grid2D(data=tdata, geodict=sampledict) model_config = container.getConfig() imtlist = container.getIMTs() textfile = os.path.join( get_data_path(), 'mapping', 'map_strings.' + config['products']['mapping']['language']) text_dict = get_text_strings(textfile) if config['products']['mapping']['fontfamily'] != '': matplotlib.rcParams['font.family'] = \ config['products']['mapping']['fontfamily'] matplotlib.rcParams['axes.unicode_minus'] = False allcities = Cities.fromDefault() states_provs = None countries = None oceans = None lakes = None faults = None roads = None if states_provs_file is not None: states_provs = ShapelyFeature( Reader(states_provs_file).geometries(), ccrs.PlateCarree(), facecolor='none') elif 'CALLED_FROM_PYTEST' not in os.environ: states_provs = cfeature.NaturalEarthFeature( category='cultural', name='admin_1_states_provinces_lines', scale='10m', facecolor='none') # The feature constructor doesn't necessarily download the # data, but we want it to so that multiple threads don't # try to do it at once when they actually access the data. # So below we just call the geometries() method to trigger # the download if necessary. _ = states_provs.geometries() if countries_file is not None: countries = ShapelyFeature(Reader(countries_file).geometries(), ccrs.PlateCarree(), facecolor='none') elif 'CALLED_FROM_PYTEST' not in os.environ: countries = cfeature.NaturalEarthFeature(category='cultural', name='admin_0_countries', scale='10m', facecolor='none') _ = countries.geometries() if oceans_file is not None: oceans = ShapelyFeature(Reader(oceans_file).geometries(), ccrs.PlateCarree(), facecolor=WATERCOLOR) elif 'CALLED_FROM_PYTEST' not in os.environ: oceans = cfeature.NaturalEarthFeature(category='physical', name='ocean', scale='10m', facecolor=WATERCOLOR) _ = oceans.geometries() if lakes_file is not None: lakes = ShapelyFeature(Reader(lakes_file).geometries(), ccrs.PlateCarree(), facecolor=WATERCOLOR) elif 'CALLED_FROM_PYTEST' not in os.environ: lakes = cfeature.NaturalEarthFeature(category='physical', name='lakes', scale='10m', facecolor=WATERCOLOR) _ = lakes.geometries() if faultfile is not None: faults = ShapelyFeature(Reader(faultfile).geometries(), ccrs.PlateCarree(), facecolor='none') if roadfile is not None: roads = ShapelyFeature(Reader(roadfile).geometries(), ccrs.PlateCarree(), facecolor='none') alist = [] llogo = config['products']['mapping'].get('license_logo') or None ltext = config['products']['mapping'].get('license_text') or None for imtype in imtlist: component, imtype = imtype.split('/') comp = container.getComponents(imtype)[0] d = { 'imtype': imtype, 'topogrid': topogrid, 'allcities': allcities, 'states_provinces': states_provs, 'countries': countries, 'oceans': oceans, 'lakes': lakes, 'roads': roads, 'faults': faults, 'datadir': datadir, 'operator': operator, 'filter_size': filter_size, 'info': info, 'component': comp, 'imtdict': container.getIMTGrids(imtype, comp), 'ruptdict': copy.deepcopy(container.getRuptureDict()), 'stationdict': container.getStationDict(), 'config': model_config, 'tdict': text_dict, 'display_magnitude': self.display_magnitude, 'pdf_dpi': config['products']['mapping']['pdf_dpi'], 'img_dpi': config['products']['mapping']['img_dpi'], 'license_logo': llogo, 'license_text': ltext, } alist.append(d) if imtype == 'MMI': g = copy.deepcopy(d) g['imtype'] = 'thumbnail' alist.append(g) h = copy.deepcopy(d) h['imtype'] = 'overlay' alist.append(h) self.contents.addFile('intensityMap', 'Intensity Map', 'Map of macroseismic intensity.', 'intensity.jpg', 'image/jpeg') self.contents.addFile('intensityMap', 'Intensity Map', 'Map of macroseismic intensity.', 'intensity.pdf', 'application/pdf') self.contents.addFile('intensityThumbnail', 'Intensity Thumbnail', 'Thumbnail of intensity map.', 'pin-thumbnail.png', 'image/png') self.contents.addFile( 'intensityOverlay', 'Intensity Overlay and World File', 'Macroseismic intensity rendered as a ' 'PNG overlay and associated world file', 'intensity_overlay.png', 'image/png') self.contents.addFile( 'intensityOverlay', 'Intensity Overlay and World File', 'Macroseismic intensity rendered as a ' 'PNG overlay and associated world file', 'intensity_overlay.pngw', 'text/plain') else: fileimt = oq_to_file(imtype) self.contents.addFile(fileimt + 'Map', fileimt.upper() + ' Map', 'Map of ' + imtype + '.', fileimt + '.jpg', 'image/jpeg') self.contents.addFile(fileimt + 'Map', fileimt.upper() + ' Map', 'Map of ' + imtype + '.', fileimt + '.pdf', 'application/pdf') if max_workers > 0: with cf.ProcessPoolExecutor(max_workers=max_workers) as ex: results = ex.map(make_map, alist) list(results) else: for adict in alist: make_map(adict) container.close()
def leaderboard(pool, n, focus='hydra.py'): '' # Saving data from games data = {name: defaultdict(int) for name in pool} # Multiprocessing games with processor.ProcessPoolExecutor() as executor: # Saving data outputs = [] games = [] # Generating and scheduling games for player1 in pool: for player2 in pool: if player1 != player2: for _ in range(n): games.append( executor.submit(new_game, player1, player2)) print(f'{len(games)} games scheduled') # Executing games in parallel for game in processor.as_completed(games): output = game.result() outputs.append(output) print( f"{len(outputs)} games completed: {output['players'][0]} vs {output['players'][1]}" ) # Updating wins/losses for output in outputs: for player in output['players']: if player == output['winner']: data[player]['wins'] += 1 elif player == output['loser']: data[player]['losses'] += 1 # Printing which agents beat focus if player == focus: print(f'{focus} was beat by {output["winner"]}') elif output['draw']: data[player]['draws'] += 1 data[player]['games'] += 1 data[player]['win%'] = round( data[player]['wins'] / data[player]['games'], 3) # Scoring agents (win% or wins - losses) for player in pool: data[player]['score'] = data[player]['wins'] - data[player]['losses'] data[player]['score'] = round(data[player]['score'], 3) # Writing data to CSV file pool.sort(key=lambda name: data[name]['score'], reverse=True) with open('leaderboard/leaderboard.csv', 'w') as file: file.write(f"{','.join(data[pool[0]].keys())}") for name in pool: output = f'\n' for item in data[name]: output += f'{data[name][item]},' output = output[:-1] file.write(output) # Generating PrettyTable and dumping to file with open('leaderboard/leaderboard_table.txt', 'w') as file: table = from_csv(open('leaderboard/leaderboard.csv')) table.align = 'l' string = table.get_string().replace('-', '_').replace('+', '|') file.write(string)
def run(self): from production import util # copy the script to the temp folder and replace the shebang file_dir = os.path.dirname(os.path.abspath(__file__)) util.copy_and_replace( os.path.join(file_dir, 'solve_subproblems.py'), os.path.join(self.tmp_folder, 'solve_subproblems.py')) with open(self.config_path) as f: config = json.load(f) initial_block_shape = config['block_shape'] n_threads = config['n_threads'] roi = config.get('roi', None) # get number of blocks factor = 2**self.scale block_shape = [factor * bs for bs in initial_block_shape] shape = z5py.File(self.graph_path).attrs['shape'] blocking = nifty.tools.blocking([0, 0, 0], shape, block_shape) # check if we have a roi and adjuse the block list if we do if roi is None: n_blocks = blocking.numberOfBlocks block_list = list(range(n_blocks)) else: block_list = blocking.getBlockIdsOverlappingBoundingBox( roi[0], roi[1], [0, 0, 0]).tolist() n_blocks = len(block_list) # find the actual number of jobs and prepare job configs n_jobs = min(n_blocks, self.max_jobs) self._prepare_jobs(n_jobs, block_list, initial_block_shape, n_threads) # submit the jobs if self.run_local: # this only works in python 3 ?! with futures.ProcessPoolExecutor(n_jobs) as tp: tasks = [ tp.submit(self._submit_job, job_id, n_threads) for job_id in range(n_jobs) ] [t.result() for t in tasks] else: for job_id in range(n_jobs): self._submit_job(job_id, n_threads) # wait till all jobs are finished if not self.run_local: util.wait_for_jobs('papec') # check the job outputs processed_blocks, times = self._collect_outputs(block_list) assert len(processed_blocks) == len(times) success = len(processed_blocks) == n_blocks # write output file if we succeed, otherwise write partial # success to different file and raise exception if success: out = self.output() # TODO does 'out' support with job? fres = out.open('w') json.dump({'times': times}, fres) fres.close() else: log_path = os.path.join( self.tmp_folder, 'solve_subproblems_s%i_partial.json' % self.scale) with open(log_path, 'w') as out: json.dump( { 'times': times, 'processed_blocks': processed_blocks }, out) raise RuntimeError("SolveSubproblemTask failed, " "%i / %i blocks processed, " "serialized partial results to %s" % (len(processed_blocks), n_blocks, log_path))
def download_many(cc_list): with futures.ProcessPoolExecutor() as executor: # <1> res = executor.map(download_one, sorted(cc_list)) return len(list(res))
def main(): import os import sys model_ext = '.ph' n_best = 15 ps.Band.load_settings() is_set_model = False parser = get_parser() args, unknownargs = parser.parse_known_args() path = os.getcwd() if args.path: path = os.path.expanduser(args.path) # print(f'-p: {path}') # Set model names names = [] if args.input: for arg in args.input: names.extend(arg2names(arg=arg, path=path, ext=model_ext)) is_set_model = True names = list(set(names)) # with unique values if len(names) == 0 and not is_set_model: # run for all files in the path names = ps.path.get_model_names(path, model_ext) if len(names) == 0: print(f'PATH: {path}') print(f'-i: {args.input}') print('I do not know the models for fitting. Please use the key -i MODEL or -i *R500* ') parser.print_help() sys.exit(2) # Set band names bnames = [] if args.bnames: for bname in args.bnames.split('-'): if not ps.band.is_exist(bname): print('No such band: ' + bname) parser.print_help() sys.exit(2) bnames.append(bname) # Get observations observations = ps.cb.observations(args) if observations is None: print('No obs data. Use key: -c: ') parser.print_help() sys.exit(2) # curves curves_o = merge_obs(observations, ps.SetLightCurve) vels_o = merge_obs(observations, ps.vel.SetVelocityCurve) # set bands as observed if curves_o is not None and len(bnames) == 0: bnames = [bn for bn in curves_o.BandNames if ps.band.is_exist(bn)] # Set distance and redshift is_sigma = args.is_fit_sigmas z = args.redshift t_diff = args.t_diff if args.distance is not None: print("Fit magnitudes on z={0:F} at distance={1:E}".format(z, args.distance)) if z > 0: print(" Cosmology D(z)={0:E} Mpc".format(ps.cosmology_D_by_z(z))) else: distance = 10 # pc if z > 0: distance = ps.cosmology_D_by_z(z) * 1e6 print("Fit magnitudes on z={0:F} with cosmology D(z)={1:E} pc".format(z, distance)) args.distance = distance if is_sigma: print("Fit magnitudes with model uncertainties.") print("Color excess E(B-V) ={:f}".format(args.color_excess)) # Time limits for models tlim = (0, float('inf')) if args.tlim: tlim = list(map(float, args.tlim.replace('\\', '').split(':'))) print('Time limits for models: {}'.format(':'.join(map(str, tlim)))) # The fit engine fitter = engines(args.engine) fitter.is_info = args.is_not_quiet # fitter = FitMPFit(is_debug=args.is_not_quiet) # fitter.is_debug = args.is_not_quiet if args.is_not_quiet: fitter.print_parameters() # The filter results by tshift if args.dtshift: dtshift = ps.str2interval(args.dtshift, llim=float("-inf"), rlim=float('inf')) else: dtshift = (float("-inf"), float("inf")) # tshift = 0. res_models = {} vels_m = {} res_chi = {} if len(names) == 1: name = names[0] if args.is_not_quiet: if tlim is not None: print("Fitting for model %s %s for %s moments" % (path, name, tlim)) else: print("Fitting for model %s %s, tweight= %f" % (path, name, args.tweight)) # curves_m = lcf.curves_compute(name, path, bnames, z=args.redshift, distance=args.distance, # t_beg=tlim[0], t_end=tlim[1], t_diff=t_diff) # res = fitter.fit_curves(curves_o, curves_m) if vels_o is None: curves_m, res, res_full = fit_mfl(args, curves_o, bnames, fitter, name, path, t_diff, tlim, is_sigma) else: curves_m, res, vel_m = fit_mfl_vel(args, curves_o, vels_o, bnames, fitter, name, path, t_diff, tlim, is_sigma, A=args.tweight) vels_m[name] = vel_m print("{}: t ime shift = {:.2f}+/-{:.4f} Measure: {:.4f} {}". format(name, res.tshift, res.tsigma, res.measure, res.comm)) # best_tshift = res.tshift res_models[name] = curves_m res_chi[name] = res res_sorted = res_chi elif len(names) > 1: if args.nodes > 1: print("Run parallel fitting: nodes={}, models {}".format(args.nodes, len(names))) with futures.ProcessPoolExecutor(max_workers=args.nodes) as executor: if vels_o is None: future_to_name = { executor.submit(fit_mfl, args, curves_o, bnames, fitter, n, path, t_diff, tlim, is_sigma, is_spline=args.is_spline): n for n in names } else: future_to_name = { executor.submit(fit_mfl_vel, args, curves_o, vels_o, bnames, fitter, n, path, t_diff, tlim, is_sigma, is_spline=args.is_spline): n for n in names } i = 0 for future in futures.as_completed(future_to_name): i += 1 name = future_to_name[future] try: data = future.result() except Exception as exc: print('%r generated an exception: %s' % (name, exc)) else: res_models[name], res, vels_m[name] = data res_chi[name] = res print("[{}/{}] {:30s} -> {}".format(i, len(names), name, res.comm)) else: for i, name in enumerate(names, start=1): txt = "Fitting [{}] for model {:30s} [{}/{}]".format(fitter.Name, name, i, len(names)) if args.is_not_quiet: print(txt) else: sys.stdout.write(u"\u001b[1000D" + txt) sys.stdout.flush() if vels_o is None: curves_m, res, res_full = fit_mfl(args, curves_o, bnames, fitter, name, path, t_diff, tlim, is_sigma, is_spline=args.is_spline) else: curves_m, res, vel_m = fit_mfl_vel(args, curves_o, vels_o, bnames, fitter, name, path, t_diff, tlim, is_sigma, is_spline=args.is_spline, A=args.tweight) vels_m[name] = vel_m res_models[name] = curves_m res_chi[name] = res # select with dtshift res_chi_sel = {} for k, v in res_chi.items(): if dtshift[0] < v.tshift < dtshift[1]: res_chi_sel[k] = v res_chi = res_chi_sel # sort with measure res_sorted = OrderedDict(sorted(res_chi.items(), key=lambda kv: kv[1].measure)) else: print("No any data about models. Path: {}".format(path)) parser.print_help() sys.exit(2) # print results print("\n Results (tshift in range:{:.2f} -- {:.2f}".format(dtshift[0], dtshift[1])) print("N || {:40s} ||{:18s}|| {:10}".format('Model', 'dt+-t_err', 'Measure')) for i, (k, v) in enumerate(res_sorted.items(), start=1): print("{} || {:40s} || {:7.2f}+/-{:7.4f} || {:.4f} || {}".format(i, k, v.tshift, v.tsigma, v.measure, v.comm)) if len(res_sorted) >= n_best: # plot chi squared plot_squared_grid(res_sorted, path, is_not_quiet=args.is_not_quiet) plot_chi_par(res_sorted, path) # plot only Nbest modeles # while len(res_sorted) > Nbest: # res_sorted.popitem() # plot_squared_3d(None, res_sorted, path, p=('R', 'M', 'E'), is_polar=True) best_mdl, res = ps.first(res_sorted.items()) # res = first(res_sorted.values())[0] print("Best fit model:") print("{}: time shift = {:.2f}+/-{:.4f} Measure: {:.4f}".format(best_mdl, res.tshift, res.tsigma, res.measure)) print("{}: ".format(best_mdl), res.comm) # shift observational data # curves_o.set_tshift(best_tshift) # plot only NbestPlot modeles while len(res_sorted) > args.plotnbest: res_sorted.popitem() if vels_o is not None and vels_o.Length > 0: # vel_o.tshift = best_tshift fig = plot_curves_vel(curves_o, vels_o, res_models, res_sorted, vels_m) else: fig = plot_curves(curves_o, res_models, res_sorted, xlim=tlim) if args.save_file is not None: fsave = args.save_file if len(os.path.dirname(fsave)) == 0: fsave = os.path.expanduser("~/{}".format(fsave)) if not fsave.endswith('.pdf'): fsave += '.pdf' print("Save plot to {0}".format(fsave)) fig.savefig(fsave, bbox_inches='tight') else: from matplotlib import pyplot as plt # plt.subplots_adjust(left=0.07, right=0.96, top=0.97, bottom=0.06) plt.ion() plt.show() plt.pause(0.0001) print('') input("===> Hit <return> to quit")
class SimulatorsJob(BaseJob): """SimulatorsJob class. Attributes: _executor (futures.Executor): executor to handle asynchronous jobs """ if sys.platform in ['darwin', 'win32']: _executor = futures.ThreadPoolExecutor() else: _executor = futures.ProcessPoolExecutor() def __init__(self, backend, job_id, fn, qobj): super().__init__(backend, job_id) self._fn = fn self._qobj = qobj self._future = None def submit(self): """Submit the job to the backend for execution. Raises: QobjValidationError: if the JSON serialization of the Qobj passed during construction does not validate against the Qobj schema. JobError: if trying to re-submit the job. """ if self._future is not None: raise JobError("We have already submitted the job!") validate_qobj_against_schema(self._qobj) self._future = self._executor.submit(self._fn, self._job_id, self._qobj) @requires_submit def result(self, timeout=None): # pylint: disable=arguments-differ """Get job result. The behavior is the same as the underlying concurrent Future objects, https://docs.python.org/3/library/concurrent.futures.html#future-objects Args: timeout (float): number of seconds to wait for results. Returns: qiskit.Result: Result object Raises: concurrent.futures.TimeoutError: if timeout occurred. concurrent.futures.CancelledError: if job cancelled before completed. """ return self._future.result(timeout=timeout) @requires_submit def cancel(self): return self._future.cancel() @requires_submit def status(self): """Gets the status of the job by querying the Python's future Returns: qiskit.providers.JobStatus: The current JobStatus Raises: JobError: If the future is in unexpected state concurrent.futures.TimeoutError: if timeout occurred. """ # The order is important here if self._future.running(): _status = JobStatus.RUNNING elif self._future.cancelled(): _status = JobStatus.CANCELLED elif self._future.done(): _status = JobStatus.DONE if self._future.exception( ) is None else JobStatus.ERROR else: # Note: There is an undocumented Future state: PENDING, that seems to show up when # the job is enqueued, waiting for someone to pick it up. We need to deal with this # state but there's no public API for it, so we are assuming that if the job is not # in any of the previous states, is PENDING, ergo INITIALIZING for us. _status = JobStatus.INITIALIZING return _status def backend(self): """Return the instance of the backend used for this job.""" return self._backend def qobj(self): """Return the Qobj submitted for this job. Returns: Qobj: the Qobj submitted for this job. """ return self._qobj
import logging import time import concurrent.futures as f import multiprocessing as m m.log_to_stderr(logging.DEBUG) def fn(n): print("start", n) time.sleep(1) return n * n logging.basicConfig(level=logging.DEBUG) print("----------------------------------------") with f.ProcessPoolExecutor() as e: futs = [] for i in range(7): futs.append(e.submit(fn, i)) for fut in f.as_completed(futs): print(fut.result()) print("----------------------------------------")
display_step=1, save_files=True, file_name=modelname, save_model=True, model_name=modelname, random_state=1) # evaluate: niters is the number of times you evaluate the network for # a single sample. higher niters = better resolved distribution of predictions niters = 100 bl, bm, bu, results = bmodel.bayesian_predict(x_test, n_iters=niters, random_state=1) # bayesian_predict returns 25, 50, 75 percentile and results dict with all predictions # evaluate performance on median predictions acc = sklearn.metrics.accuracy_score(y_test, bm.flatten()) print('Ye bmodel ka Accuracy', acc) # collect_tree_predictions gathers predictions in results dict # in a more intuitive way for easy plotting, etc p = bmodel.collect_tree_predictions(results['predictions']) a = time() with cf.ProcessPoolExecutor() as p: p.submit(model_train1) p.submit(model_train2) b = time() print("Total time taken:", b - a)
def test_max_workers_too_large(self): with self.assertRaisesRegex(ValueError, "max_workers must be <= 61"): futures.ProcessPoolExecutor(max_workers=62)
def update_fitness(INFO): import concurrent.futures as cf with cf.ProcessPoolExecutor() as e: for result_list in e.map(proc_worker, INFO): for l in result_list: print(l)