def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): dir: the directory to byte-compile maxlevels: maximum recursion level (default 10) ddir: the directory that will be prepended to the path to the file as it is compiled into each byte-code file. force: if True, force compilation, even if timestamps are up-to-date quiet: full output with False or 0, errors only with 1, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked """ ProcessPoolExecutor = None if workers is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') elif workers != 1: try: # Only import when needed, as low resource platforms may # fail to import it from concurrent.futures import ProcessPoolExecutor except ImportError: workers = 1 files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, ddir=ddir) success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: workers = workers or None with ProcessPoolExecutor(max_workers=workers) as executor: results = executor.map( partial( _compile_file_tuple, force=force, rx=rx, quiet=quiet, legacy=legacy, optimize=optimize, invalidation_mode=invalidation_mode, ), files_and_ddirs) success = min(results, default=True) else: for file, dfile in files_and_ddirs: if not compile_file(file, dfile, force, rx, quiet, legacy, optimize, invalidation_mode): success = False return success
import numpy as np import librosa.feature import librosa from concurrent.futures import ProcessPoolExecutor from local_loader import d, key_map from sklearn import svm from sklearn.multiclass import OneVsOneClassifier counter = 0 m = OneVsOneClassifier(svm.SVC(decision_function_shape='ovo', kernel='rbf')) # train with ProcessPoolExecutor() as executor: X = np.empty((0, 12 * d.window)) keys = [] for song, key, file, txt in executor.map(d.__getitem__, range(1, 10)): X2 = np.empty((0, 12 * d.window)) for idx, item in enumerate(song): X2 = np.concatenate((X2, item.reshape((1, 120))), axis=0) keys.extend(key) if X2.shape[0] > len(key): X2 = X2[:len(key), :] else: key = key[:X2.shape[0]]
import time, os, requests, json, re, pymysql import numpy as np import pandas as pd from hashlib import md5 from sqlalchemy import create_engine from datetime import datetime from concurrent.futures import ProcessPoolExecutor from my_toolkit import sql_write_read domain_name = "http://amazon.yibainetwork.com/" api = "services/amazon/amazonkeylisting/run" secret_key = '!@#$%~Ybai7895&$^^GoPingRun?' PROCESS_POOL = ProcessPoolExecutor(4) country_name_dict = { '英国': 'UK', '美国': 'US', '法国': 'FR', '德国': 'DE', '意大利': 'IT', '西班牙': 'ES', '加拿大': 'CA', '墨西哥': 'MX', '印度': 'IN', '日本': 'JP', '澳大利亚': 'AU' }
mc = get_mysql_connection().cursor() mc.execute(query) get_mysql_connection().commit() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument("--threads", "-t", type=int, default=1, required=False) args = parser.parse_args() ims = get_all_images() with ProcessPoolExecutor(max_workers=args.threads) as tpe: future_to_id = { tpe.submit(calculate, impath(filename)): (id, filename) for id, filename in ims } for i, future in enumerate( concurrent.futures.as_completed(future_to_id)): id, filename = future_to_id[future] try: r = future.result() add_to_db(id, *r) except Exception as exc: print('\n%r generated an exception: %s' % (id, exc)) else:
# if n<=2: # return 1 # return fib(n-1)+fib(n-2) # # if __name__ == "__main__": # with ThreadPoolExecutor(3) as executor: # all_task = [executor.submit(fib, (num)) for num in range(25,40)] # start_time = time.time() # for future in as_completed(all_task): # data = future.result() # print("exe result: {}".format(data)) # # print("last time is: {}".format(time.time()-start_time)) #2. 对于io操作来说,多线程优于多进程 def random_sleep(n): time.sleep(n) return n if __name__ == "__main__": # windows下ProcessPoolExecutor 必须放在if __name__ == "__main__":里,linux不需要 with ProcessPoolExecutor(3) as executor: all_task = [executor.submit(random_sleep, (num)) for num in [2] * 30] start_time = time.time() for future in as_completed(all_task): data = future.result() print("exe result: {}".format(data)) print("last time is: {}".format(time.time() - start_time))
def my_app(config: DictConfig) -> None: global logger logger = getLogger(config.verbose) logger.info(OmegaConf.to_yaml(config)) utt_list = to_absolute_path(config.utt_list) out_dir = to_absolute_path(config.out_dir) question_path_general = to_absolute_path(config.question_path) # Time-lag model # in: musical/linguistic context # out: time-lag (i.e. onset time deviation) if config.timelag.question_path is not None: question_path = config.timelag.question_path else: question_path = question_path_general in_timelag_source = MusicalLinguisticSource( utt_list, to_absolute_path(config.timelag.label_phone_score_dir), add_frame_features=False, subphone_features=None, question_path=question_path, log_f0_conditioning=config.log_f0_conditioning, ) out_timelag_source = TimeLagFeatureSource( utt_list, to_absolute_path(config.timelag.label_phone_score_dir), to_absolute_path(config.timelag.label_phone_align_dir), ) in_timelag = FileSourceDataset(in_timelag_source) out_timelag = FileSourceDataset(out_timelag_source) # Duration model # in: musical/linguistic context # out: duration if config.duration.question_path is not None: question_path = config.duration.question_path else: question_path = question_path_general in_duration_source = MusicalLinguisticSource( utt_list, to_absolute_path(config.duration.label_dir), add_frame_features=False, subphone_features=None, question_path=question_path, log_f0_conditioning=config.log_f0_conditioning, ) out_duration_source = DurationFeatureSource( utt_list, to_absolute_path(config.duration.label_dir)) in_duration = FileSourceDataset(in_duration_source) out_duration = FileSourceDataset(out_duration_source) # Acoustic model # in: musical/linguistic context # out: acoustic features if config.acoustic.question_path is not None: question_path = config.acoustic.question_path else: question_path = question_path_general in_acoustic_source = MusicalLinguisticSource( utt_list, to_absolute_path(config.acoustic.label_dir), question_path, add_frame_features=True, subphone_features=config.acoustic.subphone_features, log_f0_conditioning=config.log_f0_conditioning, ) out_acoustic_source = WORLDAcousticSource( utt_list, to_absolute_path(config.acoustic.wav_dir), to_absolute_path(config.acoustic.label_dir), question_path, use_harvest=config.acoustic.use_harvest, f0_ceil=config.acoustic.f0_ceil, f0_floor=config.acoustic.f0_floor, frame_period=config.acoustic.frame_period, mgc_order=config.acoustic.mgc_order, num_windows=config.acoustic.num_windows, relative_f0=config.acoustic.relative_f0, vibrato_mode=config.acoustic.vibrato_mode, sample_rate=config.acoustic.sample_rate, d4c_threshold=config.acoustic.d4c_threshold, trajectory_smoothing=config.acoustic.trajectory_smoothing, trajectory_smoothing_cutoff=config.acoustic. trajectory_smoothing_cutoff, correct_vuv=config.acoustic.correct_vuv, ) in_acoustic = FileSourceDataset(in_acoustic_source) out_acoustic = FileSourceDataset(out_acoustic_source) # Save as files in_timelag_root = join(out_dir, "in_timelag") out_timelag_root = join(out_dir, "out_timelag") in_duration_root = join(out_dir, "in_duration") out_duration_root = join(out_dir, "out_duration") in_acoustic_root = join(out_dir, "in_acoustic") out_acoustic_root = join(out_dir, "out_acoustic") for d in [ in_timelag_root, out_timelag_root, in_duration_root, out_duration_root, in_acoustic_root, out_acoustic_root, ]: if not os.path.exists(d): logger.info("mkdirs: %s", format(d)) os.makedirs(d) # Save features for timelag model if config.timelag.enabled: logger.info("Timelag linguistic feature dim: %s", str(in_timelag[0].shape[1])) logger.info("Timelag feature dim: %s", str(out_timelag[0].shape[1])) with ProcessPoolExecutor(max_workers=config.max_workers) as executor: futures = [ executor.submit( _prepare_timelag_feature, in_timelag_root, out_timelag_root, in_timelag, out_timelag, idx, ) for idx in range(len(in_timelag)) ] for future in tqdm(futures): future.result() # Save features for duration model if config.duration.enabled: logger.info("Duration linguistic feature dim: %s", str(in_duration[0].shape[1])) logger.info("Duration feature dim: %s", str(out_duration[0].shape[1])) with ProcessPoolExecutor(max_workers=config.max_workers) as executor: futures = [ executor.submit( _prepare_duration_feature, in_duration_root, out_duration_root, in_duration, out_duration, idx, ) for idx in range(len(in_duration)) ] for future in tqdm(futures): future.result() # Save features for acoustic model if config.acoustic.enabled: logger.info("Acoustic linguistic feature dim: %s", str(in_acoustic[0].shape[1])) logger.info("Acoustic feature dim: %s", str(out_acoustic[0][0].shape[1])) with ProcessPoolExecutor(max_workers=config.max_workers) as executor: futures = [ executor.submit( _prepare_acoustic_feature, in_acoustic_root, out_acoustic_root, in_acoustic, out_acoustic, idx, ) for idx in range(len(in_acoustic)) ] for future in tqdm(futures): future.result()
def __init__(self, loop=None): super().__init__() self.loop = loop or asyncio.get_event_loop() self.pool = ProcessPoolExecutor() pickle.DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
def resize_imgs(p): files = p.glob('*/*.JPEG') #list(map(partial(resizes, p), files)) with ProcessPoolExecutor(cpus) as e: e.map(partial(resizes, p), files)
import librosa import os from multiprocessing import Process from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed import warnings import time warnings.filterwarnings('ignore') def mfcc(input_file): for address, dirs, files in os.walk(input_file): os.mkdir("new" + address) if files: filename = "new" + address for file in files: y, sr = librosa.core.load(address + '/' + file) sign = np.empty(0) sign = librosa.feature.mfcc(y=y, sr=sr) np.save(filename + '/' + file[:-4], sign) print("Enter dir name:") dirname = input() start_time = time.time() with ProcessPoolExecutor(max_workers=5) as pool: pool.submit(mfcc, dirname) print("--- %s seconds ---" % (time.time() - start_time))
#!/usr/bin/env python from my_devices import my_devices_list from my_functions import ssh_command, ssh_command2 from netmiko import ConnectHandler from datetime import datetime from concurrent.futures import ProcessPoolExecutor, as_completed start_time = datetime.now() max_processes = len(my_devices_list) # print(max_processes) pool = ProcessPoolExecutor(max_processes) future_list = [] for my_device in my_devices_list: future = pool.submit(ssh_command2, my_device, "show version") future_list.append(future) # process as thread is completed for future in as_completed(future_list): print("Result: " + future.result()) end_time = datetime.now() print("Time spent: " + str(end_time - start_time))
def main(log, args): """Run visualmetrics.py in parallel. Args: log: The structlog logger instance. args: The parsed arguments from the argument parser. Returns: The return code that the program will exit with. """ fetch_dir = os.getenv("MOZ_FETCHES_DIR") if not fetch_dir: log.error("Expected MOZ_FETCHES_DIR environment variable.") return 1 fetch_dir = Path(fetch_dir) visualmetrics_path = fetch_dir / "visualmetrics.py" if not visualmetrics_path.exists(): log.error("Could not locate visualmetrics.py", expected_path=str(visualmetrics_path)) return 1 browsertime_results_path = fetch_dir / "browsertime-results.tgz" try: with tarfile.open(str(browsertime_results_path)) as tar: tar.extractall(path=str(fetch_dir)) except Exception: log.error("Could not read/extract browsertime results archive", path=browsertime_results_path, exc_info=True) return 1 log.info("Extracted browsertime results", path=browsertime_results_path) try: jobs_json_path = fetch_dir / "browsertime-results" / "jobs.json" jobs_json = read_json(jobs_json_path, JOB_SCHEMA) except Exception: log.error("Could not open the jobs.json file", path=jobs_json_path, exc_info=True) return 1 jobs = [] for job in jobs_json["jobs"]: browsertime_json_path = fetch_dir / job["browsertime_json_path"] try: browsertime_json = read_json(browsertime_json_path, BROWSERTIME_SCHEMA) except Exception: log.error("Could not open a browsertime.json file", path=browsertime_json_path, exc_info=True) return 1 for site in browsertime_json: for video in site["files"]["video"]: jobs.append( Job( test_name=job["test_name"], json_path=browsertime_json_path, video_path=browsertime_json_path.parent / video, )) failed_runs = 0 suites = {} with ProcessPoolExecutor(max_workers=cpu_count()) as executor: for job, result in zip( jobs, executor.map( partial( run_visual_metrics, visualmetrics_path=visualmetrics_path, options=args.visual_metrics_options, ), jobs, ), ): returncode, res = result if returncode != 0: log.error( "Failed to run visualmetrics.py", video_path=job.video_path, error=res, ) failed_runs += 1 else: # Python 3.5 requires a str object (not 3.6+) res = json.loads(res.decode("utf8")) for name, value in res.items(): append_result(log, suites, job.test_name, name, value) suites = [get_suite(suite) for suite in suites.values()] perf_data = { "framework": { "name": "browsertime" }, "application": jobs_json["application"], "type": "vismet", "suites": suites, } for entry in suites: entry["extraOptions"] = jobs_json["extra_options"] # Try to get the similarity for all possible tests, this means that we # will also get a comparison of recorded vs. live sites to check # the on-going quality of our recordings. similarity = None if "android" in os.getenv("TC_PLATFORM", ""): try: from similarity import calculate_similarity similarity = calculate_similarity(jobs_json, fetch_dir, OUTPUT_DIR, log) except Exception: log.info("Failed to calculate similarity score", exc_info=True) if similarity: suites[0]["subtests"].append({ "name": "Similarity3D", "value": similarity[0], "replicates": [similarity[0]], "lowerIsBetter": False, "unit": "a.u.", }) suites[0]["subtests"].append({ "name": "Similarity2D", "value": similarity[1], "replicates": [similarity[1]], "lowerIsBetter": False, "unit": "a.u.", }) # Validates the perf data complies with perfherder schema. # The perfherder schema uses jsonschema so we can't use voluptuous here. validate(perf_data, PERFHERDER_SCHEMA) raw_perf_data = json.dumps(perf_data) with Path(OUTPUT_DIR, "perfherder-data.json").open("w") as f: f.write(raw_perf_data) # Prints the data in logs for Perfherder to pick it up. log.info("PERFHERDER_DATA: %s" % raw_perf_data) # Lists the number of processed jobs, failures, and successes. with Path(OUTPUT_DIR, "summary.json").open("w") as f: json.dump( { "total_jobs": len(jobs), "successful_runs": len(jobs) - failed_runs, "failed_runs": failed_runs, }, f, ) # If there's one failure along the way, we want to return > 0 # to trigger a red job in TC. return failed_runs
ignore_index=True, ) df = df.sort_values('Fecha') df.drop_duplicates(inplace=True) df.to_csv( 'F:/DataSets/Tesis/Datos_Procesados/Medidor_Campo_Electrico/conjunto_datos_campo_texto.csv', sep=',', decimal='.', index=None, encoding='UTF-8') if __name__ == "__main__": #variable para vizulaizar el tiempo total de ejecucion del programa t1 = time.time() #se cargan las direcciones de los archivos y se procesan en paralelo llamando a la funcion "campo_electrico_promedio" archivos_mc = glob.glob( 'F:/DataSets/Tesis/Datos_Originales/Medidor_Campo_Electrico/*.efm') #en max_workers se pone el numero de procesadors logicos de la CPU with ProcessPoolExecutor(max_workers=8) as executor: executor.map(campo_electrico_promedio, archivos_mc) #se unen todos los archivos en uno solo con la funcion "archivos_campo_electrico_unidos" (opcional) archivos_mcp = glob.glob( 'F:/DataSets/Tesis/Datos_Procesados/Medidor_Campo_Electrico/conjunto_procesado_como_texto/*.csv' ) archivos_campo_electrico_unidos(archivos_mcp) #se imprime el tiempo de ejecucion del programa en segundos t2 = time.time() t_transcurrido = round(t2 - t1, 4) print(f'Archivos procesados en {t_transcurrido} segundos.')
def index_resources(config, policies, date=None, concurrency=5, accounts=None, tag=None, verbose=False): """index policy resources""" logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO)) logging.getLogger('botocore').setLevel(logging.WARNING) logging.getLogger('elasticsearch').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('c7n.worker').setLevel(logging.INFO) # validating the config and policy files. with open(config) as fh: config = yaml.safe_load(fh.read()) jsonschema.validate(config, CONFIG_SCHEMA) with open(policies) as fh: policies = yaml.safe_load(fh.read()) load_resources() schema.validate(policies) date = valid_date(date, delta=1) with ProcessPoolExecutor(max_workers=concurrency) as w: futures = {} jobs = [] for account in config.get('accounts'): if accounts and account['name'] not in accounts: continue if tag: found = False for t in account['tags'].values(): if tag == t: found = True break if not found: continue for region in account.get('regions'): for policy in policies.get('policies'): p = (config, account, region, policy, date) jobs.append(p) for j in jobs: log.debug("submit account:{} region:{} policy:{} date:{}".format( j[1]['name'], j[2], j[3]['name'], j[4])) futures[w.submit(index_account_resources, *j)] = j # Process completed for f in as_completed(futures): config, account, region, policy, date = futures[f] if f.exception(): log.warning( "error account:{} region:{} policy:{} error:{}".format( account['name'], region, policy['name'], f.exception())) continue log.info("complete account:{} region:{} policy:{}".format( account['name'], region, policy['name']))
def index_metrics(config, start, end, incremental=False, concurrency=5, accounts=None, period=3600, tag=None, index='policy-metrics', verbose=False): """index policy metrics""" logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO)) logging.getLogger('botocore').setLevel(logging.WARNING) logging.getLogger('elasticsearch').setLevel(logging.WARNING) logging.getLogger('urllib3').setLevel(logging.WARNING) logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('c7n.worker').setLevel(logging.INFO) with open(config) as fh: config = yaml.safe_load(fh.read()) jsonschema.validate(config, CONFIG_SCHEMA) start, end = get_date_range(start, end) p_accounts = set() p_account_stats = {} i_time = i_points = 0 t = time.time() with ProcessPoolExecutor(max_workers=concurrency) as w: futures = {} jobs = [] # Filter for account in config.get('accounts'): if accounts and account['name'] not in accounts: continue if tag: found = False for t in account['tags'].values(): if tag == t: found = True break if not found: continue p_accounts.add((account['name'])) for region in account.get('regions'): for (p_start, p_end) in get_periods(start, end, period): p = (config, index, region, account, p_start, p_end, period) jobs.append(p) # by default we'll be effectively processing in order, but thats bumps # our concurrency into rate limits on metrics retrieval in a given account # region, go ahead and shuffle, at least with lucene, the non ordering # should have minimal impact on query perf (inverted index). random.shuffle(jobs) for j in jobs: log.debug("submit account:%s region:%s start:%s end:%s" % (j[3]['name'], j[2], j[4], j[5])) futures[w.submit(index_account_metrics, *j)] = j # Process completed for f in as_completed(futures): config, index, region, account, p_start, p_end, period = futures[f] if f.exception(): log.warning("error account:%s region:%s error:%s", account['name'], region, f.exception()) continue rtime, rpoints = f.result() rstat = p_account_stats.setdefault(account['name'], {}).setdefault( region, {'points': 0}) rstat['points'] += rpoints # log.info("complete account:%s, region:%s points:%s time:%0.2f", # account['name'], region, rpoints, rtime) i_time += rtime i_points += rpoints log.info("complete accounts:%d points:%d itime:%0.2f time:%0.2f", len(p_accounts), i_points, i_time, time.time() - t)
def parallel_process(array, function_to_calculate, nb_jobs=1, use_kwargs=False, front_num=0): """ A parallel version of the map function with a progress bar. Function taken from: http://danshiebler.com/2016-09-14-parallel-progress-bar/ Parameters: array: array-like An array to iterate over. function_to_calculate: function A python function to apply to the elements of array n_jobs: int, default=16 The number of cores to use use_kwargs: bool, default=False Whether to consider the elements of array as dictionaries of keyword arguments to function front_num: int, default=3 The number of iterations to run serially before kicking off the parallel job. Useful for catching bugs Returns: [function(array[0]), function(array[1]), ...] """ # We run the first few iterations serially to catch bugs if front_num > 0: front = [ function_to_calculate( **a) if use_kwargs else function_to_calculate(a) for a in array[:front_num] ] else: front = [] # If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. if nb_jobs == 1: return front + [ function_to_calculate( **a) if use_kwargs else function_to_calculate(a) for a in tqdm(array[front_num:]) ] # Assemble the workers with ProcessPoolExecutor(max_workers=nb_jobs) as pool: # Pass the elements of array into function if use_kwargs: futures = [ pool.submit(function_to_calculate, **a) for a in array[front_num:] ] else: futures = [ pool.submit(function_to_calculate, a) for a in array[front_num:] ] kwargs = { 'total': len(futures), 'unit': 'it', 'unit_scale': True, 'leave': True } # Print out the progress as tasks complete for f in tqdm(as_completed(futures), **kwargs): pass out = [] # Get the results from the futures. for i, future in tqdm(enumerate(futures)): try: out.append(future.result()) except Exception as e: out.append(e) return front + out
def __init__(self, num_workers, num_jobs=None): self.num_workers = num_workers self._executor = ProcessPoolExecutor(max_workers=self.num_workers) self._futures = []
def main(): with open(sys.argv[1]) as f: r = csv.reader(f) with ProcessPoolExecutor(16) as exec: exec.map(do_thing, enumerate(r))
#import pyqtgraph as pg from datetime import datetime import math, os from numba import jit import numpy as np from concurrent.futures import ProcessPoolExecutor POOL = ProcessPoolExecutor(os.cpu_count()) # Parses datetime in EO format def parsedate(s): return datetime.strptime(s, "%Y-%m-%d") #def parseonlydate(s): return parsedate(s).date() def formatdate(d): return datetime.strftime(d, "%Y-%m-%d") def formatdatetime(d): return datetime.strftime(d, "%Y-%m-%d %H:%M:%S") def chunks(l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] # Converts a datetime to a year decimal. E.g. July 2019 -> ~2019.5
q = Queue() t1 = Thread(target=consumer, args=(q,)) t2 = Thread(target=producer, args=(q,)) t1.start() t2.start() # 可以在队列中存放特殊的值一次来控制是否停止 # 向队列中添加数据项时并不会复制此数据项,线程间通信实际上是在线程间传递对象引用。 # 担心对象的共享状态,那你最好只传递不可修改的数据结构(如:整型、字符串或者元组)或者一个对象的深拷贝 # block=False用来避免当执行某些特定队列操作时发生无限阻塞的情况 # 比如,一个非阻塞的put() 方法和一个固定大小的队列一起使用,这样当队列已满时就可以执行不同的代码。比如输出一条日志信息并丢弃。 # .qsize, .full和.empty 并非线程安全,可能在使用empty时队列为空但是同时另一个线程有插入了一个值,尽量不用这些方法 from concurrent.futures import ProcessPoolExecutor # 线程池 with ProcessPoolExecutor(N=3) as pool: # do work in parallel using pool pass # N为系统上可用的CPU的个数 # 向pool中提交工作的方式有两种:results = pool.map(work, data)和future = pool.submit(work, arg)/result = future.result # 前者迭代所有,后者只提交一个 # 利用生成器实现简单的并发。。。挂起
def save(self, name=None, library=None, grid_steps_per_micron=1000, parallel=False): """ Exports the layout and creates an DLW-file, if DLW-features are used. :param name: Optionally, the filename of the saved file (without ending). :param library: Name of the used library. Currently, for gds-export gdspy and gdscad are supported, for oasis-export fatamorgana is supported. Setting the library to 'gdshelpers' will select the gds_export of gdshelpers (experimental). :param grid_steps_per_micron: Defines the resolution :param parallel: Defines if palatalization is used (only supported in Python 3). Standard value will be changed to True in a future version. Deactivating can be useful for debugging reasons. """ if not name: name = self.name elif name.endswith('.gds'): name = name[:-4] library = library or gds_library elif name.endswith('.oasis'): name = name[:-6] library = library or 'fatamorgana' elif name.endswith('.dxf'): name = name[:-4] library = library or 'ezwriter' library = library or gds_library if library == 'gdshelpers': with open(name + '.gds', 'wb') as f: write_cell_to_gdsii_file( f, self, grid_steps_per_unit=grid_steps_per_micron, parallel=parallel) elif library == 'gdspy': if parallel: from concurrent.futures import ProcessPoolExecutor with ProcessPoolExecutor() as pool: self.get_gdspy_cell(pool) else: self.get_gdspy_cell() self.get_gdspy_lib( ).precision = self.get_gdspy_lib().unit / grid_steps_per_micron gdspy_cells = self.get_gdspy_lib().cell_dict.values() if parallel: from concurrent.futures import ProcessPoolExecutor with ProcessPoolExecutor() as pool: binary_cells = pool.map(gdspy.Cell.to_gds, gdspy_cells, [grid_steps_per_micron] * len(gdspy_cells)) else: binary_cells = map(gdspy.Cell.to_gds, gdspy_cells, [grid_steps_per_micron] * len(gdspy_cells)) self.get_gdspy_lib().write_gds(name + '.gds', cells=[], binary_cells=binary_cells) elif library == 'gdscad': layout = gdsCAD.core.Layout(precision=1e-6 / grid_steps_per_micron) if parallel: from concurrent.futures import ProcessPoolExecutor with ProcessPoolExecutor() as pool: layout.add(self.get_gdscad_cell(pool)) else: layout.add(self.get_gdscad_cell()) layout.save(name + '.gds') elif library == 'fatamorgana': layout = fatamorgana.OasisLayout(grid_steps_per_micron) if parallel: from concurrent.futures import ProcessPoolExecutor with ProcessPoolExecutor() as pool: cells = self.get_oasis_cells(grid_steps_per_micron, pool) else: cells = self.get_oasis_cells(grid_steps_per_micron) layout.cells = [cells[0]] + list(set(cells[1:])) # noinspection PyUnresolvedReferences def replace_names_by_ids(oasis_layout): name_id = {} for cell_id, cell in enumerate(oasis_layout.cells): if cell.name.string in name_id: raise RuntimeError( 'Each cell name should be unique, name "' + cell.name.string + '" is used multiple times') name_id[cell.name.string] = cell_id cell.name = cell_id for cell in oasis_layout.cells: for placement in cell.placements: placement.name = name_id[placement.name.string] oasis_layout.cellnames = {v: k for k, v in name_id.items()} # improves performance for reading oasis file and workaround for fatamorgana-bug replace_names_by_ids(layout) with open(name + '.oas', 'wb') as f: layout.write(f) elif library == 'ezdxf': from gdshelpers.export.dxf_export import write_cell_to_dxf_file with open(name + '.gds', 'wb') as f: write_cell_to_dxf_file(f, self, grid_steps_per_micron, parallel=parallel) else: raise ValueError( 'library must be either "gdscad", "gdspy" or "fatamorgana"') dlw_data = self.get_dlw_data() if dlw_data: with open(name + '.dlw', 'w') as f: json.dump(dlw_data, f, indent=True)
def determine_language(string_to_test: str) -> str: # initializing the scores that every language dict_language_scores: dict[str, int] = {} for language in tuple(dict_paths_to_word_collections.keys()): dict_language_scores[language] = 0 ''' saving language scores to 'dict_language_scores. Using all available cores to speed up the process''' # Save language names as strings in a tuple languages: tuple[str] = tuple(dict_paths_to_word_collections.keys()) # using all available cores to calculate the score for a with ProcessPoolExecutor() as executor: # bi-/trigrams score for the dutch language d2: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[0], 2) d3: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[0], 3) # bi-/trigrams score for english language e2: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[1], 2) e3: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[1], 3) # bi-/trigrams score for german language g2: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[2], 2) g3: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[2], 3) # bi-/trigrams score for french language f2: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[3], 2) f3: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[3], 3) # bi-/trigrams score for spanish language s2: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[4], 2) s3: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[4], 3) # bi-/trigram score for italian language i2: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[5], 2) i3: executor[int] = executor.submit(count_ngram_occurences, string_to_test, languages[5], 3) # saving bigram and trigram scores in dictionary # dutch dict_language_scores[languages[0]] = d2.result() + d3.result() # english dict_language_scores[languages[1]] = e2.result() + e3.result() # german dict_language_scores[languages[2]] = g2.result() + g3.result() # french dict_language_scores[languages[3]] = f2.result() + f3.result() # spanish dict_language_scores[languages[4]] = s2.result() + s3.result() # italian dict_language_scores[languages[5]] = i2.result() + i3.result() # return the language with the highest store language: str = max(dict_language_scores, key=dict_language_scores.get) print(f"Language : {language}") print(f"Bigram and trigram probabilities") get_gram_probabilities(string_to_test, language)
def __iter__(self): from concurrent.futures import ProcessPoolExecutor with ProcessPoolExecutor() as executor: return _coconut.iter( _coconut.list(executor.map(self.func, *self.iters)))
line = line.strip() speaker_used.append(line) print(speaker_used) #speaker_used = ['262', '272', '229', '232', '292', '293', '360', '361', '248', '251'] #speaker_used = ['p'+i for i in speaker_used] #speaker_used = ['p315'] ## Next we are to extract the acoustic features (MCEPs, lf0) and compute the corresponding stats (means, stds). # Make dirs to contain the MCEPs os.makedirs(mc_dir_train, exist_ok=True) os.makedirs(mc_dir_test, exist_ok=True) num_workers = 52 #len(speaker_used) #cpu_count() print("number of workers: ", num_workers) executor = ProcessPoolExecutor(max_workers=num_workers) work_dir = target_wavpath # spk_folders = os.listdir(work_dir) # print("processing {} speaker folders".format(len(spk_folders))) # print(spk_folders) futures = [] for spk in speaker_used: spk_path = os.path.join(work_dir, spk) futures.append( executor.submit( partial(get_spk_world_feats, spk_path, mc_dir_train, mc_dir_test, sample_rate))) result_list = [future.result() for future in tqdm(futures)] print(result_list)
def __init__(self, config: Config): self.config = config self.model = None # type: ChessModel self.dataset = deque(),deque(),deque() self.executor = ProcessPoolExecutor(max_workers=config.trainer.cleaning_processes)
def generate_random_walks(num_walks, walk_length, workers, vertices): ''' 随机游走 @:param num_walks: 重复游走的次数 @:param walk_length: 单次游走的序列长度 @:param vertices: 全部学生节点id ''' logging.info('Loading similarity_nets on disk...') graphs = load_from_disk('similarity_nets_graphs') # 多层DSN alias_method_j = load_from_disk( 'nets_weights_alias_method_j') # 随机游走需要使用到的字典 alias_method_q = load_from_disk( 'nets_weights_alias_method_q') # 随机游走需要使用到的字典 # amount_neighbours = load_from_disk('amount_neighbours') flag = 1 # 1: max, 2:max(median) 3: max(avg) # flag = 2 '''1. 获取每个节点取得最大边权的所在网络层数''' d_max_sim_layer_dict = {} # 获取各个节点在所有DSN中取得最大权值所在的网络层数 #cui--注释 # ''' 单层网络随机游走不需要获取 ''' # if (len(graphs)) > 1: # print("# of DSNs>1, now getting the d_max_sim_layer_dict...") # d_max_sim_layer_dict = get_max_sim_layer(vertices, len(graphs), flag) # else: # print("only single layer of DSN, skipping getting d_max_sim_layer_dict.") #cuiEnd logging.info('Creating Random Walks...') t0 = time.time() walks = deque() # 游走序列,使用队列存储 initialLayer = 0 if (workers > num_walks): workers = num_walks with ProcessPoolExecutor(max_workers=workers) as executor: futures = {} for walk_iter in range( num_walks): # 随机游走num_walks轮,每次对全部节点产生一组游走序列,最后将多轮游走的序列拼接在一起 random.shuffle(vertices) # 打乱节点 print('-- walk_iteration:', walk_iter) # job = executor.submit(exec_ramdom_walks_for_chunck, vertices, graphs, alias_method_j, alias_method_q, walk_length, amount_neighbours) # alias_method_j 和 alias_method_q 在采样时会使用到,返回对全部节点产生的游走序列 job = executor.submit(exec_random_walks_for_chunck, d_max_sim_layer_dict, vertices, graphs, alias_method_j, alias_method_q, walk_length) # 随机游走 futures[job] = walk_iter # part += 1 logging.info("Receiving results...") for job in as_completed(futures): walk = job.result() r = futures[job] logging.info("Iteration {} executed.".format(r)) walks.extend(walk) # 将多轮游走产生的序列拼接在一起 del futures[job] t1 = time.time() logging.info('RWs created. Time: {}m'.format((t1 - t0) / 60)) logging.info("Saving Random Walks on disk...") save_random_walks(walks, num_walks, walk_length)
break print('False:'+ name) num = num+1 count = count + 1 file_name = name[:1] + str(num) break cap.release() out.release() cv2.destroyAllWindows() res = False return clock,count,res,file_name # 检查录制状态 def check(rtsp_url,name): global res info = record(rtsp_url,name) while True: if info[0] == False: # 计时结束 break elif info[1] > 10: # 超出连接次数 print('Over count limit!!!') break elif info[2] == False: # 连接失败 res = True # 重置连接状态 print('Reopen:' + info[3]) info = record(rtsp_url, info[3]) time.sleep(5) if __name__ == '__main__': ProcessPoolExecutor(os.cpu_count()).map(check,addr,name_list)
def run(*args): if 1 in args: start = time.time() f(10_000_000) f(10_000_000) f(10_000_000) # f(10_000_000) print("1. No parallel time: {}".format(time.time() - start)) if 2 in args: start = time.time() t1 = Thread(target=f, args={10_000_000}) t2 = Thread(target=f, args={10_000_000}) t3 = Thread(target=f, args={10_000_000}) # t4 = Thread(target=f, args={10_000_000}) t1.start(), t2.start(), t3.start() t1.join(), t2.join(), t3.join() print("2. Threads time: {}".format(time.time() - start)) if 3 in args: start = time.time() with ThreadPoolExecutor(max_workers=4) as pool: pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) print("3. Thread pool time: {}".format(time.time() - start)) if 4 in args: start = time.time() p1 = Process(target=f, args={10_000_000}) p2 = Process(target=f, args={10_000_000}) p3 = Process(target=f, args={10_000_000}) # p4 = Process(target=f, args={10_000_000}) p1.start(), p2.start(), p3.start() p1.join(), p2.join(), p3.join() print("4. Process time: {}".format(time.time() - start)) if 5 in args: start = time.time() with ProcessPoolExecutor(max_workers=4) as pool: pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) pool.submit(f(10_000_0000)) print("5. Process pool time: {}".format(time.time() - start))
#!/usr/bin/python from concurrent.futures import ProcessPoolExecutor from os import getpid as pid from time import sleep L = [1, 2, 3, 4, 5] # 单进程,要用至少25秒 for i in L: sleep(5) print(pid(), "已过5秒", i) # 多进程并发,5秒 def SleepPrint(in_value): sleep(5) print(pid(), "已过5秒", in_value) pool = ProcessPoolExecutor(5) for i in L: pool.submit(SleepPrint, i)
import unicodedata import requests from gtts import gTTS import youtube_dl from config import BOTID from config import OWNERID from config import COMMANDPREFIX import discord from config import chatbot, PROCESS_POOL_EXECUTOR_COUNT, WAIT_TIME_BEFORE_TYPING, WAIT_TIME_RESPONSE_READY, COMMANDPREFIX, BOTID print("Starting process") #f = open("vc.wav", "wb") client = discord.Client() ppool = ProcessPoolExecutor(PROCESS_POOL_EXECUTOR_COUNT) async def start(self): discord.opus.load_opus(self.opus_library) @client.event async def on_ready(): print('Logged in as') print(client.user.name) print(client.user.id) print('------') await client.change_presence(game=discord.Game(name="Sayori Discord Wrapper")) def generate_response(message): return chatbot.get_response(message)
from typing import Tuple from ..fonts import Font from .text import Text def points_to_millimeters(points: float) -> int: return int(points * 0.352778) def millimeters_to_points(millimeters: float) -> int: return int(millimeters * 2.83465) loop = asyncio.get_event_loop() process_executor = ProcessPoolExecutor() class PageImageWriter: def write_text( self, position: Tuple[int, int], window: Tuple[int, int], text: str, font: Font ) -> Tuple[int, int]: """ Draw the supplied text with given font at given position :param position: position to draw text at :param window: size of the window the text must fit in :param text: string to draw in the image :param font: font info to generate text :return: None """ raise NotImplementedError()