def compile_dir(dir,
maxlevels=10,
ddir=None,
force=False,
rx=None,
quiet=0,
legacy=False,
optimize=-1,
workers=1,
invalidation_mode=None):
"""Byte-compile all modules in the given directory tree.
Arguments (only dir is required):
dir: the directory to byte-compile
maxlevels: maximum recursion level (default 10)
ddir: the directory that will be prepended to the path to the
file as it is compiled into each byte-code file.
force: if True, force compilation, even if timestamps are up-to-date
quiet: full output with False or 0, errors only with 1,
no output with 2
legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
optimize: optimization level or -1 for level of the interpreter
workers: maximum number of parallel workers
invalidation_mode: how the up-to-dateness of the pyc will be checked
"""
ProcessPoolExecutor = None
if workers is not None:
if workers < 0:
raise ValueError('workers must be greater or equal to 0')
elif workers != 1:
try:
# Only import when needed, as low resource platforms may
# fail to import it
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
files_and_ddirs = _walk_dir(dir,
quiet=quiet,
maxlevels=maxlevels,
ddir=ddir)
success = True
if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
workers = workers or None
with ProcessPoolExecutor(max_workers=workers) as executor:
results = executor.map(
partial(
_compile_file_tuple,
force=force,
rx=rx,
quiet=quiet,
legacy=legacy,
optimize=optimize,
invalidation_mode=invalidation_mode,
), files_and_ddirs)
success = min(results, default=True)
else:
for file, dfile in files_and_ddirs:
if not compile_file(file, dfile, force, rx, quiet, legacy,
optimize, invalidation_mode):
success = False
return success
import numpy as np
import librosa.feature
import librosa
from concurrent.futures import ProcessPoolExecutor
from local_loader import d, key_map
from sklearn import svm
from sklearn.multiclass import OneVsOneClassifier
counter = 0
m = OneVsOneClassifier(svm.SVC(decision_function_shape='ovo', kernel='rbf'))
# train
with ProcessPoolExecutor() as executor:
X = np.empty((0, 12 * d.window))
keys = []
for song, key, file, txt in executor.map(d.__getitem__, range(1, 10)):
X2 = np.empty((0, 12 * d.window))
for idx, item in enumerate(song):
X2 = np.concatenate((X2, item.reshape((1, 120))), axis=0)
keys.extend(key)
if X2.shape[0] > len(key):
X2 = X2[:len(key), :]
else:
key = key[:X2.shape[0]]
import time, os, requests, json, re, pymysql
import numpy as np
import pandas as pd
from hashlib import md5
from sqlalchemy import create_engine
from datetime import datetime
from concurrent.futures import ProcessPoolExecutor
from my_toolkit import sql_write_read
domain_name = "http://amazon.yibainetwork.com/"
api = "services/amazon/amazonkeylisting/run"
secret_key = '!@#$%~Ybai7895&$^^GoPingRun?'
PROCESS_POOL = ProcessPoolExecutor(4)
country_name_dict = {
'英国': 'UK',
'美国': 'US',
'法国': 'FR',
'德国': 'DE',
'意大利': 'IT',
'西班牙': 'ES',
'加拿大': 'CA',
'墨西哥': 'MX',
'印度': 'IN',
'日本': 'JP',
'澳大利亚': 'AU'
}
mc = get_mysql_connection().cursor()
mc.execute(query)
get_mysql_connection().commit()
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--threads", "-t", type=int, default=1, required=False)
args = parser.parse_args()
ims = get_all_images()
with ProcessPoolExecutor(max_workers=args.threads) as tpe:
future_to_id = {
tpe.submit(calculate, impath(filename)): (id, filename)
for id, filename in ims
}
for i, future in enumerate(
concurrent.futures.as_completed(future_to_id)):
id, filename = future_to_id[future]
try:
r = future.result()
add_to_db(id, *r)
except Exception as exc:
print('\n%r generated an exception: %s' % (id, exc))
else:
# if n<=2:
# return 1
# return fib(n-1)+fib(n-2)
#
# if __name__ == "__main__":
# with ThreadPoolExecutor(3) as executor:
# all_task = [executor.submit(fib, (num)) for num in range(25,40)]
# start_time = time.time()
# for future in as_completed(all_task):
# data = future.result()
# print("exe result: {}".format(data))
#
# print("last time is: {}".format(time.time()-start_time))
#2. 对于io操作来说，多线程优于多进程
def random_sleep(n):
time.sleep(n)
return n
if __name__ == "__main__": # windows下ProcessPoolExecutor 必须放在if __name__ == "__main__":里,linux不需要
with ProcessPoolExecutor(3) as executor:
all_task = [executor.submit(random_sleep, (num)) for num in [2] * 30]
start_time = time.time()
for future in as_completed(all_task):
data = future.result()
print("exe result: {}".format(data))
print("last time is: {}".format(time.time() - start_time))
def my_app(config: DictConfig) -> None:
global logger
logger = getLogger(config.verbose)
logger.info(OmegaConf.to_yaml(config))
utt_list = to_absolute_path(config.utt_list)
out_dir = to_absolute_path(config.out_dir)
question_path_general = to_absolute_path(config.question_path)
# Time-lag model
# in: musical/linguistic context
# out: time-lag (i.e. onset time deviation)
if config.timelag.question_path is not None:
question_path = config.timelag.question_path
else:
question_path = question_path_general
in_timelag_source = MusicalLinguisticSource(
utt_list,
to_absolute_path(config.timelag.label_phone_score_dir),
add_frame_features=False,
subphone_features=None,
question_path=question_path,
log_f0_conditioning=config.log_f0_conditioning,
)
out_timelag_source = TimeLagFeatureSource(
utt_list,
to_absolute_path(config.timelag.label_phone_score_dir),
to_absolute_path(config.timelag.label_phone_align_dir),
)
in_timelag = FileSourceDataset(in_timelag_source)
out_timelag = FileSourceDataset(out_timelag_source)
# Duration model
# in: musical/linguistic context
# out: duration
if config.duration.question_path is not None:
question_path = config.duration.question_path
else:
question_path = question_path_general
in_duration_source = MusicalLinguisticSource(
utt_list,
to_absolute_path(config.duration.label_dir),
add_frame_features=False,
subphone_features=None,
question_path=question_path,
log_f0_conditioning=config.log_f0_conditioning,
)
out_duration_source = DurationFeatureSource(
utt_list, to_absolute_path(config.duration.label_dir))
in_duration = FileSourceDataset(in_duration_source)
out_duration = FileSourceDataset(out_duration_source)
# Acoustic model
# in: musical/linguistic context
# out: acoustic features
if config.acoustic.question_path is not None:
question_path = config.acoustic.question_path
else:
question_path = question_path_general
in_acoustic_source = MusicalLinguisticSource(
utt_list,
to_absolute_path(config.acoustic.label_dir),
question_path,
add_frame_features=True,
subphone_features=config.acoustic.subphone_features,
log_f0_conditioning=config.log_f0_conditioning,
)
out_acoustic_source = WORLDAcousticSource(
utt_list,
to_absolute_path(config.acoustic.wav_dir),
to_absolute_path(config.acoustic.label_dir),
question_path,
use_harvest=config.acoustic.use_harvest,
f0_ceil=config.acoustic.f0_ceil,
f0_floor=config.acoustic.f0_floor,
frame_period=config.acoustic.frame_period,
mgc_order=config.acoustic.mgc_order,
num_windows=config.acoustic.num_windows,
relative_f0=config.acoustic.relative_f0,
vibrato_mode=config.acoustic.vibrato_mode,
sample_rate=config.acoustic.sample_rate,
d4c_threshold=config.acoustic.d4c_threshold,
trajectory_smoothing=config.acoustic.trajectory_smoothing,
trajectory_smoothing_cutoff=config.acoustic.
trajectory_smoothing_cutoff,
correct_vuv=config.acoustic.correct_vuv,
)
in_acoustic = FileSourceDataset(in_acoustic_source)
out_acoustic = FileSourceDataset(out_acoustic_source)
# Save as files
in_timelag_root = join(out_dir, "in_timelag")
out_timelag_root = join(out_dir, "out_timelag")
in_duration_root = join(out_dir, "in_duration")
out_duration_root = join(out_dir, "out_duration")
in_acoustic_root = join(out_dir, "in_acoustic")
out_acoustic_root = join(out_dir, "out_acoustic")
for d in [
in_timelag_root,
out_timelag_root,
in_duration_root,
out_duration_root,
in_acoustic_root,
out_acoustic_root,
]:
if not os.path.exists(d):
logger.info("mkdirs: %s", format(d))
os.makedirs(d)
# Save features for timelag model
if config.timelag.enabled:
logger.info("Timelag linguistic feature dim: %s",
str(in_timelag[0].shape[1]))
logger.info("Timelag feature dim: %s", str(out_timelag[0].shape[1]))
with ProcessPoolExecutor(max_workers=config.max_workers) as executor:
futures = [
executor.submit(
_prepare_timelag_feature,
in_timelag_root,
out_timelag_root,
in_timelag,
out_timelag,
idx,
) for idx in range(len(in_timelag))
]
for future in tqdm(futures):
future.result()
# Save features for duration model
if config.duration.enabled:
logger.info("Duration linguistic feature dim: %s",
str(in_duration[0].shape[1]))
logger.info("Duration feature dim: %s", str(out_duration[0].shape[1]))
with ProcessPoolExecutor(max_workers=config.max_workers) as executor:
futures = [
executor.submit(
_prepare_duration_feature,
in_duration_root,
out_duration_root,
in_duration,
out_duration,
idx,
) for idx in range(len(in_duration))
]
for future in tqdm(futures):
future.result()
# Save features for acoustic model
if config.acoustic.enabled:
logger.info("Acoustic linguistic feature dim: %s",
str(in_acoustic[0].shape[1]))
logger.info("Acoustic feature dim: %s",
str(out_acoustic[0][0].shape[1]))
with ProcessPoolExecutor(max_workers=config.max_workers) as executor:
futures = [
executor.submit(
_prepare_acoustic_feature,
in_acoustic_root,
out_acoustic_root,
in_acoustic,
out_acoustic,
idx,
) for idx in range(len(in_acoustic))
]
for future in tqdm(futures):
future.result()
def __init__(self, loop=None):
super().__init__()
self.loop = loop or asyncio.get_event_loop()
self.pool = ProcessPoolExecutor()
pickle.DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
def resize_imgs(p):
files = p.glob('*/*.JPEG')
#list(map(partial(resizes, p), files))
with ProcessPoolExecutor(cpus) as e:
e.map(partial(resizes, p), files)
import librosa
import os
from multiprocessing import Process
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
import warnings
import time
warnings.filterwarnings('ignore')
def mfcc(input_file):
for address, dirs, files in os.walk(input_file):
os.mkdir("new" + address)
if files:
filename = "new" + address
for file in files:
y, sr = librosa.core.load(address + '/' + file)
sign = np.empty(0)
sign = librosa.feature.mfcc(y=y, sr=sr)
np.save(filename + '/' + file[:-4], sign)
print("Enter dir name:")
dirname = input()
start_time = time.time()
with ProcessPoolExecutor(max_workers=5) as pool:
pool.submit(mfcc, dirname)
print("--- %s seconds ---" % (time.time() - start_time))
#!/usr/bin/env python
from my_devices import my_devices_list
from my_functions import ssh_command, ssh_command2
from netmiko import ConnectHandler
from datetime import datetime
from concurrent.futures import ProcessPoolExecutor, as_completed
start_time = datetime.now()
max_processes = len(my_devices_list)
# print(max_processes)
pool = ProcessPoolExecutor(max_processes)
future_list = []
for my_device in my_devices_list:
future = pool.submit(ssh_command2, my_device, "show version")
future_list.append(future)
# process as thread is completed
for future in as_completed(future_list):
print("Result: " + future.result())
end_time = datetime.now()
print("Time spent: " + str(end_time - start_time))
def main(log, args):
"""Run visualmetrics.py in parallel.
Args:
log: The structlog logger instance.
args: The parsed arguments from the argument parser.
Returns:
The return code that the program will exit with.
"""
fetch_dir = os.getenv("MOZ_FETCHES_DIR")
if not fetch_dir:
log.error("Expected MOZ_FETCHES_DIR environment variable.")
return 1
fetch_dir = Path(fetch_dir)
visualmetrics_path = fetch_dir / "visualmetrics.py"
if not visualmetrics_path.exists():
log.error("Could not locate visualmetrics.py",
expected_path=str(visualmetrics_path))
return 1
browsertime_results_path = fetch_dir / "browsertime-results.tgz"
try:
with tarfile.open(str(browsertime_results_path)) as tar:
tar.extractall(path=str(fetch_dir))
except Exception:
log.error("Could not read/extract browsertime results archive",
path=browsertime_results_path,
exc_info=True)
return 1
log.info("Extracted browsertime results", path=browsertime_results_path)
try:
jobs_json_path = fetch_dir / "browsertime-results" / "jobs.json"
jobs_json = read_json(jobs_json_path, JOB_SCHEMA)
except Exception:
log.error("Could not open the jobs.json file",
path=jobs_json_path,
exc_info=True)
return 1
jobs = []
for job in jobs_json["jobs"]:
browsertime_json_path = fetch_dir / job["browsertime_json_path"]
try:
browsertime_json = read_json(browsertime_json_path,
BROWSERTIME_SCHEMA)
except Exception:
log.error("Could not open a browsertime.json file",
path=browsertime_json_path,
exc_info=True)
return 1
for site in browsertime_json:
for video in site["files"]["video"]:
jobs.append(
Job(
test_name=job["test_name"],
json_path=browsertime_json_path,
video_path=browsertime_json_path.parent / video,
))
failed_runs = 0
suites = {}
with ProcessPoolExecutor(max_workers=cpu_count()) as executor:
for job, result in zip(
jobs,
executor.map(
partial(
run_visual_metrics,
visualmetrics_path=visualmetrics_path,
options=args.visual_metrics_options,
),
jobs,
),
):
returncode, res = result
if returncode != 0:
log.error(
"Failed to run visualmetrics.py",
video_path=job.video_path,
error=res,
)
failed_runs += 1
else:
# Python 3.5 requires a str object (not 3.6+)
res = json.loads(res.decode("utf8"))
for name, value in res.items():
append_result(log, suites, job.test_name, name, value)
suites = [get_suite(suite) for suite in suites.values()]
perf_data = {
"framework": {
"name": "browsertime"
},
"application": jobs_json["application"],
"type": "vismet",
"suites": suites,
}
for entry in suites:
entry["extraOptions"] = jobs_json["extra_options"]
# Try to get the similarity for all possible tests, this means that we
# will also get a comparison of recorded vs. live sites to check
# the on-going quality of our recordings.
similarity = None
if "android" in os.getenv("TC_PLATFORM", ""):
try:
from similarity import calculate_similarity
similarity = calculate_similarity(jobs_json, fetch_dir, OUTPUT_DIR,
log)
except Exception:
log.info("Failed to calculate similarity score", exc_info=True)
if similarity:
suites[0]["subtests"].append({
"name": "Similarity3D",
"value": similarity[0],
"replicates": [similarity[0]],
"lowerIsBetter": False,
"unit": "a.u.",
})
suites[0]["subtests"].append({
"name": "Similarity2D",
"value": similarity[1],
"replicates": [similarity[1]],
"lowerIsBetter": False,
"unit": "a.u.",
})
# Validates the perf data complies with perfherder schema.
# The perfherder schema uses jsonschema so we can't use voluptuous here.
validate(perf_data, PERFHERDER_SCHEMA)
raw_perf_data = json.dumps(perf_data)
with Path(OUTPUT_DIR, "perfherder-data.json").open("w") as f:
f.write(raw_perf_data)
# Prints the data in logs for Perfherder to pick it up.
log.info("PERFHERDER_DATA: %s" % raw_perf_data)
# Lists the number of processed jobs, failures, and successes.
with Path(OUTPUT_DIR, "summary.json").open("w") as f:
json.dump(
{
"total_jobs": len(jobs),
"successful_runs": len(jobs) - failed_runs,
"failed_runs": failed_runs,
},
f,
)
# If there's one failure along the way, we want to return > 0
# to trigger a red job in TC.
return failed_runs
ignore_index=True,
)
df = df.sort_values('Fecha')
df.drop_duplicates(inplace=True)
df.to_csv(
'F:/DataSets/Tesis/Datos_Procesados/Medidor_Campo_Electrico/conjunto_datos_campo_texto.csv',
sep=',',
decimal='.',
index=None,
encoding='UTF-8')
if __name__ == "__main__":
#variable para vizulaizar el tiempo total de ejecucion del programa
t1 = time.time()
#se cargan las direcciones de los archivos y se procesan en paralelo llamando a la funcion "campo_electrico_promedio"
archivos_mc = glob.glob(
'F:/DataSets/Tesis/Datos_Originales/Medidor_Campo_Electrico/*.efm')
#en max_workers se pone el numero de procesadors logicos de la CPU
with ProcessPoolExecutor(max_workers=8) as executor:
executor.map(campo_electrico_promedio, archivos_mc)
#se unen todos los archivos en uno solo con la funcion "archivos_campo_electrico_unidos" (opcional)
archivos_mcp = glob.glob(
'F:/DataSets/Tesis/Datos_Procesados/Medidor_Campo_Electrico/conjunto_procesado_como_texto/*.csv'
)
archivos_campo_electrico_unidos(archivos_mcp)
#se imprime el tiempo de ejecucion del programa en segundos
t2 = time.time()
t_transcurrido = round(t2 - t1, 4)
print(f'Archivos procesados en {t_transcurrido} segundos.')
def index_resources(config,
policies,
date=None,
concurrency=5,
accounts=None,
tag=None,
verbose=False):
"""index policy resources"""
logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
logging.getLogger('requests').setLevel(logging.WARNING)
logging.getLogger('c7n.worker').setLevel(logging.INFO)
# validating the config and policy files.
with open(config) as fh:
config = yaml.safe_load(fh.read())
jsonschema.validate(config, CONFIG_SCHEMA)
with open(policies) as fh:
policies = yaml.safe_load(fh.read())
load_resources()
schema.validate(policies)
date = valid_date(date, delta=1)
with ProcessPoolExecutor(max_workers=concurrency) as w:
futures = {}
jobs = []
for account in config.get('accounts'):
if accounts and account['name'] not in accounts:
continue
if tag:
found = False
for t in account['tags'].values():
if tag == t:
found = True
break
if not found:
continue
for region in account.get('regions'):
for policy in policies.get('policies'):
p = (config, account, region, policy, date)
jobs.append(p)
for j in jobs:
log.debug("submit account:{} region:{} policy:{} date:{}".format(
j[1]['name'], j[2], j[3]['name'], j[4]))
futures[w.submit(index_account_resources, *j)] = j
# Process completed
for f in as_completed(futures):
config, account, region, policy, date = futures[f]
if f.exception():
log.warning(
"error account:{} region:{} policy:{} error:{}".format(
account['name'], region, policy['name'],
f.exception()))
continue
log.info("complete account:{} region:{} policy:{}".format(
account['name'], region, policy['name']))
def index_metrics(config,
start,
end,
incremental=False,
concurrency=5,
accounts=None,
period=3600,
tag=None,
index='policy-metrics',
verbose=False):
"""index policy metrics"""
logging.basicConfig(level=(verbose and logging.DEBUG or logging.INFO))
logging.getLogger('botocore').setLevel(logging.WARNING)
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
logging.getLogger('requests').setLevel(logging.WARNING)
logging.getLogger('c7n.worker').setLevel(logging.INFO)
with open(config) as fh:
config = yaml.safe_load(fh.read())
jsonschema.validate(config, CONFIG_SCHEMA)
start, end = get_date_range(start, end)
p_accounts = set()
p_account_stats = {}
i_time = i_points = 0
t = time.time()
with ProcessPoolExecutor(max_workers=concurrency) as w:
futures = {}
jobs = []
# Filter
for account in config.get('accounts'):
if accounts and account['name'] not in accounts:
continue
if tag:
found = False
for t in account['tags'].values():
if tag == t:
found = True
break
if not found:
continue
p_accounts.add((account['name']))
for region in account.get('regions'):
for (p_start, p_end) in get_periods(start, end, period):
p = (config, index, region, account, p_start, p_end,
period)
jobs.append(p)
# by default we'll be effectively processing in order, but thats bumps
# our concurrency into rate limits on metrics retrieval in a given account
# region, go ahead and shuffle, at least with lucene, the non ordering
# should have minimal impact on query perf (inverted index).
random.shuffle(jobs)
for j in jobs:
log.debug("submit account:%s region:%s start:%s end:%s" %
(j[3]['name'], j[2], j[4], j[5]))
futures[w.submit(index_account_metrics, *j)] = j
# Process completed
for f in as_completed(futures):
config, index, region, account, p_start, p_end, period = futures[f]
if f.exception():
log.warning("error account:%s region:%s error:%s",
account['name'], region, f.exception())
continue
rtime, rpoints = f.result()
rstat = p_account_stats.setdefault(account['name'], {}).setdefault(
region, {'points': 0})
rstat['points'] += rpoints
# log.info("complete account:%s, region:%s points:%s time:%0.2f",
# account['name'], region, rpoints, rtime)
i_time += rtime
i_points += rpoints
log.info("complete accounts:%d points:%d itime:%0.2f time:%0.2f",
len(p_accounts), i_points, i_time,
time.time() - t)
def parallel_process(array,
function_to_calculate,
nb_jobs=1,
use_kwargs=False,
front_num=0):
""" A parallel version of the map function with a progress bar.
Function taken from: http://danshiebler.com/2016-09-14-parallel-progress-bar/
Parameters:
array: array-like
An array to iterate over.
function_to_calculate: function
A python function to apply to the elements of array
n_jobs: int, default=16
The number of cores to use
use_kwargs: bool, default=False
Whether to consider the elements of array as dictionaries of keyword arguments to function
front_num: int, default=3
The number of iterations to run serially before kicking off the parallel job. Useful for catching bugs
Returns:
[function(array[0]), function(array[1]), ...]
"""
# We run the first few iterations serially to catch bugs
if front_num > 0:
front = [
function_to_calculate(
**a) if use_kwargs else function_to_calculate(a)
for a in array[:front_num]
]
else:
front = []
# If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging.
if nb_jobs == 1:
return front + [
function_to_calculate(
**a) if use_kwargs else function_to_calculate(a)
for a in tqdm(array[front_num:])
]
# Assemble the workers
with ProcessPoolExecutor(max_workers=nb_jobs) as pool:
# Pass the elements of array into function
if use_kwargs:
futures = [
pool.submit(function_to_calculate, **a)
for a in array[front_num:]
]
else:
futures = [
pool.submit(function_to_calculate, a)
for a in array[front_num:]
]
kwargs = {
'total': len(futures),
'unit': 'it',
'unit_scale': True,
'leave': True
}
# Print out the progress as tasks complete
for f in tqdm(as_completed(futures), **kwargs):
pass
out = []
# Get the results from the futures.
for i, future in tqdm(enumerate(futures)):
try:
out.append(future.result())
except Exception as e:
out.append(e)
return front + out
def __init__(self, num_workers, num_jobs=None):
self.num_workers = num_workers
self._executor = ProcessPoolExecutor(max_workers=self.num_workers)
self._futures = []
def main():
with open(sys.argv[1]) as f:
r = csv.reader(f)
with ProcessPoolExecutor(16) as exec:
exec.map(do_thing, enumerate(r))
#import pyqtgraph as pg
from datetime import datetime
import math, os
from numba import jit
import numpy as np
from concurrent.futures import ProcessPoolExecutor
POOL = ProcessPoolExecutor(os.cpu_count())
# Parses datetime in EO format
def parsedate(s):
return datetime.strptime(s, "%Y-%m-%d")
#def parseonlydate(s): return parsedate(s).date()
def formatdate(d):
return datetime.strftime(d, "%Y-%m-%d")
def formatdatetime(d):
return datetime.strftime(d, "%Y-%m-%d %H:%M:%S")
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
# Converts a datetime to a year decimal. E.g. July 2019 -> ~2019.5
q = Queue()
t1 = Thread(target=consumer, args=(q,))
t2 = Thread(target=producer, args=(q,))
t1.start()
t2.start()
# 可以在队列中存放特殊的值一次来控制是否停止
# 向队列中添加数据项时并不会复制此数据项，线程间通信实际上是在线程间传递对象引用。
# 担心对象的共享状态，那你最好只传递不可修改的数据结构（如：整型、字符串或者元组）或者一个对象的深拷贝
# block=False用来避免当执行某些特定队列操作时发生无限阻塞的情况
# 比如，一个非阻塞的put() 方法和一个固定大小的队列一起使用，这样当队列已满时就可以执行不同的代码。比如输出一条日志信息并丢弃。
# .qsize, .full和.empty 并非线程安全，可能在使用empty时队列为空但是同时另一个线程有插入了一个值，尽量不用这些方法
from concurrent.futures import ProcessPoolExecutor # 线程池
with ProcessPoolExecutor(N=3) as pool:
# do work in parallel using pool
pass
# N为系统上可用的CPU的个数
# 向pool中提交工作的方式有两种：results = pool.map(work, data)和future = pool.submit(work, arg)/result = future.result
# 前者迭代所有,后者只提交一个
# 利用生成器实现简单的并发。。。挂起
def save(self,
name=None,
library=None,
grid_steps_per_micron=1000,
parallel=False):
"""
Exports the layout and creates an DLW-file, if DLW-features are used.
:param name: Optionally, the filename of the saved file (without ending).
:param library: Name of the used library.
Currently, for gds-export gdspy and gdscad are supported, for oasis-export fatamorgana is supported.
Setting the library to 'gdshelpers' will select the gds_export of gdshelpers (experimental).
:param grid_steps_per_micron: Defines the resolution
:param parallel: Defines if palatalization is used (only supported in Python 3).
Standard value will be changed to True in a future version.
Deactivating can be useful for debugging reasons.
"""
if not name:
name = self.name
elif name.endswith('.gds'):
name = name[:-4]
library = library or gds_library
elif name.endswith('.oasis'):
name = name[:-6]
library = library or 'fatamorgana'
elif name.endswith('.dxf'):
name = name[:-4]
library = library or 'ezwriter'
library = library or gds_library
if library == 'gdshelpers':
with open(name + '.gds', 'wb') as f:
write_cell_to_gdsii_file(
f,
self,
grid_steps_per_unit=grid_steps_per_micron,
parallel=parallel)
elif library == 'gdspy':
if parallel:
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor() as pool:
self.get_gdspy_cell(pool)
else:
self.get_gdspy_cell()
self.get_gdspy_lib(
).precision = self.get_gdspy_lib().unit / grid_steps_per_micron
gdspy_cells = self.get_gdspy_lib().cell_dict.values()
if parallel:
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor() as pool:
binary_cells = pool.map(gdspy.Cell.to_gds, gdspy_cells,
[grid_steps_per_micron] *
len(gdspy_cells))
else:
binary_cells = map(gdspy.Cell.to_gds, gdspy_cells,
[grid_steps_per_micron] * len(gdspy_cells))
self.get_gdspy_lib().write_gds(name + '.gds',
cells=[],
binary_cells=binary_cells)
elif library == 'gdscad':
layout = gdsCAD.core.Layout(precision=1e-6 / grid_steps_per_micron)
if parallel:
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor() as pool:
layout.add(self.get_gdscad_cell(pool))
else:
layout.add(self.get_gdscad_cell())
layout.save(name + '.gds')
elif library == 'fatamorgana':
layout = fatamorgana.OasisLayout(grid_steps_per_micron)
if parallel:
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor() as pool:
cells = self.get_oasis_cells(grid_steps_per_micron, pool)
else:
cells = self.get_oasis_cells(grid_steps_per_micron)
layout.cells = [cells[0]] + list(set(cells[1:]))
# noinspection PyUnresolvedReferences
def replace_names_by_ids(oasis_layout):
name_id = {}
for cell_id, cell in enumerate(oasis_layout.cells):
if cell.name.string in name_id:
raise RuntimeError(
'Each cell name should be unique, name "' +
cell.name.string + '" is used multiple times')
name_id[cell.name.string] = cell_id
cell.name = cell_id
for cell in oasis_layout.cells:
for placement in cell.placements:
placement.name = name_id[placement.name.string]
oasis_layout.cellnames = {v: k for k, v in name_id.items()}
# improves performance for reading oasis file and workaround for fatamorgana-bug
replace_names_by_ids(layout)
with open(name + '.oas', 'wb') as f:
layout.write(f)
elif library == 'ezdxf':
from gdshelpers.export.dxf_export import write_cell_to_dxf_file
with open(name + '.gds', 'wb') as f:
write_cell_to_dxf_file(f,
self,
grid_steps_per_micron,
parallel=parallel)
else:
raise ValueError(
'library must be either "gdscad", "gdspy" or "fatamorgana"')
dlw_data = self.get_dlw_data()
if dlw_data:
with open(name + '.dlw', 'w') as f:
json.dump(dlw_data, f, indent=True)
def determine_language(string_to_test: str) -> str:
# initializing the scores that every language
dict_language_scores: dict[str, int] = {}
for language in tuple(dict_paths_to_word_collections.keys()):
dict_language_scores[language] = 0
''' saving language scores to 'dict_language_scores.
Using all available cores to speed up the process'''
# Save language names as strings in a tuple
languages: tuple[str] = tuple(dict_paths_to_word_collections.keys())
# using all available cores to calculate the score for a
with ProcessPoolExecutor() as executor:
# bi-/trigrams score for the dutch language
d2: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[0],
2)
d3: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[0],
3)
# bi-/trigrams score for english language
e2: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[1],
2)
e3: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[1],
3)
# bi-/trigrams score for german language
g2: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[2],
2)
g3: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[2],
3)
# bi-/trigrams score for french language
f2: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[3],
2)
f3: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[3],
3)
# bi-/trigrams score for spanish language
s2: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[4],
2)
s3: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[4],
3)
# bi-/trigram score for italian language
i2: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[5],
2)
i3: executor[int] = executor.submit(count_ngram_occurences,
string_to_test, languages[5],
3)
# saving bigram and trigram scores in dictionary
# dutch
dict_language_scores[languages[0]] = d2.result() + d3.result()
# english
dict_language_scores[languages[1]] = e2.result() + e3.result()
# german
dict_language_scores[languages[2]] = g2.result() + g3.result()
# french
dict_language_scores[languages[3]] = f2.result() + f3.result()
# spanish
dict_language_scores[languages[4]] = s2.result() + s3.result()
# italian
dict_language_scores[languages[5]] = i2.result() + i3.result()
# return the language with the highest store
language: str = max(dict_language_scores, key=dict_language_scores.get)
print(f"Language : {language}")
print(f"Bigram and trigram probabilities")
get_gram_probabilities(string_to_test, language)
def __iter__(self):
from concurrent.futures import ProcessPoolExecutor
with ProcessPoolExecutor() as executor:
return _coconut.iter(
_coconut.list(executor.map(self.func, *self.iters)))
line = line.strip()
speaker_used.append(line)
print(speaker_used)
#speaker_used = ['262', '272', '229', '232', '292', '293', '360', '361', '248', '251']
#speaker_used = ['p'+i for i in speaker_used]
#speaker_used = ['p315']
## Next we are to extract the acoustic features (MCEPs, lf0) and compute the corresponding stats (means, stds).
# Make dirs to contain the MCEPs
os.makedirs(mc_dir_train, exist_ok=True)
os.makedirs(mc_dir_test, exist_ok=True)
num_workers = 52 #len(speaker_used) #cpu_count()
print("number of workers: ", num_workers)
executor = ProcessPoolExecutor(max_workers=num_workers)
work_dir = target_wavpath
# spk_folders = os.listdir(work_dir)
# print("processing {} speaker folders".format(len(spk_folders)))
# print(spk_folders)
futures = []
for spk in speaker_used:
spk_path = os.path.join(work_dir, spk)
futures.append(
executor.submit(
partial(get_spk_world_feats, spk_path, mc_dir_train,
mc_dir_test, sample_rate)))
result_list = [future.result() for future in tqdm(futures)]
print(result_list)
def __init__(self, config: Config):
self.config = config
self.model = None # type: ChessModel
self.dataset = deque(),deque(),deque()
self.executor = ProcessPoolExecutor(max_workers=config.trainer.cleaning_processes)
def generate_random_walks(num_walks, walk_length, workers, vertices):
'''
随机游走
@:param num_walks: 重复游走的次数
@:param walk_length: 单次游走的序列长度
@:param vertices: 全部学生节点id
'''
logging.info('Loading similarity_nets on disk...')
graphs = load_from_disk('similarity_nets_graphs') # 多层DSN
alias_method_j = load_from_disk(
'nets_weights_alias_method_j') # 随机游走需要使用到的字典
alias_method_q = load_from_disk(
'nets_weights_alias_method_q') # 随机游走需要使用到的字典
# amount_neighbours = load_from_disk('amount_neighbours')
flag = 1 # 1: max， 2：max(median) 3: max(avg)
# flag = 2
'''1. 获取每个节点取得最大边权的所在网络层数'''
d_max_sim_layer_dict = {}
# 获取各个节点在所有DSN中取得最大权值所在的网络层数
#cui--注释
# ''' 单层网络随机游走不需要获取 '''
# if (len(graphs)) > 1:
# print("# of DSNs>1, now getting the d_max_sim_layer_dict...")
# d_max_sim_layer_dict = get_max_sim_layer(vertices, len(graphs), flag)
# else:
# print("only single layer of DSN, skipping getting d_max_sim_layer_dict.")
#cuiEnd
logging.info('Creating Random Walks...')
t0 = time.time()
walks = deque() # 游走序列，使用队列存储
initialLayer = 0
if (workers > num_walks):
workers = num_walks
with ProcessPoolExecutor(max_workers=workers) as executor:
futures = {}
for walk_iter in range(
num_walks): # 随机游走num_walks轮，每次对全部节点产生一组游走序列，最后将多轮游走的序列拼接在一起
random.shuffle(vertices) # 打乱节点
print('-- walk_iteration:', walk_iter)
# job = executor.submit(exec_ramdom_walks_for_chunck, vertices, graphs, alias_method_j, alias_method_q, walk_length, amount_neighbours)
# alias_method_j 和 alias_method_q 在采样时会使用到，返回对全部节点产生的游走序列
job = executor.submit(exec_random_walks_for_chunck,
d_max_sim_layer_dict, vertices, graphs,
alias_method_j, alias_method_q,
walk_length) # 随机游走
futures[job] = walk_iter
# part += 1
logging.info("Receiving results...")
for job in as_completed(futures):
walk = job.result()
r = futures[job]
logging.info("Iteration {} executed.".format(r))
walks.extend(walk) # 将多轮游走产生的序列拼接在一起
del futures[job]
t1 = time.time()
logging.info('RWs created. Time: {}m'.format((t1 - t0) / 60))
logging.info("Saving Random Walks on disk...")
save_random_walks(walks, num_walks, walk_length)
break
print('False:'+ name)
num = num+1
count = count + 1
file_name = name[:1] + str(num)
break
cap.release()
out.release()
cv2.destroyAllWindows()
res = False
return clock,count,res,file_name
# 检查录制状态
def check(rtsp_url,name):
global res
info = record(rtsp_url,name)
while True:
if info[0] == False: # 计时结束
break
elif info[1] > 10: # 超出连接次数
print('Over count limit!!!')
break
elif info[2] == False: # 连接失败
res = True # 重置连接状态
print('Reopen:' + info[3])
info = record(rtsp_url, info[3])
time.sleep(5)
if __name__ == '__main__':
ProcessPoolExecutor(os.cpu_count()).map(check,addr,name_list)
def run(*args):
if 1 in args:
start = time.time()
f(10_000_000)
f(10_000_000)
f(10_000_000)
# f(10_000_000)
print("1. No parallel time: {}".format(time.time() - start))
if 2 in args:
start = time.time()
t1 = Thread(target=f, args={10_000_000})
t2 = Thread(target=f, args={10_000_000})
t3 = Thread(target=f, args={10_000_000})
# t4 = Thread(target=f, args={10_000_000})
t1.start(), t2.start(), t3.start()
t1.join(), t2.join(), t3.join()
print("2. Threads time: {}".format(time.time() - start))
if 3 in args:
start = time.time()
with ThreadPoolExecutor(max_workers=4) as pool:
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
print("3. Thread pool time: {}".format(time.time() - start))
if 4 in args:
start = time.time()
p1 = Process(target=f, args={10_000_000})
p2 = Process(target=f, args={10_000_000})
p3 = Process(target=f, args={10_000_000})
# p4 = Process(target=f, args={10_000_000})
p1.start(), p2.start(), p3.start()
p1.join(), p2.join(), p3.join()
print("4. Process time: {}".format(time.time() - start))
if 5 in args:
start = time.time()
with ProcessPoolExecutor(max_workers=4) as pool:
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
pool.submit(f(10_000_0000))
print("5. Process pool time: {}".format(time.time() - start))
#!/usr/bin/python
from concurrent.futures import ProcessPoolExecutor
from os import getpid as pid
from time import sleep
L = [1, 2, 3, 4, 5]
# 单进程,要用至少25秒
for i in L:
sleep(5)
print(pid(), "已过5秒", i)
# 多进程并发，5秒
def SleepPrint(in_value):
sleep(5)
print(pid(), "已过5秒", in_value)
pool = ProcessPoolExecutor(5)
for i in L:
pool.submit(SleepPrint, i)
import unicodedata
import requests
from gtts import gTTS
import youtube_dl
from config import BOTID
from config import OWNERID
from config import COMMANDPREFIX
import discord
from config import chatbot, PROCESS_POOL_EXECUTOR_COUNT, WAIT_TIME_BEFORE_TYPING, WAIT_TIME_RESPONSE_READY, COMMANDPREFIX, BOTID
print("Starting process")
#f = open("vc.wav", "wb")
client = discord.Client()
ppool = ProcessPoolExecutor(PROCESS_POOL_EXECUTOR_COUNT)
async def start(self):
discord.opus.load_opus(self.opus_library)
@client.event
async def on_ready():
print('Logged in as')
print(client.user.name)
print(client.user.id)
print('------')
await client.change_presence(game=discord.Game(name="Sayori Discord Wrapper"))
def generate_response(message):
return chatbot.get_response(message)
from typing import Tuple
from ..fonts import Font
from .text import Text
def points_to_millimeters(points: float) -> int:
return int(points * 0.352778)
def millimeters_to_points(millimeters: float) -> int:
return int(millimeters * 2.83465)
loop = asyncio.get_event_loop()
process_executor = ProcessPoolExecutor()
class PageImageWriter:
def write_text(
self, position: Tuple[int, int], window: Tuple[int, int], text: str, font: Font
) -> Tuple[int, int]:
"""
Draw the supplied text with given font at given position
:param position: position to draw text at
:param window: size of the window the text must fit in
:param text: string to draw in the image
:param font: font info to generate text
:return: None
"""
raise NotImplementedError()
