def screencap_h264(device: AdbDevice, executor: Executor):
start = time()
device.push("../scrcpy-win64/scrcpy-server.jar",
"/data/local/tmp/scrcpy-server.jar")
print("Pushed server component")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = device.adb_output("reverse", "tcp:5556", "localabstract:scrcpy")
print("[ADB]", port)
# sock.bind(("127.0.0.1", 5556))
# sock.listen()
executor.submit(lambda: print(
"[SCRCPY]",
device.shell("CLASSPATH=/data/local/tmp/scrcpy-server.jar "
"app_process / com.genymobile.scrcpy.Server 0 8000000 "
"false - false true")))
print("Executed scrcpy")
print("Waiting for connection")
# conn, info = sock.accept()
# print(info)
# print(conn.recv(1))
# print(conn.recv(64))
# print(conn.recv(2))
# print(conn.recv(2))
# print("Closing transmission")
# device.shell("\x03")
# conn.close()
# sock.close()
print("Recv h264 took %f seconds" % (time() - start))
def submit_tasks(executor: Executor, window_size: int, fn, *iterables):
""" Assuming all task take approximately the same time. Effectively executor.map, but done in batches/windows to
reduce the number of queued Future objects (seems to consume a lot of memory). """
args_iterator = zip(*iterables)
fs_queue = deque(
(executor.submit(fn, *args) for args in itertools.islice(args_iterator, window_size)),
maxlen=window_size,
)
try:
while fs_queue:
# following comment copied from Executor.map
# Careful not to keep a reference to the popped future
yield fs_queue.popleft()
fs_queue.append(executor.submit(fn, *next(args_iterator)))
except StopIteration:
# no more tasks to submit
pass
# collect remaining tasks
while fs_queue:
# Careful not to keep a reference to the popped future
yield fs_queue.popleft()
def _search_async_locked(self, pool: futures.Executor, parser: MP3Parser,
depth: int) -> List[futures.Future]:
"""
Walk self._fullpath scheduling tasks to find all songs and
sub-directories.
Checking of files and subdirectories is submitted as tasks
to the specified pool. Must be called with the lock held
"""
cache = self._load_cache()
tasks: List[futures.Future] = []
assert self._fullpath is not None
if not self._fullpath.is_dir():
raise IOError(f'Directory "{self._fullpath}" does not exist')
self.log.debug('Search %s', self._fullpath.name)
for index, filename in enumerate(self._fullpath.iterdir()):
abs_fname = str(filename)
fstats = os.stat(abs_fname)
if stat.S_ISDIR(fstats.st_mode):
subdir = Directory(self, 1000 * (self.ref_id + index),
filename)
self.subdirectories.append(subdir)
tasks.append(
pool.submit(subdir.search_async, pool, parser, depth + 1))
elif (stat.S_ISREG(fstats.st_mode)
and abs_fname.lower().endswith(".mp3")
and fstats.st_size <= self.maxFileSize):
self._todo += 1
task = pool.submit(self._parse_song, parser, cache, filename,
index)
task.add_done_callback(self._after_parse_song)
tasks.append(task)
return tasks
def inference_all_questions(self, question_folder, cpu_num=1):
base_dir = os.getcwd()
print('base_dir: ', base_dir)
# os.makedirs(question_folder, exist_ok=True)
os.chdir(question_folder)
print('numcores: ', cpu_num)
audios = os.listdir()
audios = [audio for audio in audios if audio.endswith('.wav')]
if cpu_num > 1:
audios_for_one_process = len(audios) // cpu_num
if audios_for_one_process < 1:
audios_for_one_process = 1
print('audios count: ', len(audios))
print('audio_for_one_process: ', audios_for_one_process)
tmp = []
splitted_audio = []
for audio in audios:
if audios_for_one_process == len(tmp):
splitted_audio.append(tmp)
tmp = []
tmp.append(audio)
elif audio == audios[-1]:
# 마지막까지 갔을 때..
splitted_audio.append(tmp)
else:
tmp.append(audio)
else:
splitted_audio = audios
print('splitted audios: {}'.format(len(splitted_audio)))
if cpu_num > 1:
exe = Executor(max_workers=cpu_num)
futures = []
for audio in splitted_audio:
futures.append(
exe.submit(self.inference_splitted_audios, audio))
results = [future.result() for future in tqdm(futures)]
print(results)
# print('multiprocessing Result: ', len(result))
# exe.shutdown(wait=True)
# exe.shutdown(wait=True)
# print(future.result)
# result = parmap.map(self.inference_splitted_audios, splitted_audio, pm_pbar=True, pm_processes=cpu_num)
else:
with Executor(max_workers=cpu_num) as exe:
exe.map(self.inference_audio, splitted_audio)
exe.shutdown(wait=True)
# result = parmap.map(self.inference_audio, splitted_audio, pm_pbar=True, pm_processes=cpu_num)
os.chdir(base_dir)
return question_folder, results
def svm_kernels(kernels,
model,
Cs=10.**np.arange(-3, 4),
prediction_file=None,
repeats=1,
**params):
"""
Evaluates a SVM model with the specified kernels.
- First, optimizes C value using a grid search for each of the data sets.
- Evaluates the performance of the best C .
- Trains on the full data set and generates test predictions (if prediction_file is not None).
:param kernels: (train_Ks, test_Ks) containing train and test kernels for all data sets
:param model: model constructor
:param Cs: values of C to use in the grid search
:param prediction_file: file to save the predictions to
:param repeats: number of repetitions of the k-fold cross validations.
:param params: parameters of the model (excluding C)
:return: detailed validation score over each of the 3 data sets.
"""
train_Ks, test_Ks = kernels
params = [dict(C=C, **params) for C in Cs]
total_perf = np.zeros(4)
full_results = []
with Executor(max_workers=3) as executor:
futures = [
executor.submit(grid_search, model, params, K, Y, repeats=repeats)
for K, Y in zip(train_Ks, train_Ys)
]
res = [future.result() for future in futures]
for p, performance in res:
total_perf += performance
percentages = tuple(100 * performance)
print(
'dataset: Validation {:.2f} ± {:.2f}\t Train {:.2f} ± {:.2f}\t C={:.0e}'
.format(*percentages, p['C']))
full_results.append(performance)
if prediction_file is not None:
with Executor(max_workers=3) as executor:
futures = [
executor.submit(final_train, model, p, K, Y, K_test)
for K, Y, K_test, (p,
_) in zip(train_Ks, train_Ys, test_Ks, res)
]
predictions = [future.result() for future in futures]
save_predictions(predictions, prediction_file)
total_percentages = 100 * total_perf / 3
print(
'total: Validation {:.2f} ± {:.2f}\t Train {:.2f} ± {:.2f}\t'.format(
*total_percentages))
return full_results
def status(pool: Executor, cwd: PurePath) -> VCStatus:
if which("git"):
try:
r = pool.submit(root, cwd=cwd)
s_main = pool.submit(_stat_main, cwd=cwd)
s_sub = pool.submit(_stat_sub_modules, cwd=cwd)
wait(cast(Sequence[Future], (r, s_main, s_sub)))
return _parse(r.result(), stats=chain(s_main.result(), s_sub.result()))
except CalledProcessError:
return VCStatus()
else:
return VCStatus()
def batchedPoolRunner(testgenfunc: Function, dispatchfunc: Function,
pool: Executor, size: int, validator: Function) -> int:
"""
Given a concurrent.futures.pool run the tuples produced by testgenfunc in size chunks.
Submit results back to pool using dispatchfunc and the returned result of the func.
testgenfunc is a function generator that produces a tuple with function to run and parameters to the func.
The testgenfunc must return a two tuple with a function in the first position and the function parameters
in the second position as a dictionary.
dispatchfunc must return an object, list or tuple and these should be compatible with its own inputs.
POOL can be a ThreadPoolExecutor or a ProcessPoolExecutor
size is the processing batch size for submitting to the pool.
testgenfunc should produce tuple with the first element as the function and the second element the
parameters to the function
NOTE: never create a generator that produces closures. Python internally updates the closure in place
instead of creating a new one so you'll effectively have the same closure produced throughout the
generators life. It's a nasty bug.
*** Currently Doesn't work on Process Pools. Working on a solution.
"""
td = testgenfunc()
futures = set([pool.submit(f, **p) for f, p in chunk(size, td)])
count = 0
debug("batchedPoolRunner: Starting main loop.")
while len(futures) > 0:
done = set()
for job in as_completed(futures):
if count % 1000 == 0: gc.collect()
rslt = job.result() if job is not None else None
if rslt is not None:
if not validator(rslt):
validationFailure("Test case for %s failed validation." %
rslt.Function)
else:
serviceError("FutureResult from thread pool is None.")
done.add(job)
count += 1
#futures.remove(job)
if dispatchfunc is not None:
#debug("Running dispatchfunc %s."%dispatchfunc.__name__)
if rslt: futures.add(pool.submit(dispatchfunc, rslt))
sys.stdout.write(".")
sys.stdout.flush()
futures = futures - done
if len(futures) < 1000:
debug("Adding new jobs")
futures.update(
set([pool.submit(f, **p) for f, p in chunk(size, td)]))
debug(count)
return count
def preprocess_cycles(client: InfluxDBClient, executor: Executor, manager: SyncManager, dry_run=False):
logger.info("Preprocessing charge cycles")
queue = manager.Queue()
series = client.list_series("samples")
futures = []
# TODO merge results of different detectors
for attr, where, detector in [
('charger_acvoltage', 'charger_acvoltage>0 OR veh_speed > 0',
ChargeCycleACVoltageDetection(time_epoch=client.time_epoch)),
('ischarging', 'ischarging>0 OR veh_speed > 0',
ChargeCycleIsChargingDetection(time_epoch=client.time_epoch)),
('ac_hvpower', 'ac_hvpower>0 OR veh_speed > 0',
ChargeCycleACHVPowerDetection(time_epoch=client.time_epoch)),
('hvbatt_soc', 'hvbatt_soc<200', ChargeCycleDerivDetection(time_epoch=client.time_epoch))
]:
fields = ["time", "participant", "hvbatt_soc", "veh_speed"]
if attr not in fields:
fields.append(attr)
futures += [executor.submit(preprocess_cycle,
nr, client, queue, sname, join_selectors([sselector, where]),
fields, detector, dry_run)
for nr, (sname, sselector) in enumerate(series)]
logger.debug("Tasks started, waiting for results...")
async_progress(futures, queue)
data = [f.result() for f in futures]
logger.debug("Tasks done")
data.sort(key=lambda a: a[0:1])
logger.info(__("Detected charge cycles:\n{}", tabulate(data, headers=["attr", "#", "cycles", "cycles_disc"])))
def run_prediction(project_id, model_id):
# 学習データ読み込み
try:
thread_id = "{}_{}".format(project_id, model_id)
fields = 'hyper_parameters,algorithm,algorithm_params,best_epoch_weight,dataset_def_id'
data = storage.fetch_model(project_id, model_id, fields=fields)
(id, name, ratio, train_imgs, valid_imgs, class_map, created,
updated) = storage.fetch_dataset_def(data['dataset_def_id'])
# weightのh5ファイルのパスを取得して予測する
with Executor(max_workers=MAX_THREAD_NUM) as prediction_executor:
th = PredictionThread(thread_id, model_id,
data["hyper_parameters"], data["algorithm"],
data["algorithm_params"],
data["best_epoch_weight"], class_map)
ft = prediction_executor.submit(th)
prediction_thread_pool[thread_id] = [ft, th]
ft.result()
if th.error_msg is not None:
body = json.dumps({"error_msg": th.error_msg})
else:
data = {
"predict_results": th.predict_results,
"csv": th.csv_filename,
}
body = json.dumps(data)
except Exception as e:
traceback.print_exc()
body = json.dumps({"error_msg": e.args[0]})
ret = create_response(body)
return ret
def test_exe():
cgroup_name = 'test_cgroup_executor'
setup_cgroup(cgroup_name)
print('after setup_cgroup')
def dummy(cgroup_name):
pid = os.getpid()
print('My pid: ', pid)
cg = Cgroup(cgroup_name)
print(cg.pids)
print('dummy groups', cg.cgroups)
print('dummy user', cg.user)
print('dummy cpu limit', cg.cpu_limit)
print('dummy memory limit', cg.memory_limit)
cg.add(pid)
print('leaving dummy')
executor = Executor(max_workers=1, initializer=dummy, initargs=(cgroup_name,))
with executor as exe:
f = exe.submit(mem_grower)
# exe.submit(time.sleep, 25)
# exe.shutdown(wait=True)
print(f.result())
def _parallel_pivot_finder(self, X):
N = len(X)
for k in range(self._dim):
print('Working on ' + str(k) + 'D projection')
left_pivot_index = random.randint(0, N)
right_pivot_index = left_pivot_index
for m in range(self._iters):
left_pivot_index = right_pivot_index
left_pivot = X[left_pivot_index]
with Executor(max_workers=self._cores) as executor:
map_results = executor.map(self._mapper,
[(X[i], i, left_pivot, k)
for i in range(N)])
distributor = defaultdict(list)
for key, value in map_results:
distributor[key].append(value)
reduced = executor.map(self._reducer, distributor.items())
right_pivot_index = -1
max_dist = -1
for (idx, dist) in reduced:
if dist > max_dist:
right_pivot_index = idx
max_dist = dist
print('Left Pivot: ' + str(left_pivot_index) + '\nRight Pivot: ' +
str(right_pivot_index))
left = X[left_pivot_index]
left_proj = self._i_proj(left, k)
right = X[right_pivot_index]
right_proj = self._i_proj(right, k)
final_pivots = Pivots(left, left_proj, right, right_proj, max_dist)
self._pivots.insert(k, final_pivots)
def submit(self, num_workers=None):
"""
Start a scan and distribute it on `num_workers` threads.
If `num_workers` is omitted, the value of `os.cpu_count()` is used.
Results are stored in `self.results`.
"""
num_workers = os.cpu_count() if not num_workers else num_workers
self.parallel = num_workers
logging.info('Running on host {}.'.format(os.getenv('HOSTNAME')))
logging.info('Will work on %d threads in parallel.' % num_workers)
if num_workers == 1:
self.results = self.scan(self.numparas)
return
paras_per_thread = int(self.numparas / num_workers)
remainder = self.numparas % num_workers
numparas = [paras_per_thread for p in range(num_workers)]
numparas[-1] += remainder
with Executor(num_workers) as executor:
futures = [
executor.submit(self.scan, j, i)
for i, j in enumerate(numparas)
]
self.results = [r.result() for r in as_completed(futures)]
self.results = concat(self.results, ignore_index=True)
def main(set, config):
engine = create_engine(settings.database.get("url"))
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
"""
Lists of Endpoint and Transfer objectwith locks are shared between threads.
The classes and objects store all locks and necessary information to
sychronized transfers.
"""
global endpoints
global transfers
endpoints = [Endpoint(e) for e in config.get("endpoints")]
datasets = list(config.get("datasets").keys())
params = config.get("params")
if params:
GlobusTransfer.deadline = params.get("deadline", 3600)
transfers = [
GlobusTransfer(set, s, d, t) for s in endpoints for d in endpoints
for t in datasets
]
executor = Executor(max_workers=4)
while GlobusTransfer.transfers2do > 0:
for t in transfers:
if t.done:
continue
if t.acquire():
executor.submit(t.run)
sleep(10)
def run_test():
data = [{
"sepal_length": 5.0,
"sepal_width": 3.2,
"petal_length": 1.2,
"petal_width": 0.2
}, {
"sepal_length": 5.5,
"sepal_width": 3.5,
"petal_length": 1.3,
"petal_width": 0.2
}, {
"sepal_length": 4.9,
"sepal_width": 3.1,
"petal_length": 1.5,
"petal_width": 0.1
}, {
"sepal_length": 4.4,
"sepal_width": 3.0,
"petal_length": 1.3,
"petal_width": 0.2
}]
with Executor(max_workers=4) as exe:
jobs = [exe.submit(request_task, d) for d in data]
results = [job.result() for job in jobs]
print("The tasks returned these predictions: {}".format(results))
def read_all_sequences(
cls,
input_dir: str,
executor: futures.Executor = None
) -> typing.List[typing.List[knowledge_graph.KnowledgeGraph]]:
"""Loads all knowledge-graph sequences that are discovered in the specified directory.
Args:
input_dir (str): The path of the directory that is being searched.
executor (futures.Executor, optional): An optional executor for loading multiple knowledge-graph sequences
concurrently.
Returns:
list[list[:class:`knowledge_graph.KnowledgeGraph`]]: All knowledge-graph sequences that were found in
``input_dir``.
Raises:
ValueError: If the specified directory does not exist.
"""
# sanitize args
input_dir = str(input_dir)
if not os.path.isdir(input_dir):
raise ValueError("The specified <input_dir> does not exist: '{}'!".format(input_dir))
insanity.sanitize_type("executor", executor, futures.Executor, none_allowed=True)
# find all knowledge-graph sequences in the input directory
all_seq = io.find_knowledge_graph_sequences(input_dir)
# load all knowledge graphs that were found
if executor is None:
return [cls.read_sequence(input_dir, seq) for seq in all_seq]
else:
all_seq = [os.path.join(input_dir, seq) for seq in all_seq]
return list(executor.map(cls._read_seq_from_one, all_seq))
def start(self):
self.beating = True
# 监视新的beater
ps = self._get_redis().pubsub()
ps.subscribe(self.prefix_key + ":new-interval")
def _exit(signum, frame):
ps.unsubscribe()
self.stop()
signal.signal(signal.SIGINT, _exit)
signal.signal(signal.SIGTERM, _exit)
self.workers = Executor(max_workers=self.max_beaters)
try:
for interval in self.beaters:
f = self._create_beater(interval)
f.add_done_callback(lambda x: self.beaters.discard(interval))
self.logger.warning('%d beater started' % len(self.beaters))
self._watch_new_interval(ps)
except KeyboardInterrupt:
_exit(None, None)
self.logger.warning('heartbeat maker exit')
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser()
parser.add_argument('movies', type=str, nargs='+')
group = parser.add_mutually_exclusive_group()
group.add_argument('--jobs', '-j', type=int, default=1)
group.add_argument('--csv-out', '-c', type=str)
parser.add_argument('--ending', '-e', type=int)
parser.add_argument('--viewer', '-v', type=str)
parser.add_argument('--no-video', '-V', action='store_true')
parser.add_argument('--info-dict', '-i', action='store_true')
parser.add_argument('--npy-actions', '-a', action='store_true')
parser.add_argument('--lossless', '-L', type=str, choices=['mp4', 'mp4rgb', 'png', 'ffv1'])
args = parser.parse_args(argv)
monitor_csv = None
monitor_file = None
if args.csv_out:
m0 = retro.Movie(args.movies[0])
game = m0.get_game()
reward_fields = ['r'] if m0.players == 1 else ['r%d' % i for i in range(m0.players)]
monitor_file = open(args.csv_out, 'w')
monitor_file.write('#{"t_start": 0.0, "gym_version": "gym_retro", "env_id": "%s"}\n' % game)
monitor_csv = csv.DictWriter(monitor_file, fieldnames=reward_fields + ['l', 't'])
monitor_csv.writeheader()
with Executor(args.jobs or None) as pool:
list(pool.map(_play, *zip(*[(movie, args, monitor_csv) for movie in args.movies])))
if monitor_file:
monitor_file.close()
def test_pickle():
print_testname(test_pickle.__name__)
print_runtime_flag()
print_msg("* Should be in color.")
print_msg("* Should contain title and status.")
print_msg("* Should contain stdout and stderr messages.")
print_msg("* Should contain stdout and stderr of subprocess as well.")
p = Pipe(dup=True, save=True, tty=True, text=True, mute=False)
with Executor() as excecutor:
future = excecutor.submit(cmd_print_stdout_stderr,
'foo',
with_sub=True,
_stdout=p,
_stderr=p)
cr = future.result()
print_result_flag('out')
print_msg("* Should contain all stdout lines in color.")
print_msg("* Should contain stdout and stderr of subprocess as well.")
print(cr.stdout)
print_result_flag('err')
print_msg("* Should contain all stderr lines in color.")
print_msg("* Should contain stdout and stderr of subprocess as well.")
print(cr.stderr)
def outdated_pings(outdated_clients: dict):
"""
Appends comments to clients if pings or not.
Used with synchronized decorator for parallel checks (required def with concurrent decorator)
:param outdated_clients: dict formatted with clients as keys
:return: dict with appended comments
"""
outdated = outdated_clients
with Executor(max_workers=10) as exe:
# jobs are executed in parallel with exe.submit
# params passed to exe.submit are: (function, args, args)
# in this case we pass client, True. True is to get a return as dict
# with key as client and new comments
# we use one are from a list appending a for in the list
jobs = [exe.submit(check_isup, k, True) for k in outdated.keys()]
# results are generated from jobs
results = [job.result() for job in jobs]
for i, values1 in enumerate(results):
data_client = list(values1.items())
client = data_client[0][0]
comment = data_client[0][1].get('comments')
# Append ping information to outdated_clients
# as we have same format in results as the original dict, we just update the values.
# dict returned by results is {k: {'comments': comments}}
outdated[client]['comments'] = comment
return outdated
def main() -> None:
"""The main function."""
args = parse_args()
# Check the target
target = args.target
if not os.path.isfile(target):
raise Exception('Target `%s` does not exist' % target)
# Maxmimum number of tasks. Two 2 tasks are required as a minimum: one for
# running afl-showmap, and another for updating the coverage bitmap
max_task = args.jobs
if max_task == 0:
max_task = multiprocessing.cpu_count()
# Check afl-showmap
if not which('afl-showmap'):
raise Exception('Could not find afl-showmap. Check PATH')
# Wait for fuzzer_stats to exist
out_dir = args.out_dir
fuzzer_stats_path = out_dir / 'fuzzer_stats'
if not fuzzer_stats_path.exists():
raise Exception('No fuzzer_stats in `%s`' % out_dir)
# Open CSV plot_data
csv_path = args.csv
if csv_path:
csv_path = Path(csv_path)
with open(csv_path, 'w') as outf:
CsvDictWriter(outf, fieldnames=CSV_FIELDNAMES).writeheader()
with Executor(max_workers=max_task) as executor, \
open(out_dir / 'blackbox.tar.gz', 'wb') as tar_file:
# The coverage bitmap
cov_bitmap = [255] * MAP_SIZE
cov_queue = Queue(max_task)
# Thread responsible for deduplicating entries in the output directory
# and logging coverage to a CSV
cov_thread = Thread(target=remove_duplicate_testcases,
args=(cov_bitmap, cov_queue, tar_file, csv_path))
cov_thread.daemon = True
cov_thread.start()
# Start the monitor
handler = TestCaseHandler(executor, cov_queue, target, afl_stats,
args.timeout)
observer = Observer()
observer.schedule(handler, out_dir / 'queue' / '.blackbox')
observer.start()
# Continue until interrupted
try:
while observer.is_alive():
observer.join(1)
except KeyboardInterrupt:
print('\nCtrl-C detected, goodbye')
observer.stop()
observer.join()
def main():
'''stdin is assumed to be the output of
./find_commits.sh [some project directory] [some project name]'''
worklist = flatten_lines(sys.stdin)
from concurrent.futures import ThreadPoolExecutor as Executor
with Executor(max_workers=150) as executor:
utils.exhaust(executor.map(fetch_and_compose, worklist))
def submit(self, num_workers=None):
"""
Start a scan and distribute it on `num_workers` threads.
If `num_workers` is omitted, the value of `os.cpu_count()` is used.
Results are stored in `self.results`.
"""
num_workers = os.cpu_count() if not num_workers else num_workers
if not self.scanset:
self.build(num_workers)
if num_workers == 1:
runner = self.runner(self.config['runner'])
self.results = concat([runner.run(d) for d in tqdm(self.scanset)],
ignore_index=True)
return
chunksize = min(int(self.numparas / num_workers), 1000)
chunks = range(0, self.numparas, chunksize)
logging.info('Running on host {}.'.format(os.getenv('HOSTNAME')))
logging.info('Splitting dataset into %d chunks.' % len(chunks))
logging.info('Will work on %d chunks in parallel.' % num_workers)
with Executor(num_workers) as executor:
futures = [
executor.submit(self.scan, self.scanset[i:i + chunksize])
for i in chunks
]
progresser = tqdm(as_completed(futures),
total=len(chunks),
unit='chunk')
self.results = [r.result() for r in progresser]
self.results = concat(self.results, ignore_index=True)
def _run_in_pool(executor: Executor,
func: Func[T, R],
items: List[T],
*args: Any,
chunk_size: Optional[int] = None) -> Iterable[R]:
if chunk_size is None:
chunk_size = 1
chunks: Iterable[List[T]] = _split_to_chunks(items, chunk_size)
exceptions: List[BaseException] = []
with executor:
futures: Dict[Future, int] = {}
for chunk in chunks:
future = executor.submit(func, chunk, *args)
futures[future] = len(chunk)
with tqdm(total=len(items)) as pbar:
for future in as_completed(set(futures.keys())):
pbar.update(futures[future])
del futures[future]
e = future.exception()
if e is not None:
exceptions.append(e)
else:
yield future.result()
if len(exceptions) > 0:
raise exceptions[0]
def _parallel_transform(self, X):
with Executor(max_workers=self._cores) as executor:
map_results = executor.map(
lambda x: (x[0], self._i_proj(x[1], self._dim)), enumerate(X))
map_results = list(map_results)
map_results.sort(key=lambda x: x[0])
return [result[1] for result in map_results]
def find(project: str,
look_back: datetime,
evg_api: EvergreenApi,
n_threads: int = DEFAULT_THREADS) -> Dict:
"""
Find test flips in the evergreen project.
:param project: Evergreen project to analyze.
:param look_back: Look at commits until the given project.
:param evg_api: Evergreen API.
:param n_threads: Number of threads to use.
:return: Dictionary of commits that introduced task flips.
"""
LOGGER.debug("Starting find_flips iteration")
version_iterator = evg_api.versions_by_project(project)
with Executor(max_workers=n_threads) as exe:
jobs = []
for next_version, version, prev_version in windowed_iter(
version_iterator, 3):
log = LOGGER.bind(version=version.version_id)
log.debug("Starting to look")
if version.create_time < look_back:
log.debug("done", create_time=version.create_time)
break
work_item = WorkItem(version, next_version, prev_version)
jobs.append(exe.submit(_flips_for_version, work_item))
results = [job.result() for job in jobs]
return {r.revision: r.flipped_tasks for r in results if r.flipped_tasks}
def parallel_dists(dist_fn, weights, X, Y=None, tqdm=False):
X = ag.tensor(X.astype(np.int))
if Y is None:
Y = X
Ys = [Y[i:].data for i in range(len(X))]
else:
Y = ag.tensor(Y.astype(np.int))
Ys = [Y.data for i in range(len(X))]
weights = ag.tensor(weights)
d = np.zeros((len(X), len(Y)), dtype=np.float32)
g = np.empty((len(weights), len(X), len(Y)), dtype=np.float32)
with Executor(max_workers=multiprocessing.cpu_count()) as executor:
iterator = enumerate(
executor.map(dist_fn, X.data, Ys, [weights.data] * len(X)))
if tqdm:
iterator = tqdm_notebook(iterator, total=len(X))
for i, (row, grad_row) in iterator:
if len(row) < len(X):
d[i, i:] = row
d[i, :i] = d[:i, i]
g[:, i, i:] = grad_row
g[:, i, :i] = g[:, :i, i]
else:
d[i] = row
g[:, i] = grad_row
g_tensor = ag.Tensor(g, None, children=[])
def grad_d(leaf_id):
return ag.tensordot(weights.compute_grad(leaf_id),
g_tensor,
axes=([-1], [0]))
return ag.Tensor(d, grad_d, children=[weights])
def multi_evaluate(repetitions):
"""
To be called when a chromosome needs to be evaluated.
Ace0 will provide the fitness score.
:param repetitions: number of times for the simulation to run
:return: a fitness score for the chromosome as a float
"""
# hold the total fitness
total_fitness = 0
# process pool operator creates/ submits as many process's as we need to evaluate the chromosome
with Executor(max_workers=5) as executor:
processes = [
executor.submit(ace_zero.return_fitness_ace_zero)
for x in range(repetitions)
]
# as the processes are completed, they are summed up
for f in as_completed(processes):
try:
total_fitness += f.result()
except ValueError:
print 'Multiprocessing error'
return total_fitness / repetitions,
def install_packages(project_path: str):
"""Installs packages in virtual environment
Args:
project_path (str): Path to where the project lives
"""
venv_init()
env, pip = get_env_vars(project_path)
needed_packages = get_user_packages()
print(to_be_installed_str(needed_packages, on=True))
start = time.perf_counter()
with Executor() as executor:
threads = list(
tqdm(executor.map(async_install, repeat([env, pip]), needed_packages), total=len(needed_packages)))
finish = time.perf_counter()
timing = round(finish - start, 3)
installed_packages, total_packages = get_installed_packages(project_path)
total_main_packages = len(needed_packages)
total_dependencies = total_packages - total_main_packages
print(installed_packages_str(total_main_packages, total_dependencies, timing))
print(list_packages_str(installed_packages))
def _bulk_cleanup(self, jobs: Sequence["Job"],
ex: Executor) -> Iterable["Job"]:
jobs = self.bulk_sync_status(jobs)
# safety check
for job in jobs:
assert job.driver == self.__class__
if job.status in (Job.Status.SUBMITTED, Job.Status.RUNNING):
raise InvalidJobStatus(
f"Job {job} might be running, please kill first")
logger.debug("Cleaning up %d jobs", len(jobs))
def run(job: Job) -> Job:
for d in ["log_dir", "output_dir"]:
try:
path = job.data[d]
if os.path.exists(path):
logger.debug("Path %s exists, attempting to delete",
path)
rmtree(path)
except Exception:
logger.error("Unable to remove directory %s", d)
return job
futures = [ex.submit(run, j) for j in jobs]
for f in as_completed(futures):
yield f.result()
def __init__(self, job_id=None):
self.data_dir = os.path.join(file_utils.get_project_base_directory(),
'data')
self.job_id = str(
uuid.uuid1()) if job_id is None else "{}".format(job_id)
self.meta_table = _DTable('__META__', '__META__', 'fragments', 10)
self.pool = Executor()
Standalone.__instance = self
