def process_execution():
start = time.time()
with ProcessPoolExecutor(max_workers=5) as executor:
results = executor.map(is_prime, NUMBERS)
for result in results:
print(result)
print('start : {} Time taken : {}'.format(start, time.time()-start) )
async def add_lang_features(args):
database = args["database"]
dataset_db = f"sqlite:///{database}"
loop = asyncio.get_event_loop()
with ProcessPoolExecutor(max_workers=args["max_workers"]) as executor:
await save_language_features(args["batch_size"], dataset_db, executor, loop)
def __init__(self, coin_store: CoinStore,
consensus_constants: ConsensusConstants):
self.constants: ConsensusConstants = consensus_constants
self.constants_json = recurse_jsonify(
dataclasses.asdict(self.constants))
# Transactions that were unable to enter mempool, used for retry. (they were invalid)
self.potential_txs: Dict[bytes32, Tuple[SpendBundle, CostResult,
bytes32]] = {}
# Keep track of seen spend_bundles
self.seen_bundle_hashes: Dict[bytes32, bytes32] = {}
self.coin_store = coin_store
self.mempool_max_total_cost = int(self.constants.MAX_BLOCK_COST_CLVM *
self.constants.MEMPOOL_BLOCK_BUFFER)
self.potential_cache_max_total_cost = int(
self.constants.MAX_BLOCK_COST_CLVM *
self.constants.MEMPOOL_BLOCK_BUFFER)
self.potential_cache_cost: int = 0
self.seen_cache_size = 10000
self.pool = ProcessPoolExecutor(max_workers=1)
# The mempool will correspond to a certain peak
self.peak: Optional[BlockRecord] = None
self.mempool: Mempool = Mempool(self.mempool_max_total_cost)
def run_in_process_pool(func: Func[T, R],
items: List[T],
*args: Any,
chunk_size: Optional[int] = None,
workers_count: Optional[int] = None) -> Iterable[R]:
if workers_count is not None and workers_count <= 0:
raise ValueError("workers_count must be greater than 0")
if len(items) == 0:
return
if workers_count is None:
workers_count = os.cpu_count() or 1
if chunk_size is None:
chunk_size = math.ceil(len(items) / workers_count)
if chunk_size > MAX_PROCESS_POOL_CHUNK_SIZE:
chunk_size = MAX_PROCESS_POOL_CHUNK_SIZE
executor = ProcessPoolExecutor(max_workers=workers_count)
yield from _run_in_pool(executor,
func,
items,
*args,
chunk_size=chunk_size)
def process_and_store_recordings(
self,
recordings: Sequence[Recording],
output_manifest: Optional[Pathlike] = None,
num_jobs: int = 1,
) -> FeatureSet:
if num_jobs == 1:
# Avoid spawning subprocesses for single threaded processing
feature_set = FeatureSet.from_features(
tqdm(
chain.from_iterable(
map(self._process_and_store_recording, recordings)),
total=len(recordings),
desc="Extracting and storing features",
))
else:
with ProcessPoolExecutor(
num_jobs,
mp_context=multiprocessing.get_context("spawn")) as ex:
feature_set = FeatureSet.from_features(
tqdm(
chain.from_iterable(
ex.map(self._process_and_store_recording,
recordings)),
total=len(recordings),
desc="Extracting and storing features in parallel",
))
if output_manifest is not None:
feature_set.to_file(output_manifest)
return feature_set
def test_indicators_picklable(self):
bt = Backtest(SHORT_DATA, SmaCross)
with ProcessPoolExecutor() as executor:
stats = executor.submit(Backtest.run, bt).result()
assert stats._strategy._indicators[
0]._opts, '._opts and .name were not unpickled'
bt.plot(results=stats, resample='2d', open_browser=False)
def download_all_tf_models():
with ProcessPoolExecutor() as executor:
list(
executor.map(
partial(download_tf_model, model_dir="model"),
# for a complete list of architecture name supported, see
# mmdnn/conversion/examples/tensorflow/extractor.py
[
"vgg16",
# "vgg19",
# "inception_v1",
# "inception_v3",
# "resnet_v1_50",
# # "resnet_v1_152",
# "resnet_v2_50",
# "resnet_v2_101",
# # "resnet_v2_152",
# # "resnet_v2_200",
# "mobilenet_v1_1.0",
# "mobilenet_v2_1.0_224",
# "inception_resnet_v2",
# "nasnet-a_large",
# "facenet",
# "rnn_lstm_gru_stacked",
],
))
def test(path, run_id, runs):
# load parser and data handler
parser = CYKParser.load(path)
data_handler = DataHandler(config.test_set, run_id, runs)
# parse sentences in parallel
executor = ProcessPoolExecutor(config.processes)
futures = [
executor.submit(parse_tree, parser, sent, run_id)
for sent in data_handler.generator()
]
# following code is to track progress
kwargs = {
'total': len(futures),
'unit': 'nap',
'unit_scale': True,
'leave': True
}
for _ in tqdm(as_completed(futures), **kwargs):
pass
for future in futures:
if future.exception() is not None:
print(future.exception())
# stitch files if number of runs is 1
if runs == 1:
stitch_files()
print("Done parsing")
async def test_send_signal_cancelled_future(self) -> None:
"""Process task should be able to terminate by method send_signal()."""
with ProcessPoolExecutor() as executor:
future = self.create_future(executor)
assert future.cancel()
await self.execute_test(future, SIGTERM, True, CancelledError)
assert future.done()
def parallel_processing(func, args, workers):
'''
Basic function for executing parallel process using same function for all workers.
'''
with ProcessPoolExecutor(workers) as ex:
res = ex.map(func, args)
return list(res)
async def main(*, host: str = '127.0.0.1', port: int = 50051) -> None:
with ProcessPoolExecutor(max_workers=4) as executor:
server = Server([Primes(executor)])
with graceful_exit([server]):
await server.start(host, port)
print(f'Serving on {host}:{port}')
await server.wait_closed()
async def process_response(response_data):
loop = asyncio.get_running_loop()
with ProcessPoolExecutor() as pool:
result = await loop.run_in_executor(
pool, functools.partial(parse_xml_data, response_data))
return result
def __init__(self,
m3u8_url,
video_path='/tmp/m3u8.ts',
process_workers=None,
thread_workers=None):
use_process = thread_workers is None
self.pool = ProcessPoolExecutor(
max_workers=process_workers
) if use_process else ThreadPoolExecutor(max_workers=thread_workers)
self.m3u8_url = m3u8_url
video_name = tmp = os.path.basename(video_path)
for i in windows_invalid:
if i in video_name:
tmp = tmp.replace(i, '')
video_name = tmp
self.video_folder = os.path.dirname(video_path)
self.video_path = os.path.join(self.video_folder, video_name)
self.ts_folder = os.path.join(self.video_folder,
video_name.split('.')[0])
if not os.path.exists(self.video_folder):
os.mkdir(self.video_folder)
if not os.path.exists(self.ts_folder):
os.mkdir(self.ts_folder)
def scan_object(self: 'Data') -> np.ndarray:
"""
:return: A point representing an object captured from all the cameras that are on
"""
depth_cams = [
cam for cam in self.cameras
if cam.on and cam.type is CameraType.Depth
]
lidar_cams = [
cam for cam in self.cameras
if cam.on and cam.type is CameraType.LiDAR
]
# Using processes means effectively side-stepping the Global Interpreter Lock
with ProcessPoolExecutor(
max_workers=min(self.max_workers,
len(depth_cams) + len(lidar_cams))) as pool:
futures = [
pool.submit(dcam.generate_adapted_point_cloud,
self.number_of_frames, self.number_of_dummy_frames)
for dcam in depth_cams
]
# LiDAR cameras cannot capture simultaneously, for now run them serially
# TODO: defeat python's shitty concurrency model and make it more concurrent
pcs = [
lcam.generate_adapted_point_cloud(self.number_of_frames,
self.number_of_dummy_frames)
for lcam in lidar_cams
]
for fut in futures:
pcs.append(fut.result())
return np.concatenate(pcs)
def executor(kind: str, max_workers: int, daemon=True):
"""General purpose utility to get an executor with its as_completed handler
This allows us to easily use other executors as needed.
"""
if kind == "thread":
with ThreadPoolExecutor(max_workers=max_workers) as pool_t:
yield pool_t
elif kind == "process":
with ProcessPoolExecutor(max_workers=max_workers) as pool_p:
yield pool_p
elif kind in ["dask", "dask-process", "dask-thread"]:
import dask
import distributed
from distributed.cfexecutor import ClientExecutor
processes = kind == "dask" or kind == "dask-process"
with dask.config.set({"distributed.worker.daemon": daemon}):
with distributed.LocalCluster(
n_workers=max_workers,
processes=processes,
# silence_logs='error'
) as cluster:
with distributed.Client(cluster) as client:
yield ClientExecutor(client)
else:
raise NotImplementedError("That kind is not implemented")
async def start_subprocess_and_listen(async_worker,
backtest_session,
strategy,
db_q=None):
from concurrent.futures.process import ProcessPoolExecutor
from utils.processing.async_queue import AsyncProcessQueue
q = AsyncProcessQueue()
loop = get_event_loop_with_exceptions()
print('running in executor')
loop.run_in_executor(
ProcessPoolExecutor(max_workers=1),
_worker_sync,
async_worker,
q,
backtest_session,
strategy,
db_q,
)
while True:
result = await q.coro_get()
if result is None:
return
yield result
def search(states,
planner,
nn_model,
ncpus,
time_limit_seconds,
search_budget=-1):
"""
This function runs (best-first) Levin tree search with a learned policy on a set of problems
"""
total_expanded = 0
total_generated = 0
total_cost = 0
slack_time = 600
solutions = {}
for name, state in states.items():
state.reset()
solutions[name] = (-1, -1, -1, -1)
start_time = time.time()
while len(states) > 0:
# args = [(state, name, nn_model, search_budget, start_time, time_limit_seconds, slack_time) for name, state in states.items()]
# solution_depth, expanded, generated, running_time, puzzle_name = planner.search(args[0])
with ProcessPoolExecutor(max_workers=ncpus) as executor:
args = ((state, name, nn_model, search_budget, start_time,
time_limit_seconds, slack_time)
for name, state in states.items())
results = executor.map(planner.search, args)
for result in results:
solution_depth = result[0]
expanded = result[1]
generated = result[2]
running_time = result[3]
puzzle_name = result[4]
if solution_depth > 0:
solutions[puzzle_name] = (solution_depth, expanded, generated,
running_time)
del states[puzzle_name]
if solution_depth > 0:
total_expanded += expanded
total_generated += generated
total_cost += solution_depth
partial_time = time.time()
if partial_time - start_time + slack_time > time_limit_seconds or len(
states) == 0 or search_budget >= 1000000:
for name, data in solutions.items():
print("{:s}, {:d}, {:d}, {:d}, {:.2f}".format(
name, data[0], data[1], data[2], data[3]))
return
search_budget *= 2
async def startup():
# dpc.init()
dpc.ascloop = asyncio.get_event_loop(
) # overwrting the loop var initialized when dpc is imported, it is a different loop object in uvicorn
app.state.executor = ProcessPoolExecutor(max_workers=1)
app.state.emulator = SysPoolExecutor(max_workers=1)
cfg.logger.info('Tsaisendo service environtment initiated')
def test_resolve_distributed_mode_slurm3():
args = argparse.Namespace(
multiprocessing_distributed=True,
dist_world_size=None,
dist_rank=None,
ngpu=1,
local_rank=None,
dist_launcher="slurm",
dist_backend="nccl",
dist_init_method="env://",
dist_master_addr=None,
dist_master_port=10000,
)
env = dict(
SLURM_PROCID="0",
SLURM_NTASKS="1",
SLURM_STEP_NUM_NODES="1",
SLURM_STEP_NODELIST="localhost",
SLURM_NODEID="0",
CUDA_VISIBLE_DEVICES="0,1",
)
e = ProcessPoolExecutor(max_workers=2)
with unittest.mock.patch.dict("os.environ", dict(env, SLURM_LOCALID="0")):
resolve_distributed_mode(args)
option = build_dataclass(DistributedOption, args)
fn = e.submit(option.init)
with unittest.mock.patch.dict("os.environ", dict(env, SLURM_LOCALID="0")):
option2 = build_dataclass(DistributedOption, args)
fn2 = e.submit(option2.init)
fn.result()
fn2.result()
def __init__(self, coin_store: CoinStore,
consensus_constants: ConsensusConstants):
self.constants: ConsensusConstants = consensus_constants
self.constants_json = recurse_jsonify(
dataclasses.asdict(self.constants))
# Transactions that were unable to enter mempool, used for retry. (they were invalid)
self.potential_txs: Dict[bytes32, Tuple[SpendBundle, CostResult,
bytes32]] = {}
# Keep track of seen spend_bundles
self.seen_bundle_hashes: Dict[bytes32, bytes32] = {}
self.coin_store = coin_store
tx_per_sec = self.constants.TX_PER_SEC
sec_per_block = self.constants.SUB_SLOT_TIME_TARGET // self.constants.SLOT_BLOCKS_TARGET
block_buffer_count = self.constants.MEMPOOL_BLOCK_BUFFER
# MEMPOOL_SIZE = 60000
self.mempool_size = int(tx_per_sec * sec_per_block *
block_buffer_count)
self.potential_cache_size = 300
self.seen_cache_size = 10000
self.pool = ProcessPoolExecutor(max_workers=1)
# The mempool will correspond to a certain peak
self.peak: Optional[BlockRecord] = None
self.mempool: Mempool = Mempool.create(self.mempool_size)
def process_run(run_info, args):
pssm = Pssm()
for filename in os.listdir(run_info.scratch_path):
filepath = os.path.join(run_info.scratch_path, filename)
if os.path.isdir(filepath):
shutil.rmtree(filepath)
else:
os.remove(filepath)
if run_info.interop_path is None:
run_summary = None
else:
logger.info('Summarizing run.')
run_summary = summarize_run(run_info)
with ProcessPoolExecutor(max_workers=args.max_active) as pool:
for _ in pool.map(functools.partial(process_sample,
args=args,
pssm=pssm,
use_denovo=run_info.is_denovo),
run_info.get_all_samples()):
pass
for _ in pool.map(functools.partial(process_resistance,
run_info=run_info),
run_info.sample_groups):
pass
collate_samples(run_info)
if run_summary is not None:
summarize_samples(run_info, run_summary)
if not args.keep_scratch:
shutil.rmtree(run_info.scratch_path, ignore_errors=True)
logger.info('Done.')
def get_executor(self, reset=False):
nonlocal did_get_executor
if did_get_executor:
return ProcessPoolExecutor(1)
else:
did_get_executor = True
return BrokenProcessPoolExecutor()
def __init__(self, display_data: Dict, simulation_time = LIFE_STEPS, gui_enabled = True, stop_callback: Callable = None):
print(Simulator.test)
Simulator.test += 1
self.stop_callback = stop_callback
self.display_data = display_data
self.gui_enabled = gui_enabled
if gui_enabled:
# initialize pygame
pygame.init()
self.clock: pygame.time.Clock = pygame.time.Clock() # PyGame Clock to set frame rate
self.screen: pygame.screen = pygame.display.set_mode((WIDTH, HEIGHT)) # Window where simulation is played
pygame.display.set_caption("ARS_Robot_Simulation") # Window title
icon = pygame.image.load('images/robot.png') # Window icon
pygame.display.set_icon(icon)
self.keys: List[Dict[pygame.key, Callable, bool]] = [ # Action keys with relative callback
dict(key_code=[pygame.K_r], callback=self.reinit_robots, hold=False, pressed=False),
]
self.FONT = pygame.font.SysFont(None, 28) # Font used for data visualization on top
else:
self.pool = ProcessPoolExecutor(os.cpu_count())
self.environment: Environment = Environment() # Environment where the robot is placed
self.done: bool = False # Window closed ?
self.robots: List[Robot] = []
self.simulation_time = simulation_time
self.time_left = simulation_time
async def test_rebuilds_process_pool_cooperatively(self):
"""
Make sure that two parallel diffing failures only cause the process
pool to be rebuilt once, not multiple times.
"""
# Get a custom executor that will always fail the first time, but get
# a real one that will succeed afterward.
executor_resets = 0
good_executor = ProcessPoolExecutor(1)
bad_executor = BrokenProcessPoolExecutor()
def get_executor(self, reset=False):
nonlocal executor_resets
if reset:
executor_resets += 1
if executor_resets > 0:
return good_executor
else:
return bad_executor
with patch.object(df.DiffHandler, 'get_diff_executor', get_executor):
one = self.fetch_async('/html_source_dmp?format=json&'
f'a=file://{fixture_path("empty.txt")}&'
f'b=file://{fixture_path("empty.txt")}')
two = self.fetch_async('/html_source_dmp?format=json&'
f'a=file://{fixture_path("empty.txt")}&'
f'b=file://{fixture_path("empty.txt")}')
response1, response2 = await asyncio.gather(one, two)
assert response1.code == 200
assert response2.code == 200
assert executor_resets == 1
# Ensure *both* diffs hit the bad executor, so we know we didn't
# have one reset because only one request hit the bad executor.
assert bad_executor.submit_count == 2
def __init__(self, coin_store: CoinStore,
consensus_constants: ConsensusConstants):
self.constants: ConsensusConstants = consensus_constants
self.constants_json = recurse_jsonify(
dataclasses.asdict(self.constants))
# Keep track of seen spend_bundles
self.seen_bundle_hashes: Dict[bytes32, bytes32] = {}
self.coin_store = coin_store
self.lock = asyncio.Lock()
# The fee per cost must be above this amount to consider the fee "nonzero", and thus able to kick out other
# transactions. This prevents spam. This is equivalent to 0.055 XCH per block, or about 0.00005 XCH for two
# spends.
self.nonzero_fee_minimum_fpc = 5
self.limit_factor = 0.5
self.mempool_max_total_cost = int(self.constants.MAX_BLOCK_COST_CLVM *
self.constants.MEMPOOL_BLOCK_BUFFER)
# Transactions that were unable to enter mempool, used for retry. (they were invalid)
self.potential_cache = PendingTxCache(
self.constants.MAX_BLOCK_COST_CLVM * 1)
self.seen_cache_size = 10000
self.pool = ProcessPoolExecutor(max_workers=2)
# The mempool will correspond to a certain peak
self.peak: Optional[BlockRecord] = None
self.mempool: Mempool = Mempool(self.mempool_max_total_cost)
def collect_frames_mp(_mvcap):
"""
Multiprocess version of #collect_frames. It cuts ~10 seconds on i7-8750h delegating processing to a pool of
processors. Less powerful CPUs could make this run slower than the standard version.
:param _mvcap:
:return:
"""
avgdict: Dict[int, np.ndarray] = OrderedDict({})
framecount = 0
read, frame = _mvcap.read()
def _callback(fut: Future):
index, value = fut.result()
avgdict[index] = value
with ProcessPoolExecutor() as executor:
while read:
executor.submit(_indexedtask, framecount, frame).add_done_callback(_callback)
framecount += 1
for _ in range(30): # The argument of range() is the number of skipped frames for every iteration.
if read:
read = _mvcap.grab()
if read:
read, frame = _mvcap.read()
executor.shutdown(True)
res = []
for i in range(framecount):
res.append(avgdict[i])
return res
async def create(
coin_store: CoinStore,
block_store: BlockStore,
consensus_constants: ConsensusConstants,
):
"""
Initializes a blockchain with the SubBlockRecords from disk, assuming they have all been
validated. Uses the genesis block given in override_constants, or as a fallback,
in the consensus constants config.
"""
self = Blockchain()
self.lock = asyncio.Lock() # External lock handled by full node
cpu_count = multiprocessing.cpu_count()
if cpu_count > 61:
cpu_count = 61 # Windows Server 2016 has an issue https://bugs.python.org/issue26903
num_workers = max(cpu_count - 2, 1)
log.info(f"Starting {num_workers} processes for block validation")
self.pool = ProcessPoolExecutor(max_workers=num_workers)
self.constants = consensus_constants
self.coin_store = coin_store
self.block_store = block_store
self.constants_json = recurse_jsonify(
dataclasses.asdict(self.constants))
self._shut_down = False
await self._load_chain_from_store()
return self
def __init__(self, display_data: Dict, simulation_time = Const.LIFE_STEPS, gui_enabled = True, stop_callback: Callable = None, room: int = None):
self.stop_callback = stop_callback
self.display_data = display_data
self.gui_enabled = gui_enabled
if gui_enabled:
# initialize pygame
pygame.init()
self.clock: pygame.time.Clock = pygame.time.Clock() # PyGame Clock to set frame rate
self.screen: pygame.screen = pygame.display.set_mode((Const.WIDTH, Const.HEIGHT)) # Window where simulation is played
pygame.display.set_caption("ARS_Robot_Simulation") # Window title
icon = pygame.image.load('images/robot.png') # Window icon
pygame.display.set_icon(icon)
self.keys: List[Dict[pygame.key, Callable, bool]] = [ # Action keys with relative callback
dict(key_code=[pygame.K_r], callback=self.stop_current_run, hold=False, pressed=False),
dict(key_code=[pygame.K_SPACE], callback=self.start_current_run, hold=False, pressed=False),
]
self.FONT = pygame.font.SysFont(None, 28) # Font used for data visualization on top
else:
self.pool = ProcessPoolExecutor(int(os.cpu_count() * (2/3)))
self.environment: Environment = Environment(room = room) # Environment where the robot is placed
self.done: bool = False # Window closed ?
self.robots: List[Robot] = []
for i in range(Const.N_INDIVIDUALS):
self.robots.append(Robot(init_pos = None, init_rotation = np.random.randint(low = 0, high = 360), genome = None))
self.simulation_time = simulation_time
self.time_left = simulation_time
self.hold = True
async def create(
block_store: WalletBlockStore,
consensus_constants: ConsensusConstants,
coins_of_interest_received:
Callable, # f(removals: List[Coin], additions: List[Coin], height: uint32)
reorg_rollback: Callable,
):
"""
Initializes a blockchain with the SubBlockRecords from disk, assuming they have all been
validated. Uses the genesis block given in override_constants, or as a fallback,
in the consensus constants config.
"""
self = WalletBlockchain()
self.lock = asyncio.Lock() # External lock handled by full node
cpu_count = multiprocessing.cpu_count()
if cpu_count > 61:
cpu_count = 61 # Windows Server 2016 has an issue https://bugs.python.org/issue26903
num_workers = max(cpu_count - 2, 1)
self.pool = ProcessPoolExecutor(max_workers=num_workers)
log.info(f"Started {num_workers} processes for block validation")
self.constants = consensus_constants
self.constants_json = recurse_jsonify(
dataclasses.asdict(self.constants))
self.block_store = block_store
self._shut_down = False
self.coins_of_interest_received = coins_of_interest_received
self.reorg_rollback = reorg_rollback
self.log = logging.getLogger(__name__)
await self._load_chain_from_store()
return self
def getPageRankOfDataset(ds):
if type(ds) == str:
ds = Helper.getDataFromDir(ds, mode='list')
with ProcessPoolExecutor(max_workers=cpu_count(
logical=Helper.logical())) as executor:
fts = {}
kfs = {}
dsStr = listOfTaggedToString(ds)
collection = list(map(lambda doc: doc.lower(), dsStr))
cands = getAllCandidates(ds, deliver_as='sentences')
info = getInfo(ds)
for i, file in enumerate(ds):
fts.update({
executor.submit(getKeyphrases,
dsStr[i].lower(),
info,
candidates=cands[i],
doc_i=i,
collection=collection):
file
})
for future in as_completed(fts):
file = fts[future]
kfs.update({file: future.result()})
return kfs
