async def qtloop():
qt_app = None
qtimport = False
try:
import PyQt5.QtCore, PyQt5.QtWidgets
qtimport = True
except ImportError:
pass
if qtimport:
if multiprocessing.get_start_method() != "fork":
print("""Cannot test if Qt can be started
This is because forking is not possible, you are probably running under Windows
Starting Qt blindly is not supported, as it may result in segfaults
""",
file=sys.stderr)
else:
p = Process(target=test_qt)
p.start()
p.join()
if not p.exitcode:
qt_app = PyQt5.QtWidgets.QApplication([" "])
if qt_app is None:
msg = "Qt could not be started. Qt widgets will not work" #TODO: some kind of env variable to disable this warning
print(msg,file=sys.stderr)
return
while 1:
qt_app.processEvents()
await asyncio.sleep(0.01)
def run(self):
try:
self._cmdline.this_module_name = self.__class__.__module__
if multiprocessing.get_start_method() == 'spawn':
common.set_global_options(self._cmdline)
common.sysinit()
signal.signal(signal.SIGTERM, self._sighandler)
self.id = self._channel(self._dp_name, self.__class__)
common.set_current_process(self.id)
pattern.initialize(self.id)
if hasattr(self._cmdline, 'clock') and \
self._cmdline.clock == 'Lamport':
self._logical_clock = 0
self._log = logging.getLogger(str(self))
self._start_comm_thread()
self._lock = threading.Lock()
self._lock.acquire()
self._wait_for_go()
if not hasattr(self, '_da_run_internal'):
self._log.error("Process does not have entry point!")
sys.exit(1)
result = self._da_run_internal()
self.report_times()
except Exception as e:
sys.stderr.write("Unexpected error at process %s:%r"% (str(self), e))
traceback.print_tb(e.__traceback__)
except KeyboardInterrupt as e:
self._log.debug("Received KeyboardInterrupt, exiting")
pass
def transfer_dir_parallel(self, source, destination, jobs):
"""Transfer a directory to the remote side in parallel mode."""
self.log.debug("Remote transfer: %s -> %s", source, destination)
self.rmdir(destination)
if os.path.isdir(source):
self.mkdir(destination)
self.log.info("Remote transfer in {} parallel jobs".format(
jobs))
self.log.debug("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for parallel "
"remote transfer"))
with multiprocessing.Pool(jobs) as pool:
self.log.debug("Starting async for parallel transfer")
commands = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
self.target_host[0], destination)])
commands.append(command)
results = [
pool.apply_async(self._run_command, (cmd,))
for cmd in commands
]
self.log.debug("Waiting async results for parallel transfer")
for r in results:
r.get() # self._run_command returns None
self.log.debug(("Multiprocessing for parallel transfer "
"finished"))
else:
raise cdist.Error("Source {} is not a directory".format(source))
def setUpClass(cls):
import multiprocessing as mp
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
assert mp.get_start_method() == "spawn"
write_settings()
cls.app = QApplication([cls.__name__])
def default_test_processes():
"""Default number of test processes when using the --parallel option."""
# The current implementation of the parallel test runner requires
# multiprocessing to start subprocesses with fork().
if multiprocessing.get_start_method() != 'fork':
return 1
try:
return int(os.environ['DJANGO_TEST_PROCESSES'])
except KeyError:
return multiprocessing.cpu_count()
def _extract_knob_feature_index(args):
"""extract knob feature for an index in extract_space"""
try:
if multiprocessing.get_start_method(False) == "fork":
config = _extract_space.get(args)
else:
config = args[0]
return config.get_flatten_feature()
except Exception: # pylint: disable=broad-except
return None
def _multiprocessing_is_fork(): # type () -> bool
"""Class state is only preserved when using the `fork` strategy."""
if sys.version_info >= (3, 4):
return (
multiprocessing
# https://github.com/python/typeshed/pull/3415
and multiprocessing.get_start_method() == "fork" # type: ignore
)
else:
return multiprocessing and not utils.is_windows()
def setUpClass(cls):
import multiprocessing as mp
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
assert mp.get_start_method() == "spawn"
write_settings()
cls.app = QApplication([cls.__name__])
def setUp(self):
self.working_directory = tempfile.TemporaryDirectory()
os.chdir(self.working_directory.name)
# Keep track of get_start_method
# This function gets monkey patched to ensure run_check is aware of the multiprocessing context,
# without needing to explicitely pass the context to run_check.
# The same behavior can't be achieved by multiprocessing.set_start_method as that can only run once per program
# https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
self._get_start_method = multiprocessing.get_start_method()
def main(address, port):
# Dask requirement - add support for when a program which uses multiprocessing has been frozen to produce a Windows executable.
mp.freeze_support()
if ('forkserver' in mp.get_all_start_methods()) & (mp.get_start_method(allow_none=True) != 'forkserver'):
# The CUDA runtime does not support the fork start method;
# either the spawn or forkserver start method are required to use CUDA in subprocesses.
# forkserver is used because spawn is still affected by locking issues
mp.set_start_method('forkserver', force=True)
service = DaskRemoteService(address, port)
def default_test_processes():
"""Default number of test processes when using the --parallel option."""
# The current implementation of the parallel test runner requires
# multiprocessing to start subprocesses with fork().
if multiprocessing.get_start_method() != 'fork':
return 1
try:
return int(os.environ['SERVER_TEST_PROCESSES'])
except KeyError:
return multiprocessing.cpu_count()
def initialize_suite(self):
if multiprocessing.get_start_method() == "spawn":
self.initial_settings = {
alias: connections[alias].settings_dict
for alias in connections
}
self.serialized_contents = {
alias: connections[alias]._test_serialized_contents
for alias in connections if alias in self.serialized_aliases
}
def test_multiprosessing_stack_depth(self):
def check_func(data):
for entry in data["traceEvents"]:
self.assertNotIn("fib", entry["name"].split())
if multiprocessing.get_start_method() == "fork":
self.template(["viztracer", "--log_multiprocess", "-o", "result.json", "cmdline_test.py"],
expected_output_file="result.json",
script=file_multiprocessing_stack_limit,
check_func=check_func,
concurrency="multiprocessing")
def start():
parser = create_parser()
args = parser.parse_args()
config_type = args.type
config = Config(config_type=config_type)
setup(config, args)
logger.info('Config type: %s' % (config_type))
config.opts.piece_style = args.piece_style
config.opts.bg_style = args.bg_style
config.internet.distributed = args.distributed
if args.cmd == 'self':
if args.ucci:
import cchess_alphazero.worker.play_with_ucci_engine as self_play
else:
if mp.get_start_method() == 'spawn':
import cchess_alphazero.worker.self_play_windows as self_play
else:
from cchess_alphazero.worker import self_play
return self_play.start(config)
elif args.cmd == 'opt':
from cchess_alphazero.worker import optimize
return optimize.start(config)
elif args.cmd == 'play':
if args.cli:
import cchess_alphazero.play_games.play_cli as play
else:
from cchess_alphazero.play_games import play
config.opts.light = False
pwhc = PlayWithHumanConfig()
pwhc.update_play_config(config.play)
logger.info(f"AI move first : {args.ai_move_first}")
play.start(config, not args.ai_move_first)
elif args.cmd == 'eval':
if args.elo == False:
from cchess_alphazero.worker import evaluator
else:
import cchess_alphazero.worker.compute_elo as evaluator
config.eval.update_play_config(config.play)
evaluator.start(config)
elif args.cmd == 'sl':
if args.onegreen:
import cchess_alphazero.worker.sl_onegreen as sl
sl.start(config, args.skip)
else:
from cchess_alphazero.worker import sl
sl.start(config)
elif args.cmd == 'ob':
from cchess_alphazero.play_games import ob_self_play
pwhc = PlayWithHumanConfig()
pwhc.update_play_config(config.play)
ob_self_play.start(config, args.ucci, args.ai_move_first)
def start():
parser = create_parser()
args = parser.parse_args()
config_type = args.type
config = Config(config_type=config_type)
setup(config, args)
logger.info('Config type: %s' % (config_type))
config.opts.piece_style = args.piece_style
config.opts.bg_style = args.bg_style
config.internet.distributed = args.distributed
if args.cmd == 'self':
if args.ucci:
import cchess_alphazero.worker.play_with_ucci_engine as self_play
else:
if mp.get_start_method() == 'spawn':
import cchess_alphazero.worker.self_play_windows as self_play
else:
from cchess_alphazero.worker import self_play
return self_play.start(config)
elif args.cmd == 'opt':
from cchess_alphazero.worker import optimize
return optimize.start(config)
elif args.cmd == 'play':
if args.cli:
import cchess_alphazero.play_games.play_cli as play
else:
from cchess_alphazero.play_games import play
config.opts.light = False
pwhc = PlayWithHumanConfig()
pwhc.update_play_config(config.play)
logger.info(f"AI move first : {args.ai_move_first}")
play.start(config, not args.ai_move_first)
elif args.cmd == 'eval':
if args.elo == False:
from cchess_alphazero.worker import evaluator
else:
import cchess_alphazero.worker.compute_elo as evaluator
config.eval.update_play_config(config.play)
evaluator.start(config)
elif args.cmd == 'sl':
if args.onegreen:
import cchess_alphazero.worker.sl_onegreen as sl
sl.start(config, args.skip)
else:
from cchess_alphazero.worker import sl
sl.start(config)
elif args.cmd == 'ob':
from cchess_alphazero.play_games import ob_self_play
pwhc = PlayWithHumanConfig()
pwhc.update_play_config(config.play)
ob_self_play.start(config, args.ucci, args.ai_move_first)
class TestIssue39Regr(tb.UVTestCase):
"""See https://github.com/MagicStack/uvloop/issues/39 for details.
Original code to reproduce the bug is by Jim Fulton.
"""
def on_alarm(self, sig, fr):
if self.running:
raise FailedTestError
def run_test(self):
try:
for i in range(10):
for threaded in [True, False]:
if threaded:
qin, qout = queue.Queue(), queue.Queue()
threading.Thread(target=run_server,
args=(qin, qout),
daemon=True).start()
else:
qin = multiprocessing.Queue()
qout = multiprocessing.Queue()
multiprocessing.Process(target=run_server,
args=(qin, qout),
daemon=True).start()
addr = qout.get()
loop = self.new_loop()
asyncio.set_event_loop(loop)
loop.create_task(
loop.create_connection(
lambda: EchoClientProtocol(loop),
host=addr[0],
port=addr[1]))
loop.run_forever()
loop.close()
qin.put('stop')
qout.get()
finally:
loop.close()
@unittest.skipIf(
multiprocessing.get_start_method(False) == 'spawn',
'no need to test on macOS where spawn is used instead of fork')
def test_issue39_regression(self):
signal.signal(signal.SIGALRM, self.on_alarm)
signal.alarm(5)
try:
self.running = True
self.run_test()
except FailedTestError:
self.fail('deadlocked in libuv')
finally:
self.running = False
signal.signal(signal.SIGALRM, signal.SIG_IGN)
def do_mp(func,
args_arr,
is_mp=True,
cpus=None,
do_sigkill=True,
start_method=None,
verbose=False,
used_cpus=0):
# start_method: 'spawn', 'fork', 'forkserver'
if is_mp is True:
if verbose:
my_print('pid: ' + str(os.getpid()) +
' &&&&&&&&&&&&&&&&&&&&&&&& mp starts:: children: ' +
str(len(psutil.Process().children(recursive=True))) +
' start_method: ' +
str(mp.get_start_method(allow_none=True)))
default_method = 'spawn'
if mp.get_start_method(allow_none=True) is None:
start_method = default_method if start_method is None else start_method
mp.set_start_method(start_method,
force=True) # spawn, fork, forkserver
if cpus is None:
cpus = min([int(mp.cpu_count()), len(args_arr)])
else:
cpus = min([int(cpus), int(mp.cpu_count()), len(args_arr)])
cpus = max(1, cpus - used_cpus)
with mp.Pool(processes=int(cpus)) as pool:
if verbose:
my_print('pid: ' + str(os.getpid()) +
' pre active children: ' + str(mp.active_children()))
out_list = pool.starmap(func, args_arr)
if verbose:
my_print('pid: ' + str(os.getpid()) + ' post active children: ' +
str(mp.active_children()))
if start_method == 'fork':
reap_children(do_sigkill=do_sigkill)
if verbose:
my_print('pid: ' + str(os.getpid()) +
' fork active children: ' + str(mp.active_children()))
else:
out_list = [func(*a) for a in args_arr]
return [x for x in out_list if x is not None]
def __init__(self, max_workers, kind='thread', show_progress=False):
"""
Args:
max_workers: The number of parallel jobs to run
kind: "thread" or "process" determine how to parallelize
show_progress: Display tqdm notebook-based progress bar
Example:
# Define the job you want to run
def job(*args, **kwargs):
import time
time.sleep(args[0])
return (args, kwargs)
# Initialize the parallel processor
p = ezr.Parallel(10)
# Specify a list of args and kwargs you want to run
call_tuples_list = [
([1], {'a': 1}), # (args, kwargs)
([2], {'a': 2}),
([3], {'a': 3}),
]
# Get results from a generator as jobs are completed
for result in p.as_complete(job, call_tuples_list):
print(result)
"""
from concurrent import futures as futures_module
self.futures_module = futures_module
self.show_progress = show_progress
if kind == 'thread':
self.executor = self.futures_module.ThreadPoolExecutor(
max_workers=max_workers)
elif kind == 'process':
import multiprocessing
import warnings
try:
multiprocessing.set_start_method("fork")
except RuntimeError:
if multiprocessing.get_start_method() != 'fork':
warnings.warn(
'multiprocessing.set_start_method("fork") raised an error. You may need to restart kernel'
)
self.executor = self.futures_module.ProcessPoolExecutor(
max_workers=max_workers)
else:
allowed_kinds = ['thread', 'process']
raise ValueError(f'kind must be one of {allowed_kinds}')
def setUpClass(cls):
import multiprocessing as mp
try:
mp.set_start_method("spawn")
except RuntimeError:
pass
assert mp.get_start_method() == "spawn"
write_settings()
cls.app = QApplication([cls.__name__])
logger.debug("Start new app with name {}".format(cls.app.applicationName()))
def test_multiprocessing(self):
if get_start_method(True) != "spawn":
set_start_method("spawn")
workers = 10
game_id = "sfiii3n"
output_queue = Queue()
processes = [Process(target=run_console, args=[game_id, output_queue]) for i in range(workers)]
[process.start() for process in processes]
[process.join() for process in processes]
for i in range(workers):
assert_that(output_queue.get(timeout=0.1), equal_to("384"))
def test_bound_function(self):
if multiprocessing.get_start_method() == 'fork':
self.bound_function()
return
LOG = StringIO()
with LoggingIntercept(LOG):
with self.assertRaises((TypeError, EOFError, AttributeError)):
self.bound_function()
self.assertIn("platform that does not support 'fork'", LOG.getvalue())
self.assertIn("one of its arguments is not serializable",
LOG.getvalue())
def test_closing(self):
if multiprocessing.get_start_method() != "fork":
return # dummy iterator used, skip test
it = PrefetchIterator(NativeCheckpointableIterator([0]),
buffer_size=42,
buffer_in_main_process=True)
it.close()
f = lambda: it.__next__()
self.assertRaises(RuntimeError, f)
f = lambda: it.setstate(None)
self.assertRaises(RuntimeError, f)
def reset_process_start_method() -> None:
preferred = "spawn"
current = mp.get_start_method(True)
if current != preferred:
if preferred in mp.get_all_start_methods():
log.debug(f"Set process start method to {preferred}")
mp.set_start_method(preferred, True)
return
log.warning(
f"{preferred} method not available. Have {mp.get_all_start_methods()}. Use {current}"
)
def start(self):
"""Start the Pod.
This method calls :meth:`start` in :class:`threading.Thread` or :class:`multiprocesssing.Process`.
.. #noqa: DAR201
"""
self.worker.start()
self.is_forked = multiprocessing.get_start_method().lower() == 'fork'
if not self.args.noblock_on_start:
self.wait_start_success()
return self
def dist_init(launcher, backend='nccl', **kwargs):
if mp.get_start_method(allow_none=True) is None:
mp.set_start_method('spawn')
if launcher == 'pytorch':
_init_dist_pytorch(backend, **kwargs)
elif launcher == 'mpi':
_init_dist_mpi(backend, **kwargs)
elif launcher == 'slurm':
_init_dist_slurm(backend, **kwargs)
else:
raise ValueError('Invalid launcher type: {}'.format(launcher))
def _run_global_explorers_parallel(self, out_path):
self.log.debug("Running global explorers in %s parallel jobs",
self.jobs)
self.log.trace("Multiprocessing start method is %s",
multiprocessing.get_start_method())
self.log.trace("Starting multiprocessing Pool for global explorers"
" run")
args = [
(e, out_path, ) for e in self.list_global_explorer_names()
]
mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
self.log.trace("Multiprocessing run for global explorers finished")
def get_max_test_processes():
"""
The maximum number of test processes when using the --parallel option.
"""
# The current implementation of the parallel test runner requires
# multiprocessing to start subprocesses with fork() or spawn().
if multiprocessing.get_start_method() not in {"fork", "spawn"}:
return 1
try:
return int(os.environ["DJANGO_TEST_PROCESSES"])
except KeyError:
return multiprocessing.cpu_count()
def to_tf_dataset(self, preload_buf_size: int) -> tf.data.Dataset:
assert mp.get_start_method() == 'spawn'
return (tf.data.Dataset.from_generator(
lambda: PreloadingIterator(
self.source_iterable(),
self._get_process_pool_executor(),
preload_buf_size,
self.process_source_item,
),
self.tf_dataset_dtype,
self.tf_dataset_shape,
))
def _enforce_fork_start_method():
# We need to make sure we use "fork" because "spawn" is the default in Python 3.8.
try:
multiprocessing.set_start_method("fork")
except RuntimeError as e:
if e != "context has already been set":
raise e
start_method = multiprocessing.get_start_method()
if start_method != "fork":
raise RuntimeError(
f"Could not set multiprocessing start method to 'fork', "
f"it's already set to '{start_method}'.") from e
def train(self, trainset, sampler, cut_layer=None) -> float:
"""The main training loop in a federated learning workload.
Arguments:
trainset: The training dataset.
sampler: the sampler that extracts a partition for this client.
cut_layer (optional): The layer which training should start from.
Returns:
float: Elapsed time during training.
"""
config = Config().trainer._asdict()
config['run_id'] = Config().params['run_id']
# Set the start time of training in absolute time
self.training_start_time = time.time()
if 'max_concurrency' in config:
self.start_training()
tic = time.perf_counter()
if mp.get_start_method(allow_none=True) != 'spawn':
mp.set_start_method('spawn', force=True)
train_proc = mp.Process(target=self.train_process,
args=(config, trainset, sampler,
cut_layer))
train_proc.start()
train_proc.join()
model_name = Config().trainer.model_name
filename = f"{model_name}_{self.client_id}_{Config().params['run_id']}.pth"
try:
self.load_model(filename)
except OSError as error: # the model file is not found, training failed
if 'max_concurrency' in config:
self.run_sql_statement(
"DELETE FROM trainers WHERE run_id = (?)",
(self.client_id, ))
raise ValueError(
f"Training on client {self.client_id} failed.") from error
toc = time.perf_counter()
self.pause_training()
else:
tic = time.perf_counter()
self.train_process(config, trainset, sampler, cut_layer)
toc = time.perf_counter()
training_time = toc - tic
return training_time
def _get_multiprocessing_start_method(self) -> str:
"""
Determine method of creating new processes by checking if the
mp_start_method is set in configs, else, it uses the OS default.
"""
if conf.has_option('core', 'mp_start_method'):
return conf.get('core', 'mp_start_method')
method = multiprocessing.get_start_method()
if not method:
raise ValueError("Failed to determine start method")
return method
def run(self):
if get_start_method() != 'fork':
# We haven't received the parsed arguments from the main process
cmdline.parse_arguments()
# Create the UI inside the correct process
self._ui_launcher = create_ui_launcher()
self._ui_launcher.set_audio_engine(self._audio_engine)
self._controller = self._ui_launcher.get_controller()
self._ui_launcher.set_event_queue_processor(self._process_queue)
self._ui_launcher.run_ui()
def start(self):
super().start()
if mp.get_start_method() == 'spawn':
self._popen = FakePopen()
for name in dir(self):
if name not in self._exceptions and name[0] != '_':
isCallable = callable(getattr(self, name))
setattr(
self, name,
SharedMethod(name, self.input_queue, self.output_queue))
if not isCallable:
setattr(self.__class__, name, SharedProperty(name))
def _wrap_simulate(args):
print("IN WRAP SIMULATE")
(
mol,
U_proposal,
U_target,
temperature,
masses,
steps_per_batch,
batches_per_worker,
burn_in_batches,
num_workers,
seed,
) = args
# wraps a callable fn so it runs in a subprocess with the device_count set explicitly
assert multiprocessing.get_start_method() == "spawn"
os.environ["XLA_FLAGS"] = "--xla_force_host_platform_device_count=" + str(num_workers)
kT = temperature * BOLTZ
x0 = get_romol_conf(mol)
xs_proposal = simulate(
x0,
U_proposal,
temperature,
masses,
steps_per_batch,
batches_per_worker + burn_in_batches,
num_workers,
seed,
)
num_atoms = mol.GetNumAtoms()
# discard burn-in batches and reshape into a single flat array
xs_proposal = xs_proposal[:, burn_in_batches:, :, :]
batch_U_proposal_fn = jax.pmap(jax.vmap(U_proposal))
batch_U_target_fn = jax.pmap(jax.vmap(U_target))
Us_target = batch_U_target_fn(xs_proposal)
Us_proposal = batch_U_proposal_fn(xs_proposal)
log_numerator = -Us_target.reshape(-1) / kT
log_denominator = -Us_proposal.reshape(-1) / kT
log_weights = log_numerator - log_denominator
# reshape into flat array by removing num_workers dimension
xs_proposal = xs_proposal.reshape(-1, num_atoms, 3)
return xs_proposal, log_weights
def test_multiprosessing(self):
def check_func(data):
pids = set()
for entry in data["traceEvents"]:
pids.add(entry["pid"])
self.assertGreater(len(pids), 1)
if multiprocessing.get_start_method() == "fork":
self.template(["viztracer", "--log_multiprocess", "-o", "result.json", "cmdline_test.py"], expected_output_file="result.json", script=file_multiprocessing, check_func=check_func, concurrency="multiprocessing")
self.template(["viztracer", "--log_multiprocess", "-o", "result.json", "cmdline_test.py"], expected_output_file="result.json", script=file_pool, check_func=check_func, concurrency="multiprocessing")
else:
self.template(["viztracer", "--log_multiprocess", "-o", "result.json", "cmdline_test.py"], script=file_multiprocessing, success=False, expected_output_file=None)
def main(
measures_to_estimate=[],
gui=False,
progress_callback: Callable = None,
config: Dict = None,
):
"""Main entrant of frds
Parameters
----------
measures_to_estimate : list, optional
list of measure names to estimate, by default []
gui : bool, optional
if in GUI mode, by default False
progress_callback : Callable, optional
function used to update progress message, by default None
"""
import inspect
import frds.measures
from multiprocessing import (
get_all_start_methods,
set_start_method,
get_start_method,
)
# Use 'fork' if available
if get_start_method(allow_none=True) is None:
if "fork" in get_all_start_methods():
set_start_method("fork")
# Use standard print function if not in GUI
progress_func = print if not gui else progress_callback.emit
# Default to estimate all measures using default parameters
if not gui:
measures = [
measure()
for _, measure in inspect.getmembers(frds.measures, inspect.isclass)
if not inspect.isabstract(measure)
]
else:
measures = [
measure()
for name, measure in inspect.getmembers(
frds.measures, inspect.isclass
)
if name in measures_to_estimate
]
# Professor at work!
with Professor(config=config, progress=progress_func) as prof:
prof.calculate(measures)
def _run_global_explorers_parallel(self, out_path):
self.log.debug("Running global explorers in {} parallel jobs".format(
self.jobs))
self.log.trace("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.trace(("Starting multiprocessing Pool for global "
"explorers run"))
args = [
(e, out_path, ) for e in self.list_global_explorer_names()
]
mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
self.log.trace(("Multiprocessing run for global explorers "
"finished"))
def initialize_runtime_options(options=None):
"""Sets and sanitizes runtime options.
'options' should be a dict-like object containing mappings from options
names to corresponding values.
"""
import multiprocessing
from . import compiler
global GlobalOptions
if not GlobalOptions:
GlobalOptions = dict()
if options:
GlobalOptions.update(options)
# Parse '--substitute-classes' and '--substitute-modules':
GlobalOptions['substitute_classes'] = \
_parse_items(GlobalOptions.get('substitute_classes'))
GlobalOptions['substitute_modules'] = \
_parse_items(GlobalOptions.get('substitute_modules'))
if GlobalOptions.get('nodename') is None:
GlobalOptions['nodename'] = ''
_set_hostname()
# Configure multiprocessing package to use chosen semantics:
startmeth = GlobalOptions.get('start_method')
if startmeth != multiprocessing.get_start_method(allow_none=True):
multiprocessing.set_start_method(startmeth)
# Convert 'compiler_flags' to a namespace object that can be passed directly
# to the compiler:
GlobalOptions['compiler_args'] \
= compiler.ui.parse_compiler_args(
GlobalOptions.get('compiler_flags', '').split())
# Make sure the directory for storing trace files exists:
if GlobalOptions.get('record_trace'):
if 'logdir' not in GlobalOptions:
raise ConfigurationError(
"'record_trace' enabled without setting 'logdir'")
os.makedirs(GlobalOptions['logdir'], exist_ok=True)
def _run_global_explorers_parallel(self, out_path):
self.log.info("Running global explorers in {} parallel jobs".format(
self.jobs))
self.log.debug("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for global "
"explorers run"))
with multiprocessing.Pool(self.jobs) as pool:
self.log.debug("Starting async for global explorer run")
results = [
pool.apply_async(self._run_global_explorer, (e, out_path,))
for e in self.list_global_explorer_names()
]
self.log.debug("Waiting async results for global explorer runs")
for r in results:
r.get() # self._run_global_explorer returns None
self.log.debug(("Multiprocessing run for global explorers "
"finished"))
def _transfer_dir_parallel(self, source, destination, jobs):
"""Transfer a directory to the remote side in parallel mode."""
self.log.debug("Remote transfer in {} parallel jobs".format(
jobs))
self.log.trace("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.trace(("Starting multiprocessing Pool for parallel "
"remote transfer"))
args = [
(command, )
for command in self._transfer_dir_commands(source, destination)
]
if len(args) == 1:
self.log.debug("Only one dir entry, transfering sequentially")
self._run_command(args[0])
else:
mp_pool_run(self._run_command, args, jobs=jobs)
self.log.trace(("Multiprocessing for parallel transfer "
"finished"))
def wrapper(*args, **kwargs):
future = ProcessFuture()
reader, writer = Pipe(duplex=False)
if get_start_method() != 'fork':
target = _trampoline
args = [function.__name__, function.__module__] + list(args)
else:
target = function
worker = launch_process(
_function_handler, target, args, kwargs, writer)
writer.close()
future.set_running_or_notify_cancel()
launch_thread(_worker_handler, future, worker, reader, timeout)
return future
def _real_worker_func(params):
# In a context where `multiprocessing` is using the `spawn` forking model,
# the new process doesn't inherit anything, so we lost all our logging
# configuration here. Let's set it up again.
if (hasattr(multiprocessing, 'get_start_method') and
multiprocessing.get_start_method() == 'spawn'):
from piecrust.main import _pre_parse_chef_args
_pre_parse_chef_args(sys.argv[1:])
wid = params.wid
logger.debug("Worker %d initializing..." % wid)
# We don't need those.
params.inqueue._writer.close()
params.outqueue._reader.close()
# Initialize the underlying worker class.
w = params.worker_class(*params.initargs)
w.wid = wid
try:
w.initialize()
except Exception as ex:
logger.error("Working failed to initialize:")
logger.exception(ex)
params.outqueue.put(None)
return
use_threads = False
if use_threads:
# Create threads to read/write the jobs and results from/to the
# main arbitrator process.
local_job_queue = queue.Queue()
reader_thread = threading.Thread(
target=_job_queue_reader,
args=(params.inqueue.get, local_job_queue),
name="JobQueueReaderThread")
reader_thread.start()
local_result_queue = queue.Queue()
writer_thread = threading.Thread(
target=_job_results_writer,
args=(local_result_queue, params.outqueue.put),
name="JobResultWriterThread")
writer_thread.start()
get = local_job_queue.get
put = local_result_queue.put_nowait
else:
get = params.inqueue.get
put = params.outqueue.put
# Start pumping!
completed = 0
time_in_get = 0
time_in_put = 0
while True:
get_start_time = time.perf_counter()
task = get()
time_in_get += (time.perf_counter() - get_start_time)
task_type, task_data = task
if task_type == TASK_END:
logger.debug("Worker %d got end task, exiting." % wid)
wprep = {
'WorkerTaskGet': time_in_get,
'WorkerResultPut': time_in_put}
try:
rep = (task_type, True, wid, (wid, w.getReport(wprep)))
except Exception as e:
logger.debug("Error getting report: %s" % e)
if params.wrap_exception:
e = multiprocessing.ExceptionWithTraceback(
e, e.__traceback__)
rep = (task_type, False, wid, (wid, e))
put(rep)
break
if task_type == TASK_JOB:
task_data = (task_data,)
for t in task_data:
try:
res = (TASK_JOB, True, wid, w.process(t))
except Exception as e:
if params.wrap_exception:
e = multiprocessing.ExceptionWithTraceback(
e, e.__traceback__)
res = (TASK_JOB, False, wid, e)
put_start_time = time.perf_counter()
put(res)
time_in_put += (time.perf_counter() - put_start_time)
completed += 1
if use_threads:
logger.debug("Worker %d waiting for reader/writer threads." % wid)
local_result_queue.put_nowait(None)
reader_thread.join()
writer_thread.join()
w.shutdown()
logger.debug("Worker %d completed %d tasks." % (wid, completed))
def entrypoint():
GlobalOptions = common.global_options()
if GlobalOptions.start_method != \
multiprocessing.get_start_method(allow_none=True):
multiprocessing.set_start_method(GlobalOptions.start_method)
target = GlobalOptions.file
source_dir = os.path.dirname(target)
basename = strip_suffix(os.path.basename(target))
if not os.access(target, os.R_OK):
die("Can not access source file %s" % target)
sys.path.insert(0, source_dir)
try:
module = import_da(basename,
from_dir=source_dir,
compiler_args=GlobalOptions.compiler_flags.split())
except ImportError as e:
die("ImportError: " + str(e))
if not (hasattr(module, 'main') and
isinstance(module.main, types.FunctionType)):
die("'main' function not defined!")
GlobalOptions.this_module_name = module.__name__
GlobalOptions.main_module_name = module.__name__
if GlobalOptions.inc_module_name is None:
GlobalOptions.inc_module_name = module.__name__ + "_inc"
common.sysinit()
# Start the background statistics thread:
RootLock.acquire()
stat_th = threading.Thread(target=collect_statistics,
name="Stat Thread")
stat_th.daemon = True
stat_th.start()
niters = GlobalOptions.iterations
stats = {'sent' : 0, 'usrtime': 0, 'systime' : 0, 'time' : 0,
'units' : 0, 'mem' : 0}
# Start main program
sys.argv = [target] + GlobalOptions.args
try:
for i in range(0, niters):
log.info("Running iteration %d ..." % (i+1))
walltime_start = time.perf_counter()
module.main()
print("Waiting for remaining child processes to terminate..."
"(Press \"Ctrl-C\" to force kill)")
walltime = time.perf_counter() - walltime_start
#log_performance_statistics(walltime)
r = aggregate_statistics()
for k, v in r.items():
stats[k] += v
for k in stats:
stats[k] /= niters
perffd = None
# if GlobalOptions.perffile is not None:
# perffd = open(GlobalOptions.perffile, "w")
if perffd is not None:
print_simple_statistics(perffd)
perffd.close()
dumpfd = None
# if GlobalOptions.dumpfile is not None:
# dumpfd = open(GlobalOptions.dumpfile, "wb")
if dumpfd is not None:
pickle.dump(stats, fd)
dumpfd.close()
except KeyboardInterrupt as e:
log.info("Received keyboard interrupt.")
except Exception as e:
err_info = sys.exc_info()
log.error("Caught unexpected global exception: %r", e)
traceback.print_tb(err_info[2])
log.info("Terminating...")
def check_json(json_conf, simple=False, external_dict=None, logger=None):
"""
Wrapper for the various checks performed on the configuration file.
:param json_conf: The dicitonary loaded from the configuration file.
:type json_conf: dict
:param simple: boolean flag indicating whether we desire
the simplified version of the configuration, or not.
:type simple: bool
:param external_dict: optional external dictionary with values to pass to the configuration.
:type external_dict: (dict|None)
:param logger: external logger instance
:type logger: Logger
:return json_conf
:rtype: dict
"""
# validator = extend_with_default(jsonschema.Draft4Validator)
#
# blue_print = os.path.join(
# os.path.dirname(os.path.abspath(__file__)),
# "configuration_blueprint.json"
# )
#
# with open(blue_print) as blue:
# blue_print = json.load(blue)
if not isinstance(logger, Logger):
logger = create_default_logger("check_json")
try:
validator = create_validator(simple=simple)
# config_folder = os.path.dirname(os.path.abspath(__file__))
# This will check for consistency and add the default
# values if they are missing
validator.validate(json_conf)
assert "files" in json_conf["pick"]
overwritten = False
json_conf, overwritten = _check_scoring_file(json_conf, logger)
if json_conf["pick"]["scoring_file"].endswith(("yaml", "json")):
with open(json_conf["pick"]["scoring_file"]) as scoring_file:
if json_conf["pick"]["scoring_file"].endswith("yaml"):
scoring = yaml.load(scoring_file)
else:
scoring = json.load(scoring_file)
assert isinstance(json_conf, dict) and isinstance(scoring, dict),\
(type(json_conf), type(scoring))
json_conf = merge_dictionaries(json_conf, scoring)
json_conf = check_all_requirements(json_conf)
json_conf = check_scoring(json_conf)
elif json_conf["pick"]["scoring_file"].endswith(("model", "pickle")):
with open(json_conf["pick"]["scoring_file"], "rb") as forest:
scoring = pickle.load(forest)
assert isinstance(scoring, dict)
assert "scoring" in scoring and isinstance(scoring["scoring"], (RandomForestRegressor, RandomForestClassifier))
del scoring["scoring"]
json_conf = merge_dictionaries(json_conf, scoring)
json_conf = check_all_requirements(json_conf)
else:
raise InvalidJson(
"Invalid scoring file: {0}".format(
json_conf["pick"]["scoring_file"]))
json_conf["__loaded_scoring"] = json_conf["pick"]["scoring_file"]
if external_dict is not None:
if not isinstance(external_dict, dict):
raise TypeError("Passed an invalid external dictionary, type {}".format(
type(external_dict)))
json_conf = merge_dictionaries(json_conf, external_dict)
json_conf = check_db(json_conf)
# json_conf = check_blast(json_conf, json_file)
validator.validate(json_conf)
# json_conf["prepare"]["canonical"] = tuple([tuple(_) for _
# in json_conf["prepare"]["canonical"]])
if not json_conf["multiprocessing_method"]:
json_conf["multiprocessing_method"] = get_start_method()
except Exception as exc:
logger.exception(exc)
raise
if overwritten is True:
logger.debug("Scoring parameters: {}".format("\n".join(["\n"] + [
"{}: {}".format(_, json_conf["scoring"][_]) for _ in json_conf["scoring"].keys()])))
return json_conf
# and you should call it again with exc=NULL to revert the effect"""
ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
raise SystemError("PyThreadState_SetAsyncExc failed")
class KillableThread(threading.Thread):
def kill(self, exctype=SystemError):
if not self.isAlive():
return
tid = self.ident
_async_raise(tid, exctype)
terminate = kill
USE_PROCESSES = os.environ.get("SEAMLESS_USE_PROCESSES")
if USE_PROCESSES is None:
USE_PROCESSES = True
if multiprocessing.get_start_method() != "fork":
USE_PROCESSES = False
else:
if USE_PROCESSES == "0" or USE_PROCESSES.upper() == "FALSE":
USE_PROCESSES = False
if USE_PROCESSES:
from multiprocessing import JoinableQueue as Queue
Executor = Process
else:
from queue import Queue
Executor = KillableThread
def return_preliminary(result_queue, value):
#print("return_preliminary", value)
result_queue.put((-1, value))
def __init__(self, model, device_ids=1, n_workers=None,
max_batch_size=None, max_image_size=DEFAULT_MAX_IMAGE_SIZE,
modder=None):
"""
Args:
- model (PyMjModel): MuJoCo model to use for rendering
- device_ids (int/list): list of device ids to use for rendering.
One or more workers will be assigned to each device, depending
on how many workers are requested.
- n_workers (int): number of parallel processes in the pool. Defaults
to the number of device ids.
- max_batch_size (int): maximum number of states that can be rendered
in batch using .render(). Defaults to the number of workers.
- max_image_size (int): maximum number pixels in images requested
by .render()
- modder (Modder): modder to use for domain randomization.
"""
self._closed, self.pool = False, None
if not (modder is None or inspect.isclass(modder)):
raise ValueError("modder must be a class")
if isinstance(device_ids, int):
device_ids = list(range(device_ids))
else:
assert isinstance(device_ids, list), (
"device_ids must be list of integer")
n_workers = n_workers or 1
self._max_batch_size = max_batch_size or (len(device_ids) * n_workers)
self._max_image_size = max_image_size
array_size = self._max_image_size * self._max_batch_size
self._shared_rgbs = Array(ctypes.c_uint8, array_size * 3)
self._shared_depths = Array(ctypes.c_float, array_size)
self._shared_rgbs_array = np.frombuffer(
self._shared_rgbs.get_obj(), dtype=ctypes.c_uint8)
assert self._shared_rgbs_array.size == (array_size * 3), (
"Array size is %d, expected %d" % (
self._shared_rgbs_array.size, array_size * 3))
self._shared_depths_array = np.frombuffer(
self._shared_depths.get_obj(), dtype=ctypes.c_float)
assert self._shared_depths_array.size == array_size, (
"Array size is %d, expected %d" % (
self._shared_depths_array.size, array_size))
worker_id = Value(ctypes.c_int)
worker_id.value = 0
if get_start_method() != "spawn":
raise RuntimeError(
"Start method must be set to 'spawn' for the "
"render pool to work. That is, you must add the "
"following to the _TOP_ of your main script, "
"before any other imports (since they might be "
"setting it otherwise):\n"
" import multiprocessing as mp\n"
" if __name__ == '__main__':\n"
" mp.set_start_method('spawn')\n")
self.pool = Pool(
processes=len(device_ids) * n_workers,
initializer=MjRenderPool._worker_init,
initargs=(
model.get_mjb(),
worker_id,
device_ids,
self._shared_rgbs,
self._shared_depths,
modder))
from concurrent.futures import TimeoutError
import pebble
from pebble import ProcessPool, ProcessExpired
# set start method
supported = False
if sys.version_info.major > 2:
methods = multiprocessing.get_all_start_methods()
if 'spawn' in methods:
try:
multiprocessing.set_start_method('spawn')
if multiprocessing.get_start_method() == 'spawn':
supported = True
except RuntimeError: # child process
pass
initarg = 0
def initializer(value):
global initarg
initarg = value
def long_initializer():
time.sleep(60)
def _real_worker_func(params):
# In a context where `multiprocessing` is using the `spawn` forking model,
# the new process doesn't inherit anything, so we lost all our logging
# configuration here. Let's set it up again.
if (hasattr(multiprocessing, 'get_start_method') and
multiprocessing.get_start_method() == 'spawn'):
from piecrust.main import _pre_parse_chef_args
_pre_parse_chef_args(sys.argv[1:])
wid = params.wid
logger.debug("Worker %d initializing..." % wid)
params.inqueue._writer.close()
params.outqueue._reader.close()
w = params.worker_class(*params.initargs)
w.wid = wid
try:
w.initialize()
except Exception as ex:
logger.error("Working failed to initialize:")
logger.exception(ex)
params.outqueue.put(None)
return
get = params.inqueue.get
put = params.outqueue.put
completed = 0
while True:
try:
task = get()
except (EOFError, OSError):
logger.debug("Worker %d encountered connection problem." % wid)
break
task_type, task_data = task
if task_type == TASK_END:
logger.debug("Worker %d got end task, exiting." % wid)
try:
rep = (task_type, True, wid, (wid, w.getReport()))
except Exception as e:
if params.wrap_exception:
e = multiprocessing.ExceptionWithTraceback(
e, e.__traceback__)
rep = (task_type, False, wid, (wid, e))
put(rep)
break
if task_type == TASK_JOB:
task_data = (task_data,)
for t in task_data:
try:
res = (TASK_JOB, True, wid, w.process(t))
except Exception as e:
if params.wrap_exception:
e = multiprocessing.ExceptionWithTraceback(
e, e.__traceback__)
res = (TASK_JOB, False, wid, e)
put(res)
completed += 1
logger.debug("Worker %d completed %d tasks." % (wid, completed))
def run(self, pool_size):
"""
this function manages the parallel processing of the url list using the python Pool class
the function first reads the list of urls out of the page_lists directory, cleans it
for known issues (eg common binary files), and issues with idna encoding (tricky!)
then the page list is mapped to the process_url function and executed in parallell
pool_size is defined in the run_webxray.py file, see details there
"""
# the list of url MUST be in the page_lists directory!
try:
url_list = open(os.path.dirname(os.path.abspath(__file__)) + '/../page_lists/' + self.pages_file_name, 'r', encoding='utf-8')
except:
print('File "%s" does not exist, file must be in ./page_lists directory. Exiting.' % self.pages_file_name)
exit()
# set up sql connection used to determine if items are already in the db
if self.db_engine == 'mysql':
from webxray.MySQLDriver import MySQLDriver
sql_driver = MySQLDriver(self.db_name)
elif self.db_engine == 'postgres':
from webxray.PostgreSQLDriver import PostgreSQLDriver
sql_driver = PostgreSQLDriver(self.db_name)
elif self.db_engine == 'sqlite':
from webxray.SQLiteDriver import SQLiteDriver
sql_driver = SQLiteDriver(self.db_name)
# this list gets mapped to the Pool, very important!
urls_to_process = set()
# simple counter used solely for updates to CLI
count = 0
print('\t------------------------')
print('\t Building List of Pages ')
print('\t------------------------')
for url in url_list:
# skip lines that are comments
if "#" in url[0]: continue
count += 1
# only do lines starting with https?://
if not (re.match('^https?://.+', url)):
print("\t\t%s | %-50s Not a valid address, Skipping." % (count, url[:50]))
continue
# non-ascii domains will crash phantomjs, so we need to convert them to
# idna/ascii/utf-8
# this requires splitting apart the url, converting the domain to idna,
# and pasting it all back together
split_url = urlsplit(url.strip())
idna_fixed_netloc = split_url.netloc.encode('idna').decode('utf-8')
url = urlunsplit((split_url.scheme,idna_fixed_netloc,split_url.path,split_url.query,split_url.fragment))
# if it is a m$ office or other doc, skip
if re.match('.+(pdf|ppt|pptx|doc|docx|txt|rtf|xls|xlsx)$', url):
print("\t\t%s | %-50s Not an HTML document, Skipping." % (count, url[:50]))
continue
# skip if in db already unless we are doing a timeseries
if self.allow_timeseries == False:
if sql_driver.page_exists(url):
print("\t\t%s | %-50s Exists in DB, Skipping." % (count, url[:50]))
continue
# only add if not in list already
if url not in urls_to_process:
print("\t\t%s | %-50s Adding." % (count, url[:50]))
urls_to_process.add(url)
else:
print("\t\t%s | %-50s Already queued, Skipping." % (count, url[:50]))
# close the db connection
sql_driver.close()
print('\t----------------------------------')
print('\t%s addresses will now be webXray\'d' % len(urls_to_process))
print('\t\tBrowser(s) are %s' % self.browser_types)
print('\t\tBrowser wait time is %s seconds' % self.browser_wait)
print('\t\t...you can go take a walk. ;-)')
print('\t----------------------------------')
# for macOS (darwin) we must specify start method as 'forkserver'
# this is essentially voodoo to ward off evil spirits which
# appear when large pool sizes are used on macOS
# get_start_method must be set to 'allow_none', otherwise upon
# checking the method it gets set (!) - and if we then get/set again
# we get an error
if sys.platform == 'darwin' and multiprocessing.get_start_method(allow_none=True) != 'forkserver':
multiprocessing.set_start_method('forkserver')
myPool = multiprocessing.Pool(pool_size)
myPool.map(self.process_url, urls_to_process)
# FYI
self.print_runtime()
import multiprocessing
from concurrent.futures import CancelledError, TimeoutError
from pebble import concurrent, ProcessExpired
# set start method
supported = False
if sys.version_info.major > 2 and sys.version_info.minor > 3:
methods = multiprocessing.get_all_start_methods()
if 'forkserver' in methods:
try:
multiprocessing.set_start_method('forkserver')
if multiprocessing.get_start_method() == 'forkserver':
supported = True
except RuntimeError: # child process
pass
@concurrent.process
def decorated(argument, keyword_argument=0):
"""A docstring."""
return argument + keyword_argument
@concurrent.process
def error_decorated():
raise RuntimeError("BOOM!")
def commandline(cls, args):
"""Configure remote system"""
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
# No new child process if only one host at a time.
if args.parallel == 1:
log.debug("Only 1 parallel process, doing it sequentially")
args.parallel = 0
if args.parallel or args.jobs:
# If parallel execution then also log process id
cdist.log.setupParallelLogging()
log = logging.getLogger("cdist")
if args.parallel:
import signal
signal.signal(signal.SIGTERM, mp_sig_handler)
signal.signal(signal.SIGHUP, mp_sig_handler)
cls._check_and_prepare_args(args)
failed_hosts = []
time_start = time.time()
cls.construct_remote_exec_copy_patterns(args)
base_root_path = cls.create_base_root_path(args.out_path)
hostcnt = 0
cfg = cdist.configuration.Configuration(args)
configuration = cfg.get_config(section='GLOBAL')
if args.tag or args.all_tagged_hosts:
inventory.determine_default_inventory_dir(args, configuration)
if args.all_tagged_hosts:
inv_list = inventory.InventoryList(
hosts=None, istag=True, hostfile=None,
db_basedir=args.inventory_dir)
else:
inv_list = inventory.InventoryList(
hosts=args.host, istag=True, hostfile=args.hostfile,
db_basedir=args.inventory_dir,
has_all_tags=args.has_all_tags)
it = inv_list.entries()
else:
it = itertools.chain(cls.hosts(args.host),
cls.hosts(args.hostfile))
process_args = []
if args.parallel:
log.trace("Processing hosts in parallel")
else:
log.trace("Processing hosts sequentially")
for entry in it:
if isinstance(entry, tuple):
# if configuring by specified tags
host = entry[0]
host_tags = entry[1]
else:
# if configuring by host then check inventory for tags
host = entry
inventory.determine_default_inventory_dir(args, configuration)
inv_list = inventory.InventoryList(
hosts=(host,), db_basedir=args.inventory_dir)
inv = tuple(inv_list.entries())
if inv:
# host is present in inventory and has tags
host_tags = inv[0][1]
else:
# host is not present in inventory or has no tags
host_tags = None
host_base_path, hostdir = cls.create_host_base_dirs(
host, base_root_path)
log.debug("Base root path for target host \"{}\" is \"{}\"".format(
host, host_base_path))
hostcnt += 1
if args.parallel:
pargs = (host, host_tags, host_base_path, hostdir, args, True,
configuration)
log.trace(("Args for multiprocessing operation "
"for host {}: {}".format(host, pargs)))
process_args.append(pargs)
else:
try:
cls.onehost(host, host_tags, host_base_path, hostdir,
args, parallel=False,
configuration=configuration)
except cdist.Error as e:
failed_hosts.append(host)
if args.parallel and len(process_args) == 1:
log.debug("Only 1 host for parallel processing, doing it "
"sequentially")
try:
cls.onehost(*process_args[0])
except cdist.Error as e:
failed_hosts.append(host)
elif args.parallel:
log.trace("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
log.trace(("Starting multiprocessing Pool for {} "
"parallel host operation".format(args.parallel)))
results = mp_pool_run(cls.onehost,
process_args,
jobs=args.parallel)
log.trace(("Multiprocessing for parallel host operation "
"finished"))
log.trace("Multiprocessing for parallel host operation "
"results: %s", results)
failed_hosts = [host for host, result in results if not result]
time_end = time.time()
log.verbose("Total processing time for %s host(s): %s", hostcnt,
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
elif not args.out_path:
# If tmp out path created then remove it, but only if no failed
# hosts.
shutil.rmtree(base_root_path)
__author__ = 'fpbatta'
import time
import numpy as np
import sys
import multiprocessing as mp
if __name__ == '__main__':
#mp.freeze_support()
print(mp.get_all_start_methods())
sm = mp.get_start_method()
print("sm: ", sm)
mp.set_start_method('forkserver', force=True)
print("sm 2: ", sm)
sys.path.append('/home/fpbatta/src/GUI/Plugins')
sys.path.append('/home/fpbatta/src/GUI/Plugins/multiprocessing_plugin')
#sys.path.append('/Users/fpbatta/src/GUImerge/GUI/Plugins/multiprocessing_plugin')
from multiprocessing_plugin.multiprocessing_plugin import MultiprocessingPlugin
m = MultiprocessingPlugin()
m.startup(20000.)
m.bufferfunction(np.random.random((11,1000)))
for i in range(100):
m.bufferfunction(200. * np.random.random((11,1000)))
time.sleep(0.05)
def _real_worker_func_unsafe(params):
init_start_time = time.perf_counter()
wid = params.wid
stats = ExecutionStats()
stats.registerTimer('WorkerInit')
# In a context where `multiprocessing` is using the `spawn` forking model,
# the new process doesn't inherit anything, so we lost all our logging
# configuration here. Let's set it up again.
if (hasattr(multiprocessing, 'get_start_method') and
multiprocessing.get_start_method() == 'spawn'):
if not params.is_unit_testing:
from piecrust.main import _pre_parse_chef_args
_pre_parse_chef_args(sys.argv[1:])
else:
_pre_parse_pytest_args()
elif params.is_unit_testing:
_pre_parse_pytest_args()
from piecrust.main import ColoredFormatter
root_logger = logging.getLogger()
root_logger.handlers[0].setFormatter(ColoredFormatter(
('[W-%d]' % wid) + '[%(name)s] %(message)s'))
logger.debug("Worker %d initializing..." % wid)
# We don't need those.
params.inqueue._writer.close()
params.outqueue._reader.close()
# Initialize the underlying worker class.
w = params.worker_class(*params.initargs)
w.wid = wid
try:
w.initialize()
except Exception as ex:
logger.error("Worker %d failed to initialize." % wid)
logger.exception(ex)
raise
stats.stepTimerSince('WorkerInit', init_start_time)
# Start pumping!
completed = 0
time_in_get = 0
time_in_put = 0
is_first_get = True
get = params.inqueue.get
put = params.outqueue.put
while True:
get_start_time = time.perf_counter()
task = get()
if not is_first_get:
time_in_get += (time.perf_counter() - get_start_time)
else:
is_first_get = False
task_type, task_data = task
# Job task(s)... just do it.
if task_type == TASK_JOB or task_type == TASK_JOB_BATCH:
task_data_list = task_data
if task_type == TASK_JOB:
task_data_list = [task_data]
result_list = []
for td in task_data_list:
try:
worker_res = w.process(td)
result_list.append((td, worker_res, True))
except Exception as e:
logger.debug(
"Error processing job, sending exception to main process:")
logger.debug(traceback.format_exc())
error_res = _get_worker_exception_data(wid)
result_list.append((td, error_res, False))
res = (task_type, wid, result_list)
put_start_time = time.perf_counter()
put(res)
time_in_put += (time.perf_counter() - put_start_time)
completed += len(task_data_list)
# End task... gather stats to send back to the main process.
elif task_type == TASK_END:
logger.debug("Worker %d got end task, exiting." % wid)
stats.registerTimer('Worker_%d_TaskGet' % wid, time=time_in_get)
stats.registerTimer('Worker_all_TaskGet', time=time_in_get)
stats.registerTimer('Worker_%d_ResultPut' % wid, time=time_in_put)
stats.registerTimer('Worker_all_ResultPut', time=time_in_put)
try:
stats.mergeStats(w.getStats())
stats_data = stats.toData()
rep = (task_type, wid, [(task_data, (wid, stats_data), True)])
except Exception as e:
logger.debug(
"Error getting report, sending exception to main process:")
logger.debug(traceback.format_exc())
we = _get_worker_exception_data(wid)
rep = (task_type, wid, [(task_data, (wid, we), False)])
put(rep)
break
# Emergy abort.
elif task_type == _TASK_ABORT_WORKER:
logger.debug("Worker %d got abort signal." % wid)
break
else:
raise Exception("Unknown task type: %s" % task_type)
try:
w.shutdown()
except Exception as e:
logger.error("Worker %s failed to shutdown.")
logger.exception(e)
raise
logger.debug("Worker %d completed %d tasks." % (wid, completed))
logging.root.addHandler(file_handler)
logger = logging.getLogger(__name__.replace("__", ""))
logger.info("Spawned")
#
# Applying Windows-specific hacks
#
os.environ["PATH"] = os.environ["PATH"] + ";" + os.path.dirname(sys.executable) # Otherwise it fails to load on Win 10
#
# Configuring multiprocessing.
# Start method must be configured globally, and only once. Using 'spawn' ensures full compatibility with Windoze.
# We need to check first if the start mode is already configured, because this code will be re-run for every child.
#
if multiprocessing.get_start_method(True) != "spawn":
multiprocessing.set_start_method("spawn")
#
# Importing other stuff once the logging has been configured
#
import uavcan
from PyQt5.QtWidgets import QApplication, QMainWindow, QWidget, QVBoxLayout, QSplitter, QAction
from PyQt5.QtGui import QKeySequence, QDesktopServices
from PyQt5.QtCore import QTimer, Qt, QUrl
from .version import __version__
from .iface_configurator import run_iface_config_window
from .active_data_type_detector import ActiveDataTypeDetector
from . import update_checker
def _iterate_once_parallel(self):
self.log.debug("Iteration in parallel mode in {} jobs".format(
self.jobs))
objects_changed = False
cargo = []
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_UNDEF:
"""Prepare the virgin object"""
# self.object_prepare(cdist_object)
# objects_changed = True
cargo.append(cdist_object)
n = len(cargo)
if n == 1:
self.log.debug("Only one object, preparing sequentially")
self.object_prepare(cargo[0])
objects_changed = True
elif cargo:
self.log.trace("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.trace("Multiprocessing cargo: %s", cargo)
cargo_types = set()
for c in cargo:
cargo_types.add(c.cdist_type)
self.log.trace("Multiprocessing cargo_types: %s", cargo_types)
nt = len(cargo_types)
if nt == 1:
self.log.debug(("Only one type, transfering explorers "
"sequentially"))
self.explorer.transfer_type_explorers(cargo_types.pop())
else:
self.log.trace(("Starting multiprocessing Pool for {} "
"parallel transfering types' explorers".format(
nt)))
args = [
(ct, ) for ct in cargo_types
]
mp_pool_run(self.explorer.transfer_type_explorers, args,
jobs=self.jobs)
self.log.trace(("Multiprocessing for parallel transfering "
"types' explorers finished"))
self.log.trace(("Starting multiprocessing Pool for {} parallel "
"objects preparation".format(n)))
args = [
(c, False, ) for c in cargo
]
mp_pool_run(self.object_prepare, args, jobs=self.jobs)
self.log.trace(("Multiprocessing for parallel object "
"preparation finished"))
objects_changed = True
del cargo[:]
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_PREPARED:
if cdist_object.requirements_unfinished(
cdist_object.autorequire):
"""The previous step created objects we depend on -
wait for them
"""
continue
# self.object_run(cdist_object)
# objects_changed = True
# put objects in chuncks of distinct types
# so that there is no more than one object
# of the same type in one chunk because there is a
# possibility of object's process locking which
# prevents parallel execution at remote
# and do this only for nonparallel marked types
for chunk in cargo:
for obj in chunk:
if (obj.cdist_type == cdist_object.cdist_type and
cdist_object.cdist_type.is_nonparallel):
break
else:
chunk.append(cdist_object)
break
else:
chunk = [cdist_object, ]
cargo.append(chunk)
for chunk in cargo:
self.log.trace("Running chunk: %s", chunk)
n = len(chunk)
if n == 1:
self.log.debug("Only one object, running sequentially")
self.object_run(chunk[0])
objects_changed = True
elif chunk:
self.log.trace("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.trace(("Starting multiprocessing Pool for {} "
"parallel object run".format(n)))
args = [
(c, ) for c in chunk
]
mp_pool_run(self.object_run, args, jobs=self.jobs)
self.log.trace(("Multiprocessing for parallel object "
"run finished"))
objects_changed = True
return objects_changed