def test_can_iterate_over_activities():
g = run_activities(experiments.Experiment,
configuration=None,
secrets=None,
pool=None,
dry=False)
assert isinstance(g, types.GeneratorType)
def apply_activities(experiment: Experiment,
configuration: Configuration,
secrets: Secrets,
pool: ThreadPoolExecutor,
dry: bool = False) -> List[Run]:
with controls(level="method",
experiment=experiment,
context=experiment,
configuration=configuration,
secrets=secrets) as control:
runs = list(
run_activities(experiment, configuration, secrets, pool, dry))
if pool:
logger.debug("Waiting for background activities to complete...")
pool.shutdown(wait=True)
result = []
for run in runs:
if not run:
continue
if isinstance(run, dict):
result.append(run)
else:
result.append(run.result())
control.with_state(result)
return result
def apply_activities(experiment: Experiment,
configuration: Configuration,
secrets: Secrets,
pool: ThreadPoolExecutor,
journal: Journal,
dry: bool = False) -> List[Run]:
with controls(level="method",
experiment=experiment,
context=experiment,
configuration=configuration,
secrets=secrets) as control:
result = []
runs = []
method = experiment.get("method", [])
wait_for_background_activities = True
try:
for run in run_activities(experiment, configuration, secrets, pool,
dry):
runs.append(run)
if journal["status"] in ["aborted", "failed", "interrupted"]:
break
except SystemExit as x:
# when we got a signal for an ungraceful exit, we can decide
# not to wait for background activities. Their statuses will
# remain failed.
wait_for_background_activities = x.code != 30 # see exit.py
raise
finally:
background_activity_timeout = None
if wait_for_background_activities and pool:
logger.debug("Waiting for background activities to complete")
pool.shutdown(wait=True)
elif pool:
harshly_terminate_pending_background_activities(pool)
logger.debug(
"Do not wait for the background activities to finish "
"as per signal")
background_activity_timeout = 0.2
pool.shutdown(wait=False)
for index, run in enumerate(runs):
if not run:
continue
if isinstance(run, dict):
result.append(run)
else:
try:
# background activities
result.append(
run.result(timeout=background_activity_timeout))
except TimeoutError:
# we want an entry for the background activity in our
# results anyway, we won't have anything meaningful
# to say about it
result.append({
"activity": method[index],
"status": "failed",
"output": None,
"duration": None,
"start": None,
"end": None,
"exception": None
})
# now let's ensure the journal has all activities in their correct
# order (background ones included)
journal["run"] = result
control.with_state(result)
return result
