import collections
import glob
import time
import pathlib
import json
import parsl
from parsl.app.app import bash_app
import logging
from functools import partial
logger = logging.getLogger("parsl.appworkflow")
parsl.set_stream_logger()
parsl.set_stream_logger(__name__)
logger.info("No-op log message to test log configuration")
logger.info("Parsl version is {}".format(parsl.__version__))
logger.info("Parsl file is {}".format(parsl.__file__))
# import configuration after setting parsl logging, because interesting
# construction happens during the configuration
import configuration
parsl.load(configuration.parsl_config)
# given a commandline, wrap it so that we'll invoke
# shifter appropriately (so that we don't need to
assert len(d.keys()) == n, "Only {0}/{1} keys in dict".format(
len(d.keys()), n)
[d[i].result() for i in d]
print("Duration : {0}s".format(time.time() - start))
print("[TEST STATUS] test_parallel_for [SUCCESS]")
return d
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--count",
default="10",
help="Count of apps to launch")
parser.add_argument("-d",
"--debug",
action='store_true',
help="Count of apps to launch")
args = parser.parse_args()
if args.debug:
parsl.set_stream_logger()
x = test_parallel_for()
# x = test_parallel_for(int(args.count))
# x = test_stdout()
# raise_error(0)
def run(self, debug=False):
"""
Run trials provided by the optimizer while saving results.
"""
if debug:
parsl.set_stream_logger()
self._dfk = parsl.load(self.parsl_config)
logger.info(f'Starting ParslRunner with config\n{self}')
flag = True
initialize_flag = True
result = None
for idx, parameter_configs in enumerate(self.optimizer):
try:
logger.info(
f'Writing script with configs {parameter_configs}\n')
# command_script_path, command_script_content = self._writeScript(self.command, parameter_configs, 'command')
# if self.experiment.setup_template_string != None:
# _, setup_script_content = self._writeScript(self.experiment.setup_template_string, parameter_configs, 'setup')
# else:
# setup_script_content = None
# if self.experiment.finish_template_string != None:
# _, finish_script_content = self._writeScript(self.experiment.finish_template_string, parameter_configs, 'finish')
# else:
# finish_script_content = None
setup_script_content, command_script_path, command_script_content, finish_script_content = self._createScript(
self.experiment.setup_template_string, self.command,
self.experiment.finish_template_string, parameter_configs)
# set warm-up experiments
if initialize_flag:
initialize_flag = False
logger.info(
f'[Initial trial for warm-up] Starting trial with script at {command_script_path}\n'
)
runConfig = paropt.runner.RunConfig(
command_script_content=command_script_content,
experiment_dict=self.experiment.asdict(),
setup_script_content=setup_script_content,
finish_script_content=finish_script_content,
)
initializing_func_param = {}
for key, val in self.obj_func_params.items():
initializing_func_param[key] = val
initializing_func_param['timeout'] = 300
# result = self.obj_func(runConfig, **self.obj_func_params).result()
result = self.obj_func(runConfig,
**initializing_func_param).result()
# run baseline experiment
if (self.baseline) and (self.get_baseline_output is False):
self.baseline = False
logger.info(f'Creating baseline trial')
baseline_parameter_configs = []
for parameter in self.baseline_experiment.parameters:
baseline_parameter_configs.append(
ParameterConfig(parameter=parameter,
value=parameter.minimum))
baseline_setup_script_content, baseline_command_script_path, baseline_command_script_content, baseline_finish_script_content = self._createScript(
self.experiment.setup_template_string,
self.baseline_command,
self.experiment.finish_template_string,
baseline_parameter_configs)
logger.info(
f'Starting baseline trial with script at {baseline_command_script_path}\n'
)
runConfig = paropt.runner.RunConfig(
command_script_content=baseline_command_script_content,
experiment_dict=self.baseline_experiment.asdict(),
setup_script_content=baseline_setup_script_content,
finish_script_content=baseline_finish_script_content,
)
result = None
result = self.obj_func(runConfig,
**self.obj_func_params).result()
self._validateResult(baseline_parameter_configs, result)
result['obj_parameters']['wrt_baseline'] = 1
self.baseline_obj_output = result['obj_output']
trial = Trial(
outcome=result['obj_output'],
parameter_configs=baseline_parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters=result['obj_parameters'],
)
self.storage.saveResult(self.session, trial)
self.baseline_time = result['obj_parameters'][
'caller_time']
self.get_baseline_output = True
if 'baseline_time' in self.obj_func_params.keys(
) and self.obj_func_params[
'baseline_time'] is None and self.baseline_time is not None:
self.obj_func_params['baseline_time'] = self.baseline_time
# start normal trials
logger.info(
f'Starting trial with script at {command_script_path}\n')
runConfig = paropt.runner.RunConfig(
command_script_content=command_script_content,
experiment_dict=self.experiment.asdict(),
setup_script_content=setup_script_content,
finish_script_content=finish_script_content,
)
result = None
result = self.obj_func(runConfig,
**self.obj_func_params).result()
print(result)
self._validateResult(parameter_configs, result)
if self.get_baseline_output:
result['obj_parameters']['wrt_baseline'] = result[
'obj_output'] / self.baseline_obj_output
trial = Trial(
outcome=result['obj_output'],
parameter_configs=parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters=result['obj_parameters'],
)
self.storage.saveResult(self.session, trial)
self.optimizer.register(trial)
self.run_result[
'success'] = True and self.run_result['success']
flag = flag and self.run_result['success']
self.run_result['message'][
f'experiment {self.experiment.id} run {self.run_number}, config is {ParameterConfig.configsToDict(parameter_configs)}'] = (
f'Successfully completed trials {idx} for experiment, output is {result}'
)
except Exception as e:
err_traceback = traceback.format_exc()
print(result)
if result is not None and result[
'stdout'] == 'Timeout': # for timeCommandLimitTime in lib, timeout
if self.get_baseline_output:
result['obj_parameters']['wrt_baseline'] = result[
'obj_output'] / self.baseline_obj_output
trial = Trial(
outcome=result['obj_output'],
parameter_configs=parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters=result['obj_parameters'],
)
self.optimizer.register(trial)
logger.exception(f'time out\n')
self.storage.saveResult(self.session, trial)
self.run_result['success'] = False
self.run_result['message'][
f'experiment {self.experiment.id} run {self.run_number}, config is {parameter_configs}'] = (
f'Failed to complete trials {idx} due to timeout:\nError: {e};\t{err_traceback};\toutput is {result}'
)
else: # do have error
trial = Trial(
outcome=10000000,
parameter_configs=parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters={},
)
if self.save_fail_trial:
self.storage.saveResult(self.session, trial)
self.run_result['success'] = False
self.run_result['message'][
f'experiment {self.experiment.id} run {self.run_number}, config is {parameter_configs}'] = (
f'Failed to complete trials {idx}:\nError: {e};\t{err_traceback};\toutput is {result}'
)
print(err_traceback)
print(result)
logger.info(f'Finished; Run result: {self.run_result}\n')
# plot part
if self.plot_info['draw_plot']:
try:
trials = self.storage.getTrials(self.session,
self.experiment.id)
trials_dicts = [trial.asdict() for trial in trials]
except:
self.session.rollback()
raise
logger.info(f'res: {trials_dicts}\n')
if isinstance(self.optimizer, GridSearch):
ret = GridSearch_plot(trials_dicts, self.plot_info)
else:
logger.info(f'Unsupport type of optimizer for plot\n')
if ret['success'] == False:
logger.info(f'Error when generating plot: {ret["error"]}\n')
else:
logger.info(f'Successfully generating plot {ret["error"]}\n')
else:
logger.info(f'Skip generating plot\n')
import configuration
import ingest
import tracts
import visits
import genCoaddVisitLists
from lsst_apps import lsst_app2
from workflowutils import read_and_strip
from future_combinators import combine
#############################
## Setup and Initialization
#############################
# Establish logging
logger = logging.getLogger("parsl.workflow")
parsl.set_stream_logger(level=logging.INFO)
logger.info("WFLOW: Parsl driver for DM (DRP) pipeline")
# Read in workflow configuration
configuration = configuration.load_configuration()
# TODO:
# restarts by reruns
# this should go into the user config file but I'll prototype it here
# some of groups of steps of the broad workflow should have their own
# rerun directory in the repo.
# however because some are in parallel, they can't be entirely separate
# because they need to be linear (unless there is interesting rerun-merging
# magic)
import time
from funcx.executors import HighThroughputExecutor as HTEX
from parsl import set_stream_logger
from parsl.channels import LocalChannel
from parsl.providers import LocalProvider
def double(x):
return x * 2
if __name__ == "__main__":
set_stream_logger()
executor = HTEX(
label="htex",
provider=LocalProvider(channel=LocalChannel),
address="34.207.74.221",
)
ports = executor.start()
print(f"Connect on ports {ports}")
print("Submitting")
fu = executor.submit(double, *[2])
print("Submitted")
time.sleep(600)
def main():
args = parse_args(sys.argv[1:])
# Copy V-pipe repo as main working directory
tmp_dir = join(cwd, 'tmp')
vpipe_dir = join(tmp_dir, 'vpipe')
base_params = {
'container': abspath(args.container),
'work_dir': tmp_dir,
'read_length': args.read_length,
'variant_frequency': args.variant_frequency,
'read_cutoff_bp': args.read_cutoff_bp,
'primers_bp': args.primers_bp,
'depth_cap': float(args.depth_cap),
'stdout': abspath(join(args.outputs, 'mappgene.stdout')),
}
if shutil.which('singularity') is None:
raise Exception(
f"Missing Singularity executable in PATH.\n\n" +
f"Please ensure Singularity is installed: https://sylabs.io/guides/3.0/user-guide/installation.html"
)
if not exists(base_params['container']):
raise Exception(
f"Missing container image at {base_params['container']}\n\n" +
f"Either specify another image with --container\n" +
f"Or build the container with the recipe at: {join(script_dir, 'data/container/recipe.def')}\n"
+
f"Or download the container with this command:\n\n$ singularity pull image.sif library://avilaherrera1/mappgene/image.sif:latest\n"
)
smart_remove(tmp_dir)
smart_mkdir(tmp_dir)
run(f'cp -rf /opt/vpipe {vpipe_dir}', base_params)
smart_copy(join(script_dir, 'data/extra_files'), tmp_dir)
run(f'cd {vpipe_dir} && sh init_project.sh || true', base_params)
update_permissions(tmp_dir, base_params)
if args.test:
args.inputs = join(script_dir, 'data/example_inputs/*.fastq.gz')
if isinstance(args.inputs, str):
args.inputs = glob(args.inputs)
all_params = {}
# Copy reads to subject directory
for input_read in args.inputs:
pair1 = input_read.replace('_R2', '_R1')
pair2 = input_read.replace('_R1', '_R2')
if input_read != pair1 and pair2 not in args.inputs:
raise Exception(f'Missing paired read: {pair2}')
if input_read != pair2 and pair1 not in args.inputs:
raise Exception(f'Missing paired read: {pair1}')
subject = (basename(input_read).replace('.fastq.gz', '').replace(
'.fastq', '').replace('_R1', '').replace('_R2',
'').replace('.', '_'))
subject_dir = abspath(join(args.outputs, subject))
if not subject in all_params:
smart_copy(tmp_dir, subject_dir)
params = base_params.copy()
params['work_dir'] = subject_dir
params['input_reads'] = [input_read]
params['stdout'] = join(subject_dir, 'worker.stdout')
all_params[subject] = params
else:
all_params[subject]['input_reads'].append(input_read)
if args.slurm:
executor = parsl.executors.HighThroughputExecutor(
label="worker",
address=parsl.addresses.address_by_hostname(),
provider=parsl.providers.SlurmProvider(
args.partition,
launcher=parsl.launchers.SrunLauncher(),
nodes_per_block=int(args.nnodes),
init_blocks=1,
max_blocks=1,
worker_init=f"export PYTHONPATH=$PYTHONPATH:{os.getcwd()}",
walltime=args.walltime,
scheduler_options="#SBATCH --exclusive\n#SBATCH -A {}\n".
format(args.bank),
move_files=False,
),
)
elif args.flux:
executor = parsl.executors.FluxExecutor(
label="worker",
flux_path=
"/usr/global/tools/flux/toss_3_x86_64_ib/flux-c0.28.0.pre-s0.17.0.pre/bin/flux",
provider=parsl.providers.SlurmProvider(
args.partition,
launcher=parsl.launchers.SrunLauncher(),
nodes_per_block=int(args.nnodes),
init_blocks=1,
max_blocks=1,
worker_init=f"export PYTHONPATH=$PYTHONPATH:{os.getcwd()}",
walltime=args.walltime,
scheduler_options="#SBATCH --exclusive\n#SBATCH -A {}\n".
format(args.bank),
move_files=False,
),
)
else:
executor = parsl.executors.ThreadPoolExecutor(label="worker")
config = parsl.config.Config(executors=[executor])
parsl.set_stream_logger()
parsl.load(config)
if args.ivar:
results = []
for params in all_params.values():
results.append(run_ivar(params))
for r in results:
r.result()
elif args.vpipe:
results = []
for params in all_params.values():
results.append(run_vpipe(params))
for r in results:
r.result()
def run(self, debug=False):
"""
Run trials provided by the optimizer while saving results.
"""
if debug:
parsl.set_stream_logger()
self._dfk = parsl.load(self.parsl_config)
logger.info(f'Starting ParslRunner with config\n{self}')
flag = True
initialize_flag = True
result = None
for idx, parameter_configs in enumerate(self.optimizer):
try:
logger.info(f'Writing script with configs {parameter_configs}')
command_script_path, command_script_content = self._writeScript(
self.command, parameter_configs, 'command')
if self.experiment.setup_template_string != None:
_, setup_script_content = self._writeScript(
self.experiment.setup_template_string,
parameter_configs, 'setup')
else:
setup_script_content = None
if self.experiment.finish_template_string != None:
_, finish_script_content = self._writeScript(
self.experiment.finish_template_string,
parameter_configs, 'finish')
else:
finish_script_content = None
# set warm-up experiments
if initialize_flag:
initialize_flag = False
logger.info(
f'Starting initializing trial with script at {command_script_path}'
)
runConfig = paropt.runner.RunConfig(
command_script_content=command_script_content,
experiment_dict=self.experiment.asdict(),
setup_script_content=setup_script_content,
finish_script_content=finish_script_content,
)
initializing_func_param = {}
for key, val in self.obj_func_params.items():
initializing_func_param[key] = val
initializing_func_param['timeout'] = 300
# result = self.obj_func(runConfig, **self.obj_func_params).result()
result = self.obj_func(runConfig,
**initializing_func_param).result()
logger.info(
f'Starting trial with script at {command_script_path}')
runConfig = paropt.runner.RunConfig(
command_script_content=command_script_content,
experiment_dict=self.experiment.asdict(),
setup_script_content=setup_script_content,
finish_script_content=finish_script_content,
)
result = None
result = self.obj_func(runConfig,
**self.obj_func_params).result()
self._validateResult(parameter_configs, result)
trial = Trial(
outcome=result['obj_output'],
parameter_configs=parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters=result['obj_parameters'],
)
self.storage.saveResult(self.session, trial)
self.optimizer.register(trial)
self.run_result[
'success'] = True and self.run_result['success']
flag = flag and self.run_result['success']
self.run_result['message'][
f'experiment {self.experiment.id} run {self.run_number}, config is {parameter_configs}'] = (
f'Successfully completed trials {idx} for experiment')
except Exception as e:
err_traceback = traceback.format_exc()
if result is not None and result[
'stdout'] == 'Timeout': # for timeCommandLimitTime in lib, timeout
trial = Trial(
outcome=result['obj_output'],
parameter_configs=parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters=result['obj_parameters'],
)
self.optimizer.register(trial)
logger.exception(f'time out')
self.storage.saveResult(self.session, trial)
self.run_result['success'] = False
self.run_result['message'][
f'experiment {self.experiment.id} run {self.run_number}, config is {parameter_configs}'] = (
f'Failed to complete trials {idx}:\nError: {e}\n{err_traceback}'
)
else:
trial = Trial(
outcome=10000000,
parameter_configs=parameter_configs,
run_number=self.run_number,
experiment_id=self.experiment.id,
obj_parameters={},
)
self.storage.saveResult(self.session, trial)
self.run_result['success'] = False
self.run_result['message'][
f'experiment {self.experiment.id} run {self.run_number}, config is {parameter_configs}'] = (
f'Failed to complete trials {idx}:\nError: {e}\n{err_traceback}'
)
logger.info(f'Finished; Run result: {self.run_result}')
def workflow_config(name,
nodes,
cores_per_node=24,
interval=30,
monitor=False):
import parsl
from parsl.config import Config
from parsl.channels import LocalChannel
from parsl.launchers import SrunLauncher
from parsl.providers import LocalProvider
from parsl.addresses import address_by_interface
from parsl.executors import HighThroughputExecutor
from parsl.monitoring.monitoring import MonitoringHub
parsl.set_stream_logger()
parsl.set_file_logger('script.output', level=logging.DEBUG)
logging.info('Configuring Parsl Workflow Infrastructure')
#Read where datasets are...
env_str = str()
with open('parsl.env', 'r') as reader:
env_str = reader.read()
logging.info(f'Task Environment {env_str}')
mon_hub = MonitoringHub(
workflow_name=name,
hub_address=address_by_interface('ib0'),
hub_port=60001,
resource_monitoring_enabled=True,
monitoring_debug=False,
resource_monitoring_interval=interval,
) if monitor else None
config = Config(
executors=[
HighThroughputExecutor(
label=name,
# Optional: The network interface on node 0 which compute nodes can communicate with.
# address=address_by_interface('enp4s0f0' or 'ib0')
address=address_by_interface('ib0'),
# one worker per manager / node
max_workers=cores_per_node,
provider=LocalProvider(
channel=LocalChannel(script_dir='.'),
# make sure the nodes_per_block matches the nodes requested in the submit script in the next step
nodes_per_block=nodes,
# make sure
launcher=SrunLauncher(overrides=f'-c {cores_per_node}'),
cmd_timeout=120,
init_blocks=1,
max_blocks=1,
worker_init=env_str,
),
)
],
monitoring=mon_hub,
strategy=None,
)
logging.info('Loading Parsl Config')
parsl.load(config)
return
def cli_run():
parser = argparse.ArgumentParser()
parser.add_argument(
"--redishost",
default="127.0.0.1",
help="Address at which the redis server can be reached")
parser.add_argument("--redisport",
default="6379",
help="Port on which redis is available")
parser.add_argument("-d",
"--debug",
action='store_true',
help="Count of apps to launch")
parser.add_argument("-m",
"--mac",
action='store_true',
help="Configure for Mac")
args = parser.parse_args()
if args.debug:
parsl.set_stream_logger()
if args.mac:
config = Config(
executors=[
ThreadPoolExecutor(label="htex"),
ThreadPoolExecutor(label="local_threads")
],
strategy=None,
)
else:
config = Config(
executors=[
HighThroughputExecutor(
label="htex",
# Max workers limits the concurrency exposed via mom node
max_workers=2,
provider=LocalProvider(
init_blocks=1,
max_blocks=1,
),
),
ThreadPoolExecutor(label="local_threads")
],
strategy=None,
)
parsl.load(config)
print(
'''This program creates an "MPI Method Server" that listens on an inputs queue and write on an output queue:
input_queue --> mpi_method_server --> queues
To send it a request, add an entry to the inputs queue:
run "pipeline-pump -p N" where N is an integer request
To access a value, remove it from the outout queue:
run "pipeline-pull" (blocking) or "pipeline-pull -t T" (T an integer) to time out after T seconds
TODO: Timeout does not work yet!
''')
# Get the queues for the method server
method_queues = MethodServerQueues(args.redishost, port=args.redisport)
# Start the method server
mms = ParslMethodServer([target_fun],
method_queues,
default_executors=['htex'])
mms.run()