def test_factory_1(): config = { "sites": [{ "site": "RCC_Midway_Remote", "auth": { "channel": "ssh", "hostname": "swift.rcc.uchicago.edu", "username": "******" }, "execution": { "executor": "ipp", "provider": "slurm", # LIKELY SHOULD BE BOUND TO SITE "scriptDir": ".scripts", "block": { # Definition of a block "nodes": 1, # of nodes in that block "taskBlocks": 1, # total tasks in a block "walltime": "00:05:00", "Options": { "partition": "debug", "account": "pi-wilde", "overrides": "#SBATCH--constraint=haswell" } } } }], "globals": { "lazyErrors": True } } epf = ExecProviderFactory() epf.make(config)
def __init__(self, config=None, executors=None, lazy_fail=True, rundir=None, fail_retries=2): """ Initialize the DataFlowKernel Please note that keyword args passed to the DFK here will always override options passed in via the config. KWargs: config (Dict) : A single data object encapsulating all config attributes executors (list of Executor objs): Optional, kept for (somewhat) backward compatibility with 0.2.0 lazy_fail(Bool) : Default=True, determine failure behavior rundir (str) : Path to run directory. Defaults to ./runinfo/runNNN fail_retries(int): Default=2, Set the number of retry attempts in case of failure Returns: DataFlowKernel object """ # Create run dirs for this run self.rundir = make_rundir(config=config, path=rundir) # Update config with defaults self._config = update_config(config, self.rundir) # Start the anonymized usage tracker and send init msg self.usage_tracker = UsageTracker(self) self.usage_tracker.send_message() if self._config : self._executors_managed = True # Create the executors epf = EPF() self.executors = epf.make(self.rundir, self._config) # set global vars from config self.lazy_fail = self._config["globals"].get("lazyFail", lazy_fail) self.fail_retries = self._config["globals"].get("fail_retries", fail_retries) self.flowcontrol = FlowControl(self, self._config) else: self._executors_managed = False self.fail_retries = fail_retries self.lazy_fail = lazy_fail self.executors = {i:x for i,x in enumerate(executors)} print("Executors : ", self.executors) self.flowcontrol = FlowNoControl(self, None) self.task_count = 0 self.fut_task_lookup = {} self.tasks = {} logger.debug("Using executors: {0}".format(self.executors)) atexit.register(self.cleanup)
def test_factory_2() : config = { "site" : "ipp_local", "execution" : { "executor" : "ipp", "provider" : "local", "channel" : "None", "options" : {"initParallelism" : 2, "maxParallelism" : 2, "minParallelism" : 0, "walltime" : "00:25:00", } }} epf = ExecProviderFactory() executor = epf.make(config)
def test_factory_3(): config = { "sites": [{ "site": "Local", "auth": { "channel": None }, "execution": { "executor": "threads", "provider": None, # LIKELY SHOULD BE BOUND TO SITE "maxThreads": 4 } }], "globals": { "lazyErrors": True } } epf = ExecProviderFactory() epf.make(config)
def __init__(self, config=None, executors=None, lazyErrors=True, appCache=True, rundir=None, retries=0, checkpointFiles=None, checkpointMode=None, data_manager=None): """ Initialize the DataFlowKernel. Please note that keyword args passed to the DFK here will always override options passed in via the config. KWargs: - config (dict) : A single data object encapsulating all config attributes - executors (list of Executor objs): Optional, kept for (somewhat) backward compatibility with 0.2.0 - lazyErrors(bool) : Default=True, allow workflow to continue on app failures. - appCache (bool) :Enable caching of apps - rundir (str) : Path to run directory. Defaults to ./runinfo/runNNN - retries(int): Default=0, Set the number of retry attempts in case of failure - checkpointFiles (list of str): List of filepaths to checkpoint files - checkpointMode (None, 'dfk_exit', 'task_exit', 'periodic'): Method to use. - data_manager (DataManager): User created DataManager Returns: DataFlowKernel object """ # Create run dirs for this run self.rundir = make_rundir(config=config, path=rundir) parsl.set_file_logger("{}/parsl.log".format(self.rundir), level=logging.DEBUG) logger.info("Parsl version: {}".format(parsl.__version__)) logger.info("Libsubmit version: {}".format(libsubmit.__version__)) # Update config with defaults self._config = update_config(config, self.rundir) # Set the data manager if data_manager: self.data_manager = data_manager else: self.data_manager = DataManager(config=self._config) # Start the anonymized usage tracker and send init msg self.usage_tracker = UsageTracker(self) self.usage_tracker.send_message() # Load checkpoints if any cpts = self.load_checkpoints(checkpointFiles) # Initialize the memoizer self.memoizer = Memoizer(self, memoize=appCache, checkpoint=cpts) self.checkpointed_tasks = 0 self._checkpoint_timer = None if self._config: self._executors_managed = True # Create the executors epf = EPF() self.executors = epf.make(self.rundir, self._config) # set global vars from config self.lazy_fail = self._config["globals"].get( "lazyErrors", lazyErrors) self.fail_retries = self._config["globals"].get("retries", retries) self.flowcontrol = FlowControl(self, self._config) self.checkpoint_mode = self._config["globals"].get( "checkpointMode", checkpointMode) if self.checkpoint_mode == "periodic": period = self._config["globals"].get("checkpointPeriod", "00:30:00") try: h, m, s = map(int, period.split(':')) checkpoint_period = (h * 3600) + (m * 60) + s self._checkpoint_timer = Timer(self.checkpoint, interval=checkpoint_period) except Exception as e: logger.error( "invalid checkpointPeriod provided:{0} expected HH:MM:SS" .format(period)) self._checkpoint_timer = Timer(self.checkpoint, interval=(30 * 60)) else: self._executors_managed = False self.fail_retries = retries self.lazy_fail = lazyErrors self.executors = {i: x for i, x in enumerate(executors)} self.flowcontrol = FlowNoControl(self, None) self.checkpoint_mode = checkpointMode self.task_count = 0 self.fut_task_lookup = {} self.tasks = {} self.task_launch_lock = threading.Lock() logger.debug("Using executors: {0}".format(self.executors)) atexit.register(self.cleanup)
def __init__(self, config=None, executors=None, lazyErrors=True, appCache=True, rundir=None, retries=0, checkpointFiles=None): """ Initialize the DataFlowKernel Please note that keyword args passed to the DFK here will always override options passed in via the config. KWargs: - config (Dict) : A single data object encapsulating all config attributes - executors (list of Executor objs): Optional, kept for (somewhat) backward compatibility with 0.2.0 - lazyErrors(Bool) : Default=True, allow workflow to continue on app failures. - appCache (Bool) :Enable caching of apps - rundir (str) : Path to run directory. Defaults to ./runinfo/runNNN - retries(int): Default=0, Set the number of retry attempts in case of failure - checkpointFiles (list of str): List of filepaths to checkpoint files Returns: DataFlowKernel object """ # Create run dirs for this run self.rundir = make_rundir(config=config, path=rundir) parsl.set_file_logger("{}/parsl.log".format(self.rundir), level=logging.INFO) logger.info("Parsl version: {}".format(parsl.__version__)) logger.info("Libsubmit version: {}".format(libsubmit.__version__)) # Update config with defaults self._config = update_config(config, self.rundir) # Start the anonymized usage tracker and send init msg self.usage_tracker = UsageTracker(self) self.usage_tracker.send_message() # Load checkpoints if any cpts = self.load_checkpoints(checkpointFiles) # Initialize the memoizer self.memoizer = Memoizer(self, memoize=appCache, checkpoint=cpts) if self._config: self._executors_managed = True # Create the executors epf = EPF() self.executors = epf.make(self.rundir, self._config) # set global vars from config self.lazy_fail = self._config["globals"].get( "lazyErrors", lazyErrors) self.fail_retries = self._config["globals"].get("retries", retries) self.flowcontrol = FlowControl(self, self._config) else: self._executors_managed = False self.fail_retries = retries self.lazy_fail = lazyErrors self.executors = {i: x for i, x in enumerate(executors)} self.flowcontrol = FlowNoControl(self, None) self.task_count = 0 self.fut_task_lookup = {} self.tasks = {} self.task_launch_lock = threading.Lock() logger.debug("Using executors: {0}".format(self.executors)) atexit.register(self.cleanup)
"walltime": "00:10:00", "account": "pi-chard", "submit_script_dir": ".scripts" } }} import subprocess import random print("starting ipcontroller") rand_port = random.randint(55000, 59000) p = subprocess.Popen(['ipcontroller', '--reuse', '--port={0}'.format(rand_port), '--ip=*'], stdout=None, stderr=None) epf = ExecProviderFactory() resource = epf.make(config) dfk = DataFlowKernel(resource) # Helper function to create an input file from a template def create_template(template_name, output_name, contents): fs_loader = FileSystemLoader('templates') env = Environment(loader=fs_loader) template = env.get_template(template_name) packmol_file = open(output_name, 'w') packmol_file.write(template.render(contents)) packmol_file.close() @App('bash', dfk)