def __init__(self, config, channel=None): ''' Initialize the Torque class Args: - Config (dict): Dictionary with all the config options. KWargs: - Channel (None): A channel is required for torque. ''' self.channel = channel if self.channel is None: logger.error( "Provider:Torque cannot be initialized without a channel") raise (ep_error.ChannelRequired( self.__class__.__name__, "Missing a channel to execute commands")) self.config = config self.sitename = config['site'] self.current_blocksize = 0 self.scriptDir = self.config["execution"]["scriptDir"] if not os.path.exists(self.scriptDir): os.makedirs(self.scriptDir) # Dictionary that keeps track of jobs, keyed on job_id self.resources = {}
def __init__(self, config, channel=None): ''' Initialize the Condor class Args: - Config (dict): Dictionary with all the config options. KWargs: - Channel (none): A channel is required for htcondor. ''' self.channel = channel if self.channel == None: logger.error( "Provider:Condor cannot be initialized without a channel") raise (ep_error.ChannelRequired( self.__class__.__name__, "Missing a channel to execute commands")) self.config = config self.sitename = config['site'] self.current_blocksize = 0 self.max_walltime = wtime_to_minutes( self.config["execution"]["block"].get("walltime", '01:00:00')) if not os.path.exists(self.config["execution"].get( "scriptDir", '.scripts')): os.makedirs(self.config["execution"]["scriptDir"]) # Dictionary that keeps track of jobs, keyed on job_id self.resources = {}
def __init__(self, config, channel=None): ''' Here we do initialization that is common across all cluster-style providers Args: - Config (dict): Dictionary with all the config options. KWargs: - Channel (None): A channel is required for all cluster-style providers ''' self._scaling_enabled = True self._channels_required = True self.channel = channel if self.channel is None: logger.error("Provider: Cannot be initialized without a channel") raise (ep_error.ChannelRequired( self.__class__.__name__, "Missing a channel to execute commands")) self.config = config self.sitename = config['site'] self.current_blocksize = 0 launcher_name = self.config["execution"]["block"].get( "launcher", "singleNode") self.launcher = Launchers.get(launcher_name, None) self.max_walltime = wtime_to_minutes( self.config["execution"]["block"].get("walltime", '01:00:00')) self.scriptDir = self.config["execution"]["scriptDir"] if not os.path.exists(self.scriptDir): os.makedirs(self.scriptDir) # Dictionary that keeps track of jobs, keyed on job_id self.resources = {}
def __init__(self, channel=None, label='condor', nodes_per_block=1, max_blocks=10, environment=None, script_dir='parsl_scripts', project='', overrides='', worker_setup='', requirements=''): self.channel = channel if self.channel is None: logger.error( "Condor provider cannot be initialized without a channel") raise (ep_error.ChannelRequired( self.__class__.__name__, "Missing a channel to execute commands")) self.label = label self.nodes_per_block = nodes_per_block self.provisioned_blocks = 0 self.max_blocks = max_blocks self.environment = environment if environment is not None else {} for key, value in self.environment.items(): # To escape literal quote marks, double them # See: http://research.cs.wisc.edu/htcondor/manual/v8.6/condor_submit.html try: self.environment[key] = "'{}'".format( value.replace("'", '"').replace('"', '""')) except AttributeError: pass self.script_dir = script_dir if not os.path.exists(self.script_dir): os.makedirs(self.script_dir) self.project = project self.overrides = overrides self.worker_setup = worker_setup self.requirements = requirements self.resources = { } # Dictionary that keeps track of jobs, keyed on job_id
def __init__(self, channel, label='cobalt', script_dir='parsl_scripts', launcher='single_node', nodes_per_block=1, tasks_per_node=1, min_blocks=0, max_blocks=10, walltime="00:10:00", account=None, queue=None, overrides=''): if channel is None: logger.error("Cobalt cannot be initialized without a channel") raise (ep_error.ChannelRequired( self.__class__.__name__, "Missing a channel to execute commands")) self.channel = channel self.label = label self.provisioned_blocks = 0 self.nodes_per_block = nodes_per_block self.launcher = launcher self.tasks_per_node = tasks_per_node self.min_blocks = min_blocks self.max_blocks = max_blocks self.walltime = wtime_to_minutes(walltime) self.account = account self.overrides = overrides self.script_dir = script_dir if not os.path.exists(self.script_dir): os.makedirs(self.script_dir) # Dictionary that keeps track of jobs, keyed on job_id self.resources = {} if launcher in ['srun', 'srun_mpi']: logger.warning( "Use of {} launcher is usually appropriate for Slurm providers. " "Recommended options include 'single_node' or 'aprun'.".format( launcher))
def __init__(self, channel, account, queue=None, overrides='', label='torque', script_dir='parsl_scripts', nodes_per_block=1, tasks_per_node=1, init_blocks=1, min_blocks=0, max_blocks=100, parallelism=1, walltime="00:20:00"): self.channel = channel if self.channel is None: logger.error("Provider:Torque cannot be initialized without a channel") raise (ep_error.ChannelRequired(self.__class__.__name__, "Missing a channel to execute commands")) self.account = account self.queue = queue self.overrides = overrides self.label = label self.nodes_per_block = nodes_per_block self.tasks_per_node = tasks_per_node self.init_blocks = init_blocks self.min_blocks = min_blocks self.max_blocks = max_blocks self.parallelism = parallelism self.walltime = walltime self.provisioned_blocks = 0 self.script_dir = script_dir if not os.path.exists(self.script_dir): os.makedirs(self.script_dir) # Dictionary that keeps track of jobs, keyed on job_id self.resources = {}