def __init__(self, config, channel=None):
''' Initialize the Torque class
Args:
- Config (dict): Dictionary with all the config options.
KWargs:
- Channel (None): A channel is required for torque.
'''
self.channel = channel
if self.channel is None:
logger.error(
"Provider:Torque cannot be initialized without a channel")
raise (ep_error.ChannelRequired(
self.__class__.__name__,
"Missing a channel to execute commands"))
self.config = config
self.sitename = config['site']
self.current_blocksize = 0
self.scriptDir = self.config["execution"]["scriptDir"]
if not os.path.exists(self.scriptDir):
os.makedirs(self.scriptDir)
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
def __init__(self, config, channel=None):
''' Initialize the Condor class
Args:
- Config (dict): Dictionary with all the config options.
KWargs:
- Channel (none): A channel is required for htcondor.
'''
self.channel = channel
if self.channel == None:
logger.error(
"Provider:Condor cannot be initialized without a channel")
raise (ep_error.ChannelRequired(
self.__class__.__name__,
"Missing a channel to execute commands"))
self.config = config
self.sitename = config['site']
self.current_blocksize = 0
self.max_walltime = wtime_to_minutes(
self.config["execution"]["block"].get("walltime", '01:00:00'))
if not os.path.exists(self.config["execution"].get(
"scriptDir", '.scripts')):
os.makedirs(self.config["execution"]["scriptDir"])
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
def __init__(self, config, channel=None):
''' Here we do initialization that is common across all cluster-style providers
Args:
- Config (dict): Dictionary with all the config options.
KWargs:
- Channel (None): A channel is required for all cluster-style providers
'''
self._scaling_enabled = True
self._channels_required = True
self.channel = channel
if self.channel is None:
logger.error("Provider: Cannot be initialized without a channel")
raise (ep_error.ChannelRequired(
self.__class__.__name__,
"Missing a channel to execute commands"))
self.config = config
self.sitename = config['site']
self.current_blocksize = 0
launcher_name = self.config["execution"]["block"].get(
"launcher", "singleNode")
self.launcher = Launchers.get(launcher_name, None)
self.max_walltime = wtime_to_minutes(
self.config["execution"]["block"].get("walltime", '01:00:00'))
self.scriptDir = self.config["execution"]["scriptDir"]
if not os.path.exists(self.scriptDir):
os.makedirs(self.scriptDir)
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
def __init__(self,
channel=None,
label='condor',
nodes_per_block=1,
max_blocks=10,
environment=None,
script_dir='parsl_scripts',
project='',
overrides='',
worker_setup='',
requirements=''):
self.channel = channel
if self.channel is None:
logger.error(
"Condor provider cannot be initialized without a channel")
raise (ep_error.ChannelRequired(
self.__class__.__name__,
"Missing a channel to execute commands"))
self.label = label
self.nodes_per_block = nodes_per_block
self.provisioned_blocks = 0
self.max_blocks = max_blocks
self.environment = environment if environment is not None else {}
for key, value in self.environment.items():
# To escape literal quote marks, double them
# See: http://research.cs.wisc.edu/htcondor/manual/v8.6/condor_submit.html
try:
self.environment[key] = "'{}'".format(
value.replace("'", '"').replace('"', '""'))
except AttributeError:
pass
self.script_dir = script_dir
if not os.path.exists(self.script_dir):
os.makedirs(self.script_dir)
self.project = project
self.overrides = overrides
self.worker_setup = worker_setup
self.requirements = requirements
self.resources = {
} # Dictionary that keeps track of jobs, keyed on job_id
def __init__(self,
channel,
label='cobalt',
script_dir='parsl_scripts',
launcher='single_node',
nodes_per_block=1,
tasks_per_node=1,
min_blocks=0,
max_blocks=10,
walltime="00:10:00",
account=None,
queue=None,
overrides=''):
if channel is None:
logger.error("Cobalt cannot be initialized without a channel")
raise (ep_error.ChannelRequired(
self.__class__.__name__,
"Missing a channel to execute commands"))
self.channel = channel
self.label = label
self.provisioned_blocks = 0
self.nodes_per_block = nodes_per_block
self.launcher = launcher
self.tasks_per_node = tasks_per_node
self.min_blocks = min_blocks
self.max_blocks = max_blocks
self.walltime = wtime_to_minutes(walltime)
self.account = account
self.overrides = overrides
self.script_dir = script_dir
if not os.path.exists(self.script_dir):
os.makedirs(self.script_dir)
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
if launcher in ['srun', 'srun_mpi']:
logger.warning(
"Use of {} launcher is usually appropriate for Slurm providers. "
"Recommended options include 'single_node' or 'aprun'.".format(
launcher))
def __init__(self,
channel,
account,
queue=None,
overrides='',
label='torque',
script_dir='parsl_scripts',
nodes_per_block=1,
tasks_per_node=1,
init_blocks=1,
min_blocks=0,
max_blocks=100,
parallelism=1,
walltime="00:20:00"):
self.channel = channel
if self.channel is None:
logger.error("Provider:Torque cannot be initialized without a channel")
raise (ep_error.ChannelRequired(self.__class__.__name__, "Missing a channel to execute commands"))
self.account = account
self.queue = queue
self.overrides = overrides
self.label = label
self.nodes_per_block = nodes_per_block
self.tasks_per_node = tasks_per_node
self.init_blocks = init_blocks
self.min_blocks = min_blocks
self.max_blocks = max_blocks
self.parallelism = parallelism
self.walltime = walltime
self.provisioned_blocks = 0
self.script_dir = script_dir
if not os.path.exists(self.script_dir):
os.makedirs(self.script_dir)
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
