def submit(self, command, blocksize, job_name="parsl.auto"):
"""Submit the command as a slurm job of blocksize parallel elements.
Parameters
----------
command : str
Command to be made on the remote side.
blocksize : int
Not implemented.
job_name : str
Name for the job (must be unique).
Returns
-------
None or str
If at capacity, returns None; otherwise, a string identifier for the job
"""
if self.provisioned_blocks >= self.max_blocks:
logger.warn("Slurm provider '{}' is at capacity (no more blocks will be added)".format(self.label))
return None
job_name = "{0}.{1}".format(job_name, time.time())
script_path = "{0}/{1}.submit".format(self.script_dir, job_name)
script_path = os.path.abspath(script_path)
logger.debug("Requesting one block with {} nodes".format(self.nodes_per_block))
job_config = {}
job_config["submit_script_dir"] = self.channel.script_dir
job_config["nodes"] = self.nodes_per_block
job_config["tasks_per_node"] = self.tasks_per_node
job_config["walltime"] = wtime_to_minutes(self.walltime)
job_config["overrides"] = self.overrides
job_config["partition"] = self.partition
job_config["user_script"] = command
# Wrap the command
job_config["user_script"] = self.launcher(command,
self.tasks_per_node,
self.nodes_per_block)
logger.debug("Writing submit script")
self._write_submit_script(template_string, script_path, job_name, job_config)
channel_script_path = self.channel.push_file(script_path, self.channel.script_dir)
retcode, stdout, stderr = super(SlurmProvider, self).execute_wait("sbatch {0}".format(channel_script_path))
job_id = None
if retcode == 0:
for line in stdout.split('\n'):
if line.startswith("Submitted batch job"):
job_id = line.split("Submitted batch job")[1].strip()
self.resources[job_id] = {'job_id': job_id, 'status': 'PENDING', 'blocksize': blocksize}
else:
print("Submission of command to scale_out failed")
logger.error("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(), stderr.strip())
return job_id
def __init__(self,
label,
channel,
script_dir,
nodes_per_block,
tasks_per_node,
init_blocks,
min_blocks,
max_blocks,
parallelism,
walltime,
launcher):
self._scaling_enabled = True
self.label = label
self.channel = channel
self.tasks_per_block = nodes_per_block * tasks_per_node
self.nodes_per_block = nodes_per_block
self.tasks_per_node = tasks_per_node
self.init_blocks = init_blocks
self.min_blocks = min_blocks
self.max_blocks = max_blocks
self.parallelism = parallelism
self.provisioned_blocks = 0
self.launcher = launcher
self.walltime = wtime_to_minutes(walltime)
self.script_dir = script_dir
if not os.path.exists(self.script_dir):
os.makedirs(self.script_dir)
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
def get_configs(self, command):
"""Compose a dictionary with information for writing the submit script."""
logger.debug(
"Requesting one block with {} nodes per block and {} tasks per node"
.format(self.nodes_per_block, self.tasks_per_node))
job_config = {}
job_config["submit_script_dir"] = self.channel.script_dir
job_config["nodes"] = self.nodes_per_block
job_config["walltime"] = wtime_to_minutes(self.walltime)
job_config["overrides"] = self.overrides
job_config["user_script"] = command
job_config["user_script"] = self.launcher(command, self.tasks_per_node,
self.nodes_per_block)
return job_config
def __init__(self,
channel,
label='cobalt',
script_dir='parsl_scripts',
launcher='single_node',
nodes_per_block=1,
tasks_per_node=1,
min_blocks=0,
max_blocks=10,
walltime="00:10:00",
account=None,
queue=None,
overrides=''):
if channel is None:
logger.error("Cobalt cannot be initialized without a channel")
raise (ep_error.ChannelRequired(
self.__class__.__name__,
"Missing a channel to execute commands"))
self.channel = channel
self.label = label
self.provisioned_blocks = 0
self.nodes_per_block = nodes_per_block
self.launcher = launcher
self.tasks_per_node = tasks_per_node
self.min_blocks = min_blocks
self.max_blocks = max_blocks
self.walltime = wtime_to_minutes(walltime)
self.account = account
self.overrides = overrides
self.script_dir = script_dir
if not os.path.exists(self.script_dir):
os.makedirs(self.script_dir)
# Dictionary that keeps track of jobs, keyed on job_id
self.resources = {}
if launcher in ['srun', 'srun_mpi']:
logger.warning(
"Use of {} launcher is usually appropriate for Slurm providers. "
"Recommended options include 'single_node' or 'aprun'.".format(
launcher))
def submit(self, command, blocksize, job_name="parsl.auto"):
""" Submits the command onto an Local Resource Manager job of blocksize parallel elements.
Submit returns an ID that corresponds to the task that was just submitted.
If tasks_per_node < 1 : ! This is illegal. tasks_per_node should be integer
If tasks_per_node == 1:
A single node is provisioned
If tasks_per_node > 1 :
tasks_per_node * blocksize number of nodes are provisioned.
Args:
- command :(String) Commandline invocation to be made on the remote side.
- blocksize :(float)
Kwargs:
- job_name (String): Name for job, must be unique
Returns:
- None: At capacity, cannot provision more
- job_id: (string) Identifier for the job
"""
if self.provisioned_blocks >= self.max_blocks:
logger.warn("[%s] at capacity, cannot add more blocks now",
self.label)
return None
# Note: Fix this later to avoid confusing behavior.
# We should always allocate blocks in integer counts of node_granularity
if blocksize < self.nodes_per_block:
blocksize = self.nodes_per_block
account_opt = '-A {}'.format(
self.account) if self.account is not None else ''
job_name = "parsl.{0}.{1}".format(job_name, time.time())
script_path = "{0}/{1}.submit".format(self.script_dir, job_name)
script_path = os.path.abspath(script_path)
job_config = {}
job_config["overrides"] = self.overrides
logger.debug(
"Requesting blocksize:%s nodes_per_block:%s tasks_per_node:%s",
blocksize, self.nodes_per_block, self.tasks_per_node)
# Wrap the command
job_config["user_script"] = self.launcher(command, self.tasks_per_node,
self.nodes_per_block)
queue_opt = '-q {}'.format(
self.queue) if self.queue is not None else ''
logger.debug("Writing submit script")
self._write_submit_script(template_string, script_path, job_name,
job_config)
channel_script_path = self.channel.push_file(script_path,
self.channel.script_dir)
command = 'qsub -n {0} {1} -t {2} {3} {4}'.format(
self.nodes_per_block, queue_opt, wtime_to_minutes(self.walltime),
account_opt, channel_script_path)
logger.debug("Executing {}".format(command))
retcode, stdout, stderr = super().execute_wait(command)
# TODO : FIX this block
if retcode != 0:
logger.error("Failed command: {0}".format(command))
logger.error("Launch failed stdout:\n{0} \nstderr:{1}\n".format(
stdout, stderr))
logger.debug("Retcode:%s STDOUT:%s STDERR:%s", retcode, stdout.strip(),
stderr.strip())
job_id = None
if retcode == 0:
# We should be getting only one line back
job_id = stdout.strip()
self.resources[job_id] = {
'job_id': job_id,
'status': 'PENDING',
'blocksize': blocksize
}
else:
logger.error(
"Submission of command to scale_out failed: {0}".format(
stderr))
raise (ep_error.ScaleOutFailed(
self.__class__,
"Request to submit job to local scheduler failed"))
logger.debug("Returning job id : {0}".format(job_id))
return job_id