def _load(file):
try:
with open(file, 'rt') as fd:
metadata = json.load(fd)
except OSError as e:
raise QMapError(str(e))
return metadata
def update(self):
"""
Update the status of the execution and submit UNSUBMITTED jobs
for execution if the number of RUN and PENDING jobs is lower than
the maximum permitted (which a parameter).
The submission of new jobs can be stopped/started using the
:meth:`terminate` method and :obj:`is_submission_enabled` flag.
"""
self.status.update()
unsubmitted_jobs = self.get_jobs(JobStatus.UNSUBMITTED)
if len(unsubmitted_jobs) > 0 and self.is_submission_enabled:
# Submit new jobs
running_and_pending = len(self.get_jobs(JobStatus.RUN)) + len(
self.get_jobs(JobStatus.PENDING))
to_run = self.max_running - running_and_pending
if to_run > 0:
ids_to_run = unsubmitted_jobs[:
to_run] # make a copy because the list is going to be altered
errors = []
for id_ in ids_to_run:
try:
job = self._jobs[id_]
job.run(self.job_params
) # job can change the params object
except QMapError:
errors.append(id_)
continue
if len(errors) > 0:
raise QMapError('Error running {}'.format(
', '.join(errors)))
def save(filename, metadata):
"""Save a dict into a file"""
file = '{}.{}'.format(filename, EXTENSION)
try:
with open(file, 'wt') as fd:
json.dump(metadata, fd, indent=4)
except OSError as e:
raise QMapError(str(e))
def save(filename, variables):
"""Create an env file from a dict"""
file = '{}.{}'.format(filename, EXTENSION)
try:
with open(file, 'wt') as fd:
for k, v in variables.items():
fd.write('#{}={}\n'.format(k, v))
fd.write('export QMAP_{}="{}"\n'.format(k.upper(), v))
except OSError as e:
raise QMapError(str(e))
def run(self, params):
"""
Ask the underlying cluster to put the job in the queue with some default params.
Those are overridden by the job specific _params
Args:
params: job default _params
Raises:
QMapError. If the job status is not UNSUBMITTED or the internal job id
is not None and if the submission command of the cluster fails.
"""
if self.executor_id is None and self.status == Status.UNSUBMITTED: # only jobs with the right status are allowed
params = params.copy() # do not modify the original object
params.update(self.params)
try:
if 'prefix' in params: # build the job name
params['name'] = '{}.{}'.format(params['prefix'],
self.executor_id)
self.executor_id, cmd = executor.run_job(self._f_script,
params,
out=self.f_stdout,
err=self.f_stderr)
except ExecutorError as e:
self.status = Status.FAILED
self._notify(Status.UNSUBMITTED)
self.save_metadata()
raise QMapError(
'Job {} cannot be submitted. Error message: '.format(
self.id, e))
else:
# Update metadata
self.metadata[Job.MD_JOB_ID] = self.executor_id
self.metadata[Job.MD_JOB_CMD] = cmd
# Change status & notify
self.status = Status.PENDING
self._notify(Status.UNSUBMITTED)
# Save metadata
self.save_metadata()
else: # job with invalid state for running
raise QMapError('Job {} cannot run due to incorrect status'.format(
self.id))
def execute_command(cmd):
"""Execute a shell command using subprocess"""
try:
# out = subprocess.run(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, shell=True)
out = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError:
raise QMapError('Error executing {}'.format(cmd))
else:
# return out.stdout.decode()
return out.decode()
def find_and_load(folder):
"""Find all metadata files in a folder"""
files = find(folder)
if len(files) == 0:
raise QMapError(
'No .{} files found in {}. Are you sure is a valid output BqQmap directory?'
.format(EXTENSION, path.abspath(folder)))
for file in files:
metadata = _load(file)
id_ = path.splitext(path.basename(file))[0]
yield id_, metadata
def load(filename):
"""Load a dict from an env file"""
file = '{}.{}'.format(filename, EXTENSION)
variables = {}
try:
with open(file, 'rt') as fd:
for line in fd:
if line.startswith('#'):
k, v = line.split('=', 1)
variables[k[1:].strip()] = v.strip()
except OSError as e:
raise QMapError(str(e))
return variables
def terminate(self): # Not recommended
"""
Cancel a job.
If your cluster supports cancelling for multiple jobs at once,
that option should be preferred.
"""
if Status not in [Status.FAILED, Status.DONE, Status.UNSUBMITTED]:
# only jobs with a valid status can be cancelled
executor.terminate_jobs([self.executor_id])
else:
raise QMapError(
'Job {} is not in a valid state for termination'.format(
self.executor_id))
def resubmit(self, parameters=None, **kwargs):
"""
Prepare the job for resubmission
Args:
parameters (Parameters): object with specific job parameters (replaces current)
kwargs: dict with new specific job parameters (updates current)
Raises:
BqQmapError. When the job status does not allow resubmission.
Valid status are FAILED and DONE
"""
if self.status not in [Status.FAILED, Status.DONE]:
# only job with certain status can be resubmitted
raise QMapError('Job {} is not in a valid state for re-run'.format(
self.id))
else:
if self.status != Status.UNSUBMITTED or self.executor_id is not None:
# Clean _params
self.executor_id = None
previous_status = self.status
self.status = Status.UNSUBMITTED
self.retries += 1
# Clean metadata
self.metadata = {
Job.MD_CMD: self.command,
Job.MD_RETRY: self.retries
}
# Reconfigure specific job parameters
if parameters is not None:
self.params = parameters
self.params.update(**kwargs)
if len(self.params) > 0:
env_file.save(self._f_env, self.params)
self.save_metadata()
# Remove files from previous execution
remove_file_if_exits(self.f_stdout)
remove_file_if_exits(self.f_stderr)
# Notify that its status has change (inform that is should be added to the unsubmitted queue)
self._notify(previous_status)
def filter_commands(folder, status_to_filter, only_jobs=False):
"""
Return a trimmed version of the original
input file
Args
folder (str): path to the output folder:
status_to_filter (list): status to filter by
Yield:
str. Line
"""
# TODO check if status is ALL possible status and simply return the input file (collapsed or not)
try:
map_file = get(folder)
except FileNotFoundError:
raise QMapError(
'No commands file found in {}. Are you sure is a valid output BqQmap directory?'
.format(folder))
sections = _parse(map_file)
group_size = metadata_file.load(
path.join(folder, EXECUTION_METADATA_FILE_NAME)).get('groups', None)
if group_size is None:
group_size = 1
commands = sections['jobs']
commands_searched_for = []
fmt = _get_name_fmt(len(commands))
for index, cmd in enumerate(
commands[pos:pos + group_size]
for pos in range(0, len(commands), group_size)):
metadata = metadata_file.load(
path.join(folder, fmt.format(index * group_size)))
if metadata[Job.MD_STATUS] in status_to_filter: # filter by status
commands_searched_for += cmd
if only_jobs:
sections = {'jobs': commands_searched_for}
else:
sections['jobs'] = commands_searched_for
return sections
def __init__(self, output_folder, force=False, max_running_jobs=None):
"""
Creates and execution from a existing output folder.
Args:
output_folder (str): path to a previous execution output folder
force (bool): try to load as many jobs as possible regardless of loading errors
max_running_jobs (int): maximum jobs that can be submitted to the cluster at once.
Each job is identified by each script file.
If no jobs can be loaded a QMapError is raised.
"""
super().__init__(output_folder)
# Load metadata
metadata = metadata_file.load(self._f_metadata)
profile_conf = metadata[Manager.MD_PROFILE]
profile_conf['params'] = env_file.load(self._f_env)
self._profile = Profile(profile_conf)
self.max_running = metadata[
Manager.
MD_MAX_RUNNING] if max_running_jobs is None else max_running_jobs
self._group_size = metadata.get(Manager.MD_GROUP_SIZE, None)
# Load jobs
self._jobs = OrderedDict()
try:
self.__load_execution()
except QMapError as e:
if force:
logger.warning(e)
else:
raise e
self.status = Status(self._jobs)
if len(self._jobs) == 0:
raise QMapError('No jobs found in folder {}'.format(output_folder))
self.is_submission_enabled = False
self.update()
def __load_execution(self):
ids = []
for file in metadata_file.find(self.out_folder):
file_name = path.splitext(path.basename(file))[0]
if not file_name == EXECUTION_METADATA_FILE_NAME:
ids.append(file_name)
corrupt_jobs = []
for id_ in sorted(ids):
try:
self._jobs[id_] = ReattachedJob(id_, self.out_folder)
except QMapError:
corrupt_jobs.append(id_)
continue
if len(corrupt_jobs) > 0:
raise QMapError('Error loading the following jobs: {}'.format(
', '.join(corrupt_jobs)))
def __init__(self,
input_file,
output_folder,
profile_conf,
max_running_jobs=None,
group_size=None,
cli_params=None):
"""
Use a jobs file to create a set of jobs for submission
Args:
input_file (str): path to file with the commands (see :func:`~qmap.file.jobs`).
output_folder (str): path where to save the job related files. It must be empty.
profile_conf (:class:`~qmap.profile.Profile`): profile configuration
max_running_jobs (int): maximum jobs that can be submitted to the cluster at once.
Defaults to all.
group_size (int): number of commands to group under the same job
cli_params (:class:`~qmap.parameters.Parameters`): configuration for the jobs received from command line
The input file is copied to the output_folder (and renamed).
"""
super().__init__(output_folder)
try:
os.makedirs(self.out_folder)
except OSError:
if os.listdir(self.out_folder): # directory not empty
raise QMapError(
'Output folder [{}] is not empty. '
'Please give a different folder to write the output files.'
.format(self.out_folder))
self._profile = profile_conf
self._group_size = group_size
self.__load_input(input_file, cli_params)
self.max_running = len(
self._jobs) if max_running_jobs is None else max_running_jobs
self._save_metadata()
self._save_environment()
self.status = Status(self._jobs)
self.update()
def resubmit_failed(self, **kwargs):
"""
Resubmit all jobs that have FAILED
Raises:
QMapError. When any job cannot be resubmitted
"""
errors = []
ids_to_resubmit = self.status.groups[
JobStatus.
FAILED][:] # make a copy because the list is going to be altered
for id_ in ids_to_resubmit:
try:
self._jobs[id_].resubmit(**kwargs)
except QMapError:
errors.append(id_)
continue
if len(errors) > 0:
raise QMapError('Error resubmitting {}'.format(', '.join(errors)))
def close(self):
"""
Save metadata of all jobs and the manager itself.
This is a method to be called before closing the manager
Raises:
QMapError.
"""
errors = []
for id_, job in self._jobs.items():
try:
job.save_metadata()
except QMapError:
errors.append(id_)
continue
if len(errors) > 0:
raise QMapError('Error saving metadata of {}'.format(
', '.join(errors)))
self._save_metadata() # The max running jobs might have changed
def parse(status):
"""
Parse status
Args:
status (list):
Returns:
list. Valid status to filter the metadata files.
"""
stat_fields = set()
for stat in status:
if stat == 'all' or stat == 'a':
stat_fields.update(_job_status)
elif stat in _job_status:
stat_fields.add(stat)
elif stat in _job_status_short:
stat_fields.add(_job_status[_job_status_short.index(stat)])
else:
raise QMapError('Invalid option for status: {}'.format(stat))
return [s.upper() for s in stat_fields]
def write(file, iterable, sep=None):
"""
Write anything on the iterable to a file.
Args:
file: file path or STDOUT if None
iterable: values to write
sep (string, optional): split values with this separator. Provide only if the iterable returns multiple items each time
"""
if file and path.isfile(file):
raise QMapError(
'File {} already exist. Please, provide a different file name'.
format(file))
with file_open(file, 'w') as fd:
for v in iterable:
if sep is None:
fd.write(v)
fd.write('\n')
else:
fd.write(sep.join(v))
fd.write('\n')
def __load_input(self, in_file, cli_params=None):
pre, job, post, general_parameters = jobs_file.parse(in_file)
if len(job) > int(self._profile.get(
'max_ungrouped',
len(job) + 1)) and self._group_size is None:
raise QMapError(
'To submit more than {} jobs, please specify the group parameter.'
'This parameter indicate the size of each group.'
'For small jobs, the bigger the group the better.'
'Please, note that the job specific _params will be ignored'.
format(self._profile['max_ungrouped']))
job_parameters = self._profile.parameters # default _params
job_parameters.update(general_parameters) # global input file _params
if cli_params is not None:
job_parameters.update(cli_params) # execution command line _params
job_list = []
if self._group_size is None or self._group_size == 1:
for i, c in job.items():
cmd = c.split('##')
params = None
if len(cmd) > 1: # if the job has specific _params
params = jobs_file.parse_inline_parameters(
cmd[1]) # job specific _params
job_list.append((i,
SubmittedJob(i,
self.out_folder,
cmd[0].strip(),
params,
pre_commands=pre,
post_commands=post)))
else:
logger.warning("Specific job execution _params ignored")
cmds_in_group = []
group_name = None
cmds_counter = 0
for i, c in job.items():
if group_name is None:
group_name = i
command = c.split('##')[0].strip()
cmds_in_group.append(command)
cmds_counter += 1
if cmds_counter >= self._group_size:
job_list.append((group_name,
SubmittedJob(group_name,
self.out_folder,
cmds_in_group,
None,
pre_commands=pre,
post_commands=post)))
group_name = None
cmds_in_group = []
cmds_counter = 0
else:
if len(cmds_in_group
) > 0: # in case there is a remaining group
job_list.append((group_name,
SubmittedJob(group_name,
self.out_folder,
cmds_in_group,
None,
pre_commands=pre,
post_commands=post)))
self._jobs = OrderedDict(job_list)
jobs_file.save(in_file, self.out_folder)