def generate_input_json(self, processing):
try:
from idds.core import (catalog as core_catalog)
output_collection = self.get_output_collections()[0]
contents = core_catalog.get_contents_by_coll_id_status(
coll_id=output_collection.coll_id)
points = []
for content in contents:
# point = content['content_metadata']['point']
point = json.loads(content['path'])
points.append(point)
job_dir = self.get_working_dir(processing)
if 'input_json' in self.agent_attributes and self.agent_attributes[
'input_json']:
input_json = self.agent_attributes['input_json']
else:
input_json = 'idds_input.json'
opt_points = {'points': points, 'opt_space': self.opt_space}
with open(os.path.join(job_dir, input_json), 'w') as f:
json.dump(opt_points, f)
return input_json
except Exception as e:
raise Exception("Failed to generate idds inputs for HPO: %s" %
str(e))
def generate_new_contents(self, transform, input_collection,
output_collection, points):
if not isinstance(points, (tuple, list)):
points = [points]
avail_points = core_catalog.get_contents_by_coll_id_status(
coll_id=output_collection['coll_id'])
output_contents = []
i = len(avail_points)
for point in points:
content_metadata = {
'input_collection_id': input_collection['coll_id']
}
content = {
'coll_id': output_collection['coll_id'],
# 'scope': output_collection['scope'],
'scope': 'hpo',
'name': str(i),
'min_id': 0,
'max_id': 0,
'path': json.dumps((point, None)),
'status': ContentStatus.New,
'content_type': ContentType.PseudoContent,
'content_metadata': content_metadata
}
output_contents.append(content)
i += 1
return output_contents
def generate_transform_outputs(self, transform, collections):
self.logger.debug(
"Generating transform outputs: transform: %s, collections: %s" %
(transform, collections))
input_collection = None
output_collection = None
for collection in collections:
if collection['relation_type'] == CollectionRelationType.Input:
input_collection = collection
if collection['relation_type'] == CollectionRelationType.Output:
output_collection = collection
status = [ContentStatus.New, ContentStatus.Failed]
contents = core_catalog.get_contents_by_coll_id_status(
coll_id=input_collection['coll_id'], status=status)
output_contents = self.generate_transform_output_contents(
transform, input_collection, output_collection, contents)
self.logger.debug(
"Generating transform number of output contents: %s" %
len(output_contents))
to_cancel_processing = []
if transform['status'] == TransformStatus.Extend:
processings = core_processings.get_processings_by_transform_id(
transform['transform_id'])
for processing in processings:
to_cancel_processing.append(processing['processing_id'])
new_processing = None
if output_contents or transform['status'] == TransformStatus.Extend:
processing_metadata = {
'transform_id': transform['transform_id'],
'input_collection': input_collection['coll_id'],
'output_collection': output_collection['coll_id']
}
for key in transform['transform_metadata']:
processing_metadata[key] = transform['transform_metadata'][key]
new_processing = {
'transform_id': transform['transform_id'],
'status': ProcessingStatus.New,
'processing_metadata': processing_metadata
}
self.logger.debug("Generating transform output processing: %s" %
new_processing)
return {
'transform': transform,
'input_collection': input_collection,
'output_collection': output_collection,
'input_contents': contents,
'output_contents': output_contents,
'processing': new_processing,
'to_cancel_processing': to_cancel_processing
}
def __call__(self, processing, transform, input_collection,
output_collection):
try:
contents = core_catalog.get_contents_by_coll_id_status(
coll_id=input_collection['coll_id'])
files = []
for content in contents:
file = '%s:%s' % (content['scope'], content['name'])
files.append(file)
input_list = ','.join(files)
job_dir = self.get_job_dir(processing['processing_id'])
input_json = 'idds_input.json'
with open(os.path.join(job_dir, input_json), 'w') as f:
json.dump(files, f)
sandbox = None
if 'sandbox' in transform['transform_metadata']:
sandbox = transform['transform_metadata']['sandbox']
executable = transform['transform_metadata']['executable']
arguments = transform['transform_metadata']['arguments']
output_json = None
if 'output_json' in transform['transform_metadata']:
output_json = transform['transform_metadata']['output_json']
job_id, outputs = self.submit_job(processing['processing_id'],
sandbox, executable, arguments,
input_list, input_json,
output_json)
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = job_id
processing_metadata['submitter'] = self.name
if not job_id:
processing_metadata['submit_errors'] = outputs
else:
processing_metadata['submit_errors'] = None
ret = {
'processing_id':
processing['processing_id'],
'status':
ProcessingStatus.Submitted,
'next_poll_at':
datetime.datetime.utcnow() +
datetime.timedelta(seconds=self.poll_time_period),
'processing_metadata':
processing_metadata
}
return ret
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.AgentPluginError(
'%s: %s' % (str(ex), traceback.format_exc()))
def process_monitor_processing(self, processing):
transform_id = processing['transform_id']
processing_metadata = processing['processing_metadata']
input_coll_id = processing_metadata['input_collection']
input_collection = core_catalog.get_collection(coll_id=input_coll_id)
output_coll_id = processing_metadata['output_collection']
output_collection = core_catalog.get_collection(coll_id=output_coll_id)
output_contents = core_catalog.get_contents_by_coll_id_status(coll_id=output_coll_id)
transform = core_transforms.get_transform(transform_id)
ret_poll = self.poll_processing(processing, transform, input_collection, output_collection, output_contents)
if not ret_poll:
return {'processing_id': processing['processing_id'],
'locking': ProcessingLocking.Idle}
new_files = []
if 'new_files' in ret_poll:
new_files = ret_poll['new_files']
updated_files = ret_poll['updated_files']
file_msg = []
if updated_files:
file_msg = self.generate_file_message(transform, updated_files)
processing_parameters = {'status': ret_poll['processing_updates']['status'],
'locking': ProcessingLocking.Idle,
'processing_metadata': ret_poll['processing_updates']['processing_metadata']}
if 'output_metadata' in ret_poll['processing_updates']:
processing_parameters['output_metadata'] = ret_poll['processing_updates']['output_metadata']
updated_processing = {'processing_id': processing['processing_id'],
'parameters': processing_parameters}
ret = {'transform': transform,
'processing_updates': updated_processing,
'updated_files': updated_files,
'new_files': new_files,
'file_message': file_msg}
return ret
def __call__(self, processing, transform, input_collection,
output_collection):
try:
contents = core_catalog.get_contents_by_coll_id_status(
coll_id=output_collection['coll_id'])
points = []
unevaluated_points = 0
for content in contents:
point = content['content_metadata']['point']
points.append(point)
if not content['status'] == ContentStatus.Available:
unevaluated_points += 1
if unevaluated_points >= self.min_unevaluated_points:
# not submit the job
processing_metadata = processing['processing_metadata']
processing_metadata['unevaluated_points'] = unevaluated_points
ret = {
'processing_id': processing['processing_id'],
'status': ProcessingStatus.New,
'processing_metadata': processing_metadata
}
return ret
job_dir = self.get_job_dir(processing['processing_id'])
input_json = 'idds_input.json'
with open(os.path.join(job_dir, input_json), 'w') as f:
json.dump(points, f)
sandbox = None
if 'sandbox' in transform['transform_metadata']:
sandbox = transform['transform_metadata']['sandbox']
executable = transform['transform_metadata']['executable']
arguments = transform['transform_metadata']['arguments']
output_json = None
if 'output_json' in transform['transform_metadata']:
output_json = transform['transform_metadata']['output_json']
param_values = {
'NUM_POINTS': self.max_unevaluated_points - unevaluated_points,
'IN': 'input_json',
'OUT': output_json
}
executable = replace_parameters_with_values(
executable, param_values)
arguments = replace_parameters_with_values(arguments, param_values)
input_list = None
job_id, outputs = self.submit_job(processing['processing_id'],
sandbox, executable, arguments,
input_list, input_json,
output_json)
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = job_id
processing_metadata['submitter'] = self.name
if not job_id:
processing_metadata['submit_errors'] = outputs
else:
processing_metadata['submit_errors'] = None
ret = {
'processing_id': processing['processing_id'],
'status': ProcessingStatus.Submitted,
'processing_metadata': processing_metadata
}
return ret
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.AgentPluginError(
'%s: %s' % (str(ex), traceback.format_exc()))
def __call__(self, processing, transform, input_collection, output_collection):
try:
contents = core_catalog.get_contents_by_coll_id_status(coll_id=output_collection['coll_id'])
points = []
unevaluated_points = 0
for content in contents:
# point = content['content_metadata']['point']
point = json.loads(content['path'])
points.append(point)
if not content['status'] == ContentStatus.Available:
unevaluated_points += 1
job_dir = self.get_job_dir(processing['processing_id'])
input_json = 'idds_input.json'
opt_space = None
opt_points = {'points': points}
if 'opt_space' in transform['transform_metadata']:
opt_space = transform['transform_metadata']['opt_space']
opt_points['opt_space'] = opt_space
else:
opt_points['opt_space'] = None
with open(os.path.join(job_dir, input_json), 'w') as f:
json.dump(opt_points, f)
if 'method' in transform['transform_metadata'] and transform['transform_metadata']['method']:
status, errors, sandbox, executable, arguments, input_json, output_json, should_transfer_executable = self.get_executable_arguments_for_method(transform['transform_metadata'], input_json, unevaluated_points)
else:
status, errors, sandbox, executable, arguments, input_json, output_json, should_transfer_executable = self.get_executable_arguments_for_sandbox(transform['transform_metadata'], input_json, unevaluated_points)
if status != 0:
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = None
processing_metadata['submitter'] = self.name
processing_metadata['submit_errors'] = errors
processing_metadata['output_json'] = output_json
processing_metadata['max_points'] = self.get_max_points(transform['transform_metadata'])
# processing_metadata['job_dir'] = job_dir
ret = {'processing_id': processing['processing_id'],
'status': ProcessingStatus.Submitted,
'processing_metadata': processing_metadata}
else:
input_list = None
job_id, outputs = self.submit_job(processing['processing_id'], sandbox, executable, arguments, input_list, input_json, output_json, should_transfer_executable)
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = job_id
processing_metadata['submitter'] = self.name
processing_metadata['output_json'] = output_json
processing_metadata['max_points'] = self.get_max_points(transform['transform_metadata'])
# processing_metadata['job_dir'] = job_dir
if not job_id:
processing_metadata['submit_errors'] = outputs
else:
processing_metadata['submit_errors'] = None
ret = {'processing_id': processing['processing_id'],
'status': ProcessingStatus.Submitted,
'next_poll_at': datetime.datetime.utcnow() + datetime.timedelta(seconds=self.poll_time_period),
'processing_metadata': processing_metadata}
return ret
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.AgentPluginError('%s: %s' % (str(ex), traceback.format_exc()))
def __call__(self, processing, transform, input_collection,
output_collection):
try:
contents = core_catalog.get_contents_by_coll_id_status(
coll_id=output_collection['coll_id'])
points = []
unevaluated_points = 0
for content in contents:
# point = content['content_metadata']['point']
point = json.loads(content['path'])
points.append(point)
if not content['status'] == ContentStatus.Available:
unevaluated_points += 1
"""
if self.min_unevaluated_points and unevaluated_points >= self.min_unevaluated_points:
# not submit the job
processing_metadata = processing['processing_metadata']
processing_metadata['unevaluated_points'] = unevaluated_points
processing_metadata['not_submit'] = 'unevaluated_points(%s) > min_unevaluated_points(%s)' % (unevaluated_points, self.min_unevaluated_points)
self.logger.info("processing_id(%s) not submit currently because unevaluated_points(%s) >= min_unevaluated_points(%s)" % (processing['processing_id'], unevaluated_points, self.min_unevaluated_points))
ret = {'processing_id': processing['processing_id'],
'status': ProcessingStatus.New,
'processing_metadata': processing_metadata}
return ret
if 'not_submit' in processing_metadata:
del processing_metadata['not_submit']
"""
job_dir = self.get_job_dir(processing['processing_id'])
input_json = 'idds_input.json'
opt_space = None
opt_points = {'points': points}
if 'opt_space' in transform['transform_metadata']:
opt_space = transform['transform_metadata']['opt_space']
opt_points['opt_space'] = opt_space
else:
opt_points['opt_space'] = None
with open(os.path.join(job_dir, input_json), 'w') as f:
json.dump(opt_points, f)
if 'method' in transform['transform_metadata'] and transform[
'transform_metadata']['method']:
status, errors, sandbox, executable, arguments, input_json, output_json, should_transfer_executable = self.get_executable_arguments_for_method(
transform['transform_metadata'], input_json,
unevaluated_points)
else:
status, errors, sandbox, executable, arguments, input_json, output_json, should_transfer_executable = self.get_executable_arguments_for_sandbox(
transform['transform_metadata'], input_json,
unevaluated_points)
if status != 0:
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = None
processing_metadata['submitter'] = self.name
processing_metadata['submit_errors'] = errors
processing_metadata['output_json'] = output_json
ret = {
'processing_id': processing['processing_id'],
'status': ProcessingStatus.Submitted,
'processing_metadata': processing_metadata
}
else:
input_list = None
job_id, outputs = self.submit_job(processing['processing_id'],
sandbox, executable,
arguments, input_list,
input_json, output_json,
should_transfer_executable)
processing_metadata = processing['processing_metadata']
processing_metadata['job_id'] = job_id
processing_metadata['submitter'] = self.name
processing_metadata['output_json'] = output_json
if not job_id:
processing_metadata['submit_errors'] = outputs
else:
processing_metadata['submit_errors'] = None
ret = {
'processing_id': processing['processing_id'],
'status': ProcessingStatus.Submitted,
'processing_metadata': processing_metadata
}
return ret
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.AgentPluginError(
'%s: %s' % (str(ex), traceback.format_exc()))