def extend_request(request_id=None,
workload_id=None,
lifetime=30,
session=None):
"""
extend an request's lifetime.
:param request_id: The id of the request.
:param workload_id: The workload_id of the request.
:param lifetime: The life time as umber of days.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
"""
try:
if not request_id and workload_id:
request_ids = get_request_ids_by_workload_id(workload_id)
if request_ids and len(request_ids) > 1:
raise exceptions.IDDSException(
"More than one request with the same workload_id")
request_id = request_ids[0]
req_update = "update atlas_idds.requests set expired_at=:expired_at where request_id=:request_id"
req_stmt = text(req_update)
session.execute(
req_stmt, {
'expired_at':
datetime.datetime.utcnow() + datetime.timedelta(days=lifetime),
'request_id':
request_id
})
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject('Request %s cannot be found: %s' %
(request_id, error))
def get_input_contents(self):
"""
Get all input contents from DDM.
"""
try:
ret_files = []
coll = self.collections[self._primary_input_collection]
ret_file = {
'coll_id': coll['coll_id'],
'scope': coll['scope'],
'name': coll['name'],
'bytes': coll.coll_metadata['bytes'],
'adler32': None,
'min_id': 0,
'max_id': coll.coll_metadata['total_files'],
'content_type': ContentType.File,
'content_metadata': {
'total_files': coll['coll_metadata']['total_files']
}
}
ret_files.append(ret_file)
return ret_files
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def get_input_contents(self):
"""
Get all input contents from DDM.
"""
try:
ret_files = []
rucio_client = self.get_rucio_client()
files = rucio_client.list_files(scope=self.collections[self._primary_input_collection].scope,
name=self.collections[self._primary_input_collection].name)
for file in files:
ret_file = {'coll_id': self.collections[self._primary_input_collection].coll_id,
'scope': file['scope'],
'name': file['name'],
'bytes': file['bytes'],
'adler32': file['adler32'],
'min_id': 0,
'max_id': file['events'],
'content_type': ContentType.File,
'content_metadata': {'events': file['events']}}
ret_files.append(ret_file)
return ret_files
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_external_collection(self, coll):
try:
if 'coll_metadata' in coll and 'is_open' in coll[
'coll_metadata'] and not coll['coll_metadata']['is_open']:
return coll
else:
meta = {}
did_meta = self.client.get_metadata(scope=coll['scope'],
name=coll['name'])
meta = {
'scope': coll['scope'],
'name': coll['name'],
'coll_metadata': {
'bytes': did_meta['bytes'],
'total_files': did_meta['length'],
'availability': did_meta['availability'],
'events': did_meta['events'],
'is_open': did_meta['is_open'],
'run_number': did_meta['run_number'],
'did_type': did_meta['did_type'],
'list_all_files': False
}
}
return meta
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def get_hyperparameters(self, workload_id, request_id, id=None, status=None, limit=None):
"""
Get hyperparameters from the Head service.
:param workload_id: the workload id.
:param request_id: the request id.
:param status: the status of the hyperparameters.
:param limit: limit number of hyperparameters
:raise exceptions if it's not got successfully.
"""
path = self.HPO_BASEURL
if not workload_id:
workload_id = 'null'
if not request_id:
request_id = 'null'
if id is None:
id = 'null'
if status is None:
status = 'null'
if limit is None:
limit = 'null'
if workload_id == 'null' and request_id == 'null':
raise exceptions.IDDSException("One of workload_id and request_id should not be None or empty")
url = self.build_url(self.host, path=os.path.join(path, str(workload_id), str(request_id), str(id), str(status), str(limit)))
params = self.get_request_response(url, type='GET')
return params
def poll_external_collection(self, coll):
try:
# if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']:
if 'status' in coll and coll['status'] in [CollectionStatus.Closed]:
return coll
else:
# client = self.get_rucio_client()
# did_meta = client.get_metadata(scope=coll['scope'], name=coll['name'])
if 'coll_metadata' not in coll:
coll['coll_metadata'] = {}
coll['coll_metadata']['bytes'] = 1
# coll['coll_metadata']['total_files'] = 1
coll['coll_metadata']['availability'] = 1
coll['coll_metadata']['events'] = 1
coll['coll_metadata']['is_open'] = True
coll['coll_metadata']['run_number'] = 1
coll['coll_metadata']['did_type'] = 'DATASET'
coll['coll_metadata']['list_all_files'] = False
if 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']:
coll_status = CollectionStatus.Closed
else:
coll_status = CollectionStatus.Open
coll['status'] = coll_status
coll['coll_type'] = CollectionType.Dataset
return coll
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_external_collection(self, coll):
try:
if coll.status in [CollectionStatus.Closed]:
return coll
else:
coll.coll_metadata['bytes'] = 1
coll.coll_metadata['availability'] = 1
coll.coll_metadata['events'] = 1
coll.coll_metadata['is_open'] = True
coll.coll_metadata['run_number'] = 1
coll.coll_metadata['did_type'] = 'DATASET'
coll.coll_metadata['list_all_files'] = False
# if (not self.dependency_map_deleted and not self.dependency_map):
if not self.has_new_inputs:
coll.coll_metadata['is_open'] = False
if 'is_open' in coll.coll_metadata and not coll.coll_metadata[
'is_open']:
coll_status = CollectionStatus.Closed
else:
coll_status = CollectionStatus.Open
coll.status = coll_status
coll.coll_metadata['coll_type'] = CollectionType.Dataset
return coll
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def delete_request(request_id=None, workload_id=None, session=None):
"""
delete an request.
:param request_id: The id of the request.
:param workload_id: The workload_id of the request.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:raises DatabaseException: If there is a database error.
"""
try:
if not request_id and workload_id:
request_ids = get_request_ids_by_workload_id(workload_id)
if request_ids and len(request_ids) > 1:
raise exceptions.IDDSException(
"More than one request with the same workload_id")
request_id = request_ids[0]
req2workload_delete = "delete from atlas_idds.req2workload where request_id=:request_id"
req2workload_stmt = text(req2workload_delete)
session.execute(req2workload_stmt, {'request_id': request_id})
req_delete = "delete from atlas_idds.requests where request_id=:request_id"
req_stmt = text(req_delete)
session.execute(req_stmt, {'request_id': request_id})
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject('Request %s cannot be found: %s' %
(request_id, error))
def update_hyperparameter(self, workload_id, request_id, id, loss):
"""
Update hyperparameter to the Head service.
:param workload_id: the workload id.
:param request_id: the request.
:param id: id of the hyper parameter.
:param loss: the loss.
:raise exceptions if it's not updated successfully.
"""
if not workload_id:
workload_id = 'null'
if not request_id:
request_id = 'null'
if workload_id == 'null' and request_id == 'null':
raise exceptions.IDDSException(
"One of workload_id and request_id should not be None or empty"
)
path = self.HPO_BASEURL
url = self.build_url(self.host,
path=os.path.join(path, str(workload_id),
str(request_id), str(id),
str(loss)))
r = self.get_request_response(url, type='PUT')
return r
def get_rucio_client(self):
try:
client = RucioClient()
except RucioCannotAuthenticate as error:
self.logger.error(error)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc()))
return client
def kill_processing(self, processing):
try:
if processing:
from pandatools import Client
task_id = processing.workload_id
Client.killTask(task_id)
except Exception as ex:
msg = "Failed to check the processing (%s) status: %s" % (str(
processing['processing_id']), str(ex))
raise exceptions.IDDSException(msg)
def get_rucio_download_client(self):
try:
from rucio.client.downloadclient import DownloadClient
client = DownloadClient()
except RucioCannotAuthenticate as error:
self.logger.error(error)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException(
'%s: %s' % (str(error), traceback.format_exc()))
return client
def get_input_contents(self):
"""
Get all input contents from DDM.
"""
try:
ret_files = []
return ret_files
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_external_collection(self, coll):
try:
if coll.status in [CollectionStatus.Closed]:
if not self.terminated:
self.logger.info(
"Work is not terminated, reopen collection")
coll.coll_metadata['is_open'] = True
coll.status = CollectionStatus.Open
return coll
else:
coll.coll_metadata['bytes'] = 0
coll.coll_metadata['total_files'] = 0
coll.coll_metadata['availability'] = True
coll.coll_metadata['events'] = 0
coll.coll_metadata['is_open'] = True
coll.coll_metadata['run_number'] = None
coll.coll_metadata['did_type'] = 'DATASET'
coll.coll_metadata['list_all_files'] = False
if self.terminated:
self.logger.info(
"Work is terminated. Closing input dataset.")
coll.coll_metadata['is_open'] = False
if self.points_to_generate <= 0:
self.logger.info(
"points_to_generate(%s) is equal or smaller than 0. Closing input dataset."
% self.points_to_generate)
coll.coll_metadata['is_open'] = False
if 'is_open' in coll.coll_metadata and not coll.coll_metadata[
'is_open']:
coll_status = CollectionStatus.Closed
else:
coll_status = CollectionStatus.Open
coll.status = coll_status
if 'did_type' in coll.coll_metadata:
if coll.coll_metadata['did_type'] == 'DATASET':
coll_type = CollectionType.Dataset
elif coll.coll_metadata['did_type'] == 'CONTAINER':
coll_type = CollectionType.Container
else:
coll_type = CollectionType.File
else:
coll_type = CollectionType.Dataset
coll.coll_metadata['coll_type'] = coll_type
return coll
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def kill_processing_force(self, processing):
try:
if processing:
from pandaclient import Client
proc = processing['processing_metadata']['processing']
task_id = proc.workload_id
# task_id = processing['processing_metadata']['task_id']
Client.killTask(task_id)
# Client.finishTask(task_id, soft=True)
except Exception as ex:
msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex))
raise exceptions.IDDSException(msg)
def parse_output_file(self, pfn):
try:
if not os.path.exists(pfn):
self.logger.warn("%s doesn't exist" % pfn)
return {}
else:
with open(pfn, 'r') as f:
data = f.read()
outputs = json.loads(data)
return outputs
except Exception as ex:
# self.logger.error(ex)
# self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
return {}
def reactivate_processing(self, processing):
try:
if processing:
from pandaclient import Client
# task_id = processing['processing_metadata']['task_id']
proc = processing['processing_metadata']['processing']
task_id = proc.workload_id
# Client.retryTask(task_id)
status, out = Client.retryTask(task_id, newParams={})
self.logger.warn("Retry processing(%s) with task id(%s): %s, %s" % (processing['processing_id'], task_id, status, out))
# Client.reactivateTask(task_id)
# Client.resumeTask(task_id)
except Exception as ex:
msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex))
raise exceptions.IDDSException(msg)
def get_input_contents(self, point_index=1):
"""
Get all input contents from DDM.
"""
try:
if self.terminated:
return []
if self.unfinished_points > 0:
return []
ret_files = []
coll = self.collections[self._primary_input_collection]
if self.max_points and (self.max_points - self.finished_points <
self.num_points_per_iteration):
self.points_to_generate = self.max_points - self.finished_points
# call external processing to generate points
points = self.generate_points()
self.logger.info("points generated: %s" % str(points))
loss = None
for point in points:
ret_file = {
'coll_id': coll.coll_id,
'scope': coll.scope,
'name': str(point_index),
'bytes': 0,
'adler32': None,
'min_id': 0,
'max_id': 0,
'path': json.dumps((point, loss)),
'content_type': ContentType.File,
'content_metadata': {
'events': 0
}
}
ret_files.append(ret_file)
point_index += 1
return ret_files
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def poll_panda_task(self, processing=None, task_name=None):
try:
if not self.pandamonitor:
self.pandamonitor = self.load_panda_monitor()
self.logger.info("panda server: %s" % self.pandamonitor)
task_id = None
task_info = None
if processing:
task_id = processing['processing_metadata']['task_id']
task_url = self.pandamonitor + '/task/?json&jeditaskid=' + str(task_id)
task_info = self.download_payload_json(task_url)
elif task_name:
task_url = self.pandamonitor + '/tasks/?taskname=' + str(task_name) + "&json"
self.logger.debug("poll_panda_task, task_url: %s" % str(task_url))
self.logger.debug("poll_panda_task, task_url: %s" % str(self.download_payload_json(task_url)))
task_json = self.download_payload_json(task_url)
if len(task_json) > 0:
task_info = task_json[0]
else:
return "No status", {}
if not task_id:
task_id = task_info.get('jeditaskid', None)
if not task_id:
return "No status", {}
jobs_url = self.pandamonitor + '/jobs/?json&datasets=yes&jeditaskid=' + str(task_id)
jobs_list = self.download_payload_json(jobs_url)
outputs_status = {}
for job_info in jobs_list['jobs']:
if 'jobstatus' in job_info and 'datasets' in job_info and len(job_info['datasets']) > 0:
output_index = job_info['datasets'][0]['lfn'].split(':')[1]
status = self.jobs_to_idd_ds_status(job_info['jobstatus'])
outputs_status[output_index] = status
task_status = None
self.logger.debug("poll_panda_task, task_info: %s" % str(task_info))
if task_info.get("task", None) is not None:
task_status = task_info["task"]["status"]
return task_status, outputs_status
except Exception as ex:
msg = "Failed to check the panda task(%s) status: %s" % (str(task_id), str(ex))
raise exceptions.IDDSException(msg)
def poll_external_collection(self, coll):
try:
if coll.status in [CollectionStatus.Closed]:
return coll
else:
client = self.get_rucio_client()
did_meta = client.get_metadata(scope=coll.scope,
name=coll.name)
coll.coll_metadata['bytes'] = did_meta['bytes']
coll.coll_metadata['total_files'] = did_meta['length']
coll.coll_metadata['availability'] = did_meta['availability']
coll.coll_metadata['events'] = did_meta['events']
coll.coll_metadata['is_open'] = did_meta['is_open']
coll.coll_metadata['run_number'] = did_meta['run_number']
coll.coll_metadata['did_type'] = did_meta['did_type']
coll.coll_metadata['list_all_files'] = False
if (('is_open' in coll.coll_metadata
and not coll.coll_metadata['is_open']) or
('force_close' in coll.coll_metadata
and coll.coll_metadata['force_close'])): # noqa: W503
coll_status = CollectionStatus.Closed
else:
coll_status = CollectionStatus.Open
coll.status = coll_status
if 'did_type' in coll.coll_metadata:
if coll.coll_metadata['did_type'] == 'DATASET':
coll_type = CollectionType.Dataset
elif coll.coll_metadata['did_type'] == 'CONTAINER':
coll_type = CollectionType.Container
else:
coll_type = CollectionType.File
else:
coll_type = CollectionType.Dataset
coll.coll_metadata['coll_type'] = coll_type
return coll
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def submit_panda_task(self, processing):
try:
from pandatools import Client
status, tmpOut = Client.insertTaskParams(self.panda_task_paramsmap,
False, True)
if status == 0:
tmp_status, tmp_output = tmpOut
m = re.search("jediTaskID=(\d+)", tmp_output) # noqa W605
task_id = int(m.group(1))
processing.workload_id = task_id
else:
self.add_errors(tmpOut)
raise Exception(tmpOut)
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def get_request(request_id=None, workload_id=None, session=None):
"""
Get a request or raise a NoObject exception.
:param request_id: The id of the request.
:param workload_id: The workload_id of the request.
:param session: The database session in use.
:raises NoObject: If no request is founded.
:returns: Request.
"""
try:
if not request_id and workload_id:
request_ids = get_request_ids_by_workload_id(workload_id)
if request_ids and len(request_ids) > 1:
raise exceptions.IDDSException(
"More than one request with the same workload_id")
request_id = request_ids[0]
req_select = """select request_id, scope, name, requester, request_type, transform_tag, priority,
status, locking, workload_id, created_at, updated_at, accessed_at, expired_at, errors,
request_metadata, processing_metadata
from atlas_idds.requests where request_id=:request_id
"""
req_stmt = text(req_select)
result = session.execute(req_stmt, {'request_id': request_id})
request = result.fetchone()
if request is None:
raise exceptions.NoObject(
'request request_id: %s, workload_id: %s cannot be found' %
(request_id, workload_id))
request = convert_request_to_dict(request)
return request
except sqlalchemy.orm.exc.NoResultFound as error:
raise exceptions.NoObject(
'request request_id: %s, workload_id: %s cannot be found: %s' %
(request_id, workload_id, error))
def get(self, vo, auth_type='oidc'):
""" Get sign url for user to approve.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: dictionary with sign url.
"""
try:
if auth_type == 'oidc':
oidc = OIDCAuthentication()
status, sign_url = oidc.get_oidc_sign_url(vo)
if status:
rets = sign_url
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data=rets)
else:
raise exceptions.IDDSException(
"Failed to get oidc sign url: %s" % str(sign_url))
else:
raise exceptions.AuthenticationNotSupported(
"auth_type %s is not supported to call this function." %
str(auth_type))
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(traceback.format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
def poll_external_collection(self, coll):
try:
# if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']:
if coll.status in [CollectionStatus.Closed]:
return coll
else:
# client = self.get_rucio_client()
# did_meta = client.get_metadata(scope=coll['scope'], name=coll['name'])
coll.coll_metadata['bytes'] = 0
coll.coll_metadata['total_files'] = 0
coll.coll_metadata['availability'] = True
coll.coll_metadata['events'] = 0
coll.coll_metadata['is_open'] = False
coll.coll_metadata['run_number'] = None
coll.coll_metadata['did_type'] = 'DATASET'
coll.coll_metadata['list_all_files'] = False
if 'is_open' in coll.coll_metadata and not coll.coll_metadata[
'is_open']:
coll_status = CollectionStatus.Closed
else:
coll_status = CollectionStatus.Open
coll.status = coll_status
if 'did_type' in coll.coll_metadata:
if coll.coll_metadata['did_type'] == 'DATASET':
coll_type = CollectionType.Dataset
elif coll.coll_metadata['did_type'] == 'CONTAINER':
coll_type = CollectionType.Container
else:
coll_type = CollectionType.File
else:
coll_type = CollectionType.Dataset
coll.coll_metadata['coll_type'] = coll_type
return coll
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' %
(str(ex), traceback.format_exc()))
def get(self, vo, device_code, interval=5, expires_in=60):
""" Get id token.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: dictionary with sign url.
"""
try:
oidc = OIDCAuthentication()
status, id_token = oidc.get_id_token(vo, device_code, interval,
expires_in)
if status:
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data=id_token)
else:
if 'error' in id_token and 'authorization_pending' in id_token[
'error']:
raise exceptions.AuthenticationPending(str(id_token))
else:
raise exceptions.IDDSException(
"Failed to get oidc token: %s" % str(id_token))
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(traceback.format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
def post(self, vo):
""" Refresh the token.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
500 Internal Error
"""
try:
parameters = self.get_request().data and json.loads(
self.get_request().data)
refresh_token = parameters['refresh_token']
oidc = OIDCAuthentication()
status, id_token = oidc.refresh_id_token(vo, refresh_token)
if status:
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data=id_token)
else:
raise exceptions.IDDSException(
"Failed to refresh oidc token: %s" % str(id_token))
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(traceback.format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
def get_input_contents(self):
"""
Get all input contents from DDM.
"""
try:
files = self.check_dependencies()
ret_files = []
coll = self.collections[self._primary_input_collection]
for file in files:
ret_file = {'coll_id': coll['coll_id'],
'scope': coll['scope'],
'name': file, # or a different file name from the dataset name
'bytes': 1,
'adler32': '12345678',
'min_id': 0,
'max_id': 1,
'content_type': ContentType.File,
'content_metadata': {'events': 1}} # here events is all events for eventservice, not used here.
ret_files.append(ret_file)
return ret_files
except Exception as ex:
self.logger.error(ex)
self.logger.error(traceback.format_exc())
raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def get_request_response(self, url, type='GET', data=None, headers=None):
"""
Send request to the IDDS server and get the response.
:param url: http url to connection.
:param type: request type(GET, PUT, POST, DEL).
:param data: data to be sent to the IDDS server.
:param headers: http headers.
:returns: response data as json.
:raises:
"""
result = None
for retry in range(self.retries):
try:
if type == 'GET':
result = self.session.get(url, cert=(self.client_proxy, self.client_proxy), timeout=self.timeout, headers=headers, verify=False)
elif type == 'PUT':
result = self.session.put(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False)
elif type == 'POST':
result = self.session.post(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False)
elif type == 'DEL':
result = self.session.delete(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False)
else:
return
except requests.exceptions.ConnectionError as error:
logging.warning('ConnectionError: ' + str(error))
if retry >= self.retries - 1:
raise exceptions.ConnectionException('ConnectionError: ' + str(error))
if result is not None:
# print(result.text)
if result.status_code in [HTTP_STATUS_CODE.BadRequest,
HTTP_STATUS_CODE.Unauthorized,
HTTP_STATUS_CODE.Forbidden,
HTTP_STATUS_CODE.NotFound,
HTTP_STATUS_CODE.NoMethod,
HTTP_STATUS_CODE.InternalError]:
raise exceptions.IDDSException(result.text)
elif result.status_code == HTTP_STATUS_CODE.OK:
# print(result.text)
if result.text:
return json_loads(result.text)
else:
return None
else:
try:
if result.headers and 'ExceptionClass' in result.headers:
cls = getattr(exceptions, result.headers['ExceptionClass'])
msg = result.headers['ExceptionMessage']
raise cls(msg)
else:
if result.text:
data = json_loads(result.text)
raise exceptions.IDDSException(**data)
else:
raise exceptions.IDDSException("Unknow exception: %s" % (result.text))
except AttributeError:
raise exceptions.IDDSException(result.text)
if result is None:
raise exceptions.IDDSException('Response is None')
def __init__(
self,
executable=None,
arguments=None,
parameters=None,
setup=None,
work_tag='hpo',
exec_type='local',
sandbox=None,
work_id=None,
name=None,
# primary_input_collection=None, other_input_collections=None,
# output_collections=None, log_collections=None,
logger=None,
workload_id=None,
agent_attributes=None,
method=None,
container_workdir=None,
output_json=None,
opt_space=None,
initial_points=None,
max_points=None,
num_points_per_iteration=10):
"""
Init a work/task/transformation.
:param setup: A string to setup the executable enviroment, it can be None.
:param executable: The executable.
:param arguments: The arguments.
:param parameters: A dict with arguments needed to be replaced.
:param work_type: The work type like data carousel, hyperparameteroptimization and so on.
:param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on.
:param sandbox: The sandbox.
:param work_id: The work/task id.
:param primary_input_collection: The primary input collection.
:param other_input_collections: List of the input collections.
:param output_collections: List of the output collections.
# :param workflow: The workflow the current work belongs to.
:param method: The HPO methd to use. It can be 'nevergrad', 'container' or 'sandbox'.
:param sandbox: The sandbox to be uploaded or the container path.
:param executable: The executable command.
:param arguments: The arguments for the executable.
:param container_workdir: The working directory for container.
:param opt_space: The optimization space.
:param initial_points: The initial points.
:param max_points: The maximum number of points.
:param number_points_per_iteration: The number of points to be generated per iteration.
"""
if not name:
if workload_id:
name = 'hpo.' + str(
workload_id) + "." + datetime.datetime.utcnow().strftime(
"%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000))
else:
name = 'hpo.' + datetime.datetime.utcnow().strftime(
"%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000))
primary_input_collection = {'scope': 'HPO', 'name': name}
other_input_collections = None
output_collections = [{'scope': 'HPO', 'name': name + 'output'}]
log_collections = None
super(ATLASHPOWork,
self).__init__(executable=executable,
arguments=arguments,
parameters=parameters,
setup=setup,
work_type=TransformType.HyperParameterOpt,
exec_type=exec_type,
sandbox=sandbox,
work_id=work_id,
primary_input_collection=primary_input_collection,
other_input_collections=other_input_collections,
output_collections=output_collections,
log_collections=log_collections,
logger=logger,
agent_attributes=agent_attributes)
self.method = method
self.sandbox = sandbox
self.executable = executable
self.arguments = arguments
self.container_workdir = container_workdir
self.opt_space = opt_space
self.initial_points = initial_points
self.max_points = max_points
self.num_points_per_iteration = num_points_per_iteration
self.unfinished_points = 0
self.input_json = None
self.output_json = output_json
self.finished_points = 0
self.points_to_generate = self.num_points_per_iteration
self.point_index = 0
self.terminated = False
self.polling_retries = 0
if not self.num_points_per_iteration or self.num_points_per_iteration < 0:
raise exceptions.IDDSException(
"num_points_per_iteration must be integer bigger than 0")
self.num_points_per_iteration = int(self.num_points_per_iteration)
if not self.method and self.executable and 'docker' in self.executable:
self.method = 'docker'
# if self.agent_attributes and 'atlashpowork' in self.agent_attributes:
# self.agent_attributes = self.agent_attributes['atlashpowork']
# self.logger.info("agent_attributes: %s" % self.agent_attributes)
# if self.agent_attributes and 'workdir' in self.agent_attributes and self.agent_attributes['workdir']:
# self.set_workdir(self.agent_attributes['workdir'])
# self.logger.info("workdir: %s" % self.get_workdir())
if agent_attributes:
self.set_agent_attributes(agent_attributes)
def get_request_response(self,
url,
type='GET',
data=None,
headers=None,
auth_setup_step=False):
"""
Send request to the IDDS server and get the response.
:param url: http url to connection.
:param type: request type(GET, PUT, POST, DEL).
:param data: data to be sent to the IDDS server.
:param headers: http headers.
:returns: response data as json.
:raises:
"""
result = None
if not headers:
headers = {}
headers['X-IDDS-Auth-Type'] = self.auth_type
headers['X-IDDS-Auth-VO'] = self.vo
for retry in range(self.retries):
try:
if self.auth_type in ['x509_proxy']:
if type == 'GET':
result = self.session.get(url,
cert=(self.client_proxy,
self.client_proxy),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
cert=(self.client_proxy,
self.client_proxy),
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
cert=(self.client_proxy,
self.client_proxy),
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
cert=(self.client_proxy,
self.client_proxy),
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
elif self.auth_type in ['oidc']:
if auth_setup_step:
if type == 'GET':
result = self.session.get(url,
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
else:
oidc_utils = OIDCAuthenticationUtils()
status, token = oidc_utils.load_token(self.oidc_token)
if not status:
raise exceptions.IDDSException(
"Token %s cannot be loaded: %s" %
(self.oidc_token, str(token)))
is_expired, errors = oidc_utils.is_token_expired(token)
if is_expired:
raise exceptions.IDDSException(
"Token is already expired: %s" % errors)
headers['X-IDDS-Auth-Token'] = token['id_token']
if type == 'GET':
result = self.session.get(url,
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
except requests.exceptions.ConnectionError as error:
logging.warning('ConnectionError: ' + str(error))
if retry >= self.retries - 1:
raise exceptions.ConnectionException('ConnectionError: ' +
str(error))
if result is not None:
# print(result.text)
# print(result.headers)
# print(result.status_code)
if result.status_code == HTTP_STATUS_CODE.OK:
# print(result.text)
if result.text:
return json_loads(result.text)
else:
return None
elif result.headers and 'ExceptionClass' in result.headers:
try:
if result.headers and 'ExceptionClass' in result.headers:
cls = getattr(exceptions,
result.headers['ExceptionClass'])
msg = result.headers['ExceptionMessage']
raise cls(msg)
else:
if result.text:
data = json_loads(result.text)
raise exceptions.IDDSException(**data)
else:
raise exceptions.IDDSException(
"Unknow exception: %s" % (result.text))
except AttributeError:
raise exceptions.IDDSException(result.text)
elif result.status_code in [
HTTP_STATUS_CODE.BadRequest,
HTTP_STATUS_CODE.Unauthorized,
HTTP_STATUS_CODE.Forbidden, HTTP_STATUS_CODE.NotFound,
HTTP_STATUS_CODE.NoMethod,
HTTP_STATUS_CODE.InternalError
]:
raise exceptions.IDDSException(result.text)
else:
try:
if result.headers and 'ExceptionClass' in result.headers:
cls = getattr(exceptions,
result.headers['ExceptionClass'])
msg = result.headers['ExceptionMessage']
raise cls(msg)
else:
if result.text:
data = json_loads(result.text)
raise exceptions.IDDSException(**data)
else:
raise exceptions.IDDSException(
"Unknow exception: %s" % (result.text))
except AttributeError:
raise exceptions.IDDSException(result.text)
if result is None:
raise exceptions.IDDSException('Response is None')