def extend_request(request_id=None, workload_id=None, lifetime=30, session=None): """ extend an request's lifetime. :param request_id: The id of the request. :param workload_id: The workload_id of the request. :param lifetime: The life time as umber of days. :param session: The database session in use. :raises NoObject: If no request is founded. :raises DatabaseException: If there is a database error. """ try: if not request_id and workload_id: request_ids = get_request_ids_by_workload_id(workload_id) if request_ids and len(request_ids) > 1: raise exceptions.IDDSException( "More than one request with the same workload_id") request_id = request_ids[0] req_update = "update atlas_idds.requests set expired_at=:expired_at where request_id=:request_id" req_stmt = text(req_update) session.execute( req_stmt, { 'expired_at': datetime.datetime.utcnow() + datetime.timedelta(days=lifetime), 'request_id': request_id }) except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('Request %s cannot be found: %s' % (request_id, error))
def get_input_contents(self): """ Get all input contents from DDM. """ try: ret_files = [] coll = self.collections[self._primary_input_collection] ret_file = { 'coll_id': coll['coll_id'], 'scope': coll['scope'], 'name': coll['name'], 'bytes': coll.coll_metadata['bytes'], 'adler32': None, 'min_id': 0, 'max_id': coll.coll_metadata['total_files'], 'content_type': ContentType.File, 'content_metadata': { 'total_files': coll['coll_metadata']['total_files'] } } ret_files.append(ret_file) return ret_files except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def get_input_contents(self): """ Get all input contents from DDM. """ try: ret_files = [] rucio_client = self.get_rucio_client() files = rucio_client.list_files(scope=self.collections[self._primary_input_collection].scope, name=self.collections[self._primary_input_collection].name) for file in files: ret_file = {'coll_id': self.collections[self._primary_input_collection].coll_id, 'scope': file['scope'], 'name': file['name'], 'bytes': file['bytes'], 'adler32': file['adler32'], 'min_id': 0, 'max_id': file['events'], 'content_type': ContentType.File, 'content_metadata': {'events': file['events']}} ret_files.append(ret_file) return ret_files except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_external_collection(self, coll): try: if 'coll_metadata' in coll and 'is_open' in coll[ 'coll_metadata'] and not coll['coll_metadata']['is_open']: return coll else: meta = {} did_meta = self.client.get_metadata(scope=coll['scope'], name=coll['name']) meta = { 'scope': coll['scope'], 'name': coll['name'], 'coll_metadata': { 'bytes': did_meta['bytes'], 'total_files': did_meta['length'], 'availability': did_meta['availability'], 'events': did_meta['events'], 'is_open': did_meta['is_open'], 'run_number': did_meta['run_number'], 'did_type': did_meta['did_type'], 'list_all_files': False } } return meta except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def get_hyperparameters(self, workload_id, request_id, id=None, status=None, limit=None): """ Get hyperparameters from the Head service. :param workload_id: the workload id. :param request_id: the request id. :param status: the status of the hyperparameters. :param limit: limit number of hyperparameters :raise exceptions if it's not got successfully. """ path = self.HPO_BASEURL if not workload_id: workload_id = 'null' if not request_id: request_id = 'null' if id is None: id = 'null' if status is None: status = 'null' if limit is None: limit = 'null' if workload_id == 'null' and request_id == 'null': raise exceptions.IDDSException("One of workload_id and request_id should not be None or empty") url = self.build_url(self.host, path=os.path.join(path, str(workload_id), str(request_id), str(id), str(status), str(limit))) params = self.get_request_response(url, type='GET') return params
def poll_external_collection(self, coll): try: # if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: if 'status' in coll and coll['status'] in [CollectionStatus.Closed]: return coll else: # client = self.get_rucio_client() # did_meta = client.get_metadata(scope=coll['scope'], name=coll['name']) if 'coll_metadata' not in coll: coll['coll_metadata'] = {} coll['coll_metadata']['bytes'] = 1 # coll['coll_metadata']['total_files'] = 1 coll['coll_metadata']['availability'] = 1 coll['coll_metadata']['events'] = 1 coll['coll_metadata']['is_open'] = True coll['coll_metadata']['run_number'] = 1 coll['coll_metadata']['did_type'] = 'DATASET' coll['coll_metadata']['list_all_files'] = False if 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: coll_status = CollectionStatus.Closed else: coll_status = CollectionStatus.Open coll['status'] = coll_status coll['coll_type'] = CollectionType.Dataset return coll except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_external_collection(self, coll): try: if coll.status in [CollectionStatus.Closed]: return coll else: coll.coll_metadata['bytes'] = 1 coll.coll_metadata['availability'] = 1 coll.coll_metadata['events'] = 1 coll.coll_metadata['is_open'] = True coll.coll_metadata['run_number'] = 1 coll.coll_metadata['did_type'] = 'DATASET' coll.coll_metadata['list_all_files'] = False # if (not self.dependency_map_deleted and not self.dependency_map): if not self.has_new_inputs: coll.coll_metadata['is_open'] = False if 'is_open' in coll.coll_metadata and not coll.coll_metadata[ 'is_open']: coll_status = CollectionStatus.Closed else: coll_status = CollectionStatus.Open coll.status = coll_status coll.coll_metadata['coll_type'] = CollectionType.Dataset return coll except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def delete_request(request_id=None, workload_id=None, session=None): """ delete an request. :param request_id: The id of the request. :param workload_id: The workload_id of the request. :param session: The database session in use. :raises NoObject: If no request is founded. :raises DatabaseException: If there is a database error. """ try: if not request_id and workload_id: request_ids = get_request_ids_by_workload_id(workload_id) if request_ids and len(request_ids) > 1: raise exceptions.IDDSException( "More than one request with the same workload_id") request_id = request_ids[0] req2workload_delete = "delete from atlas_idds.req2workload where request_id=:request_id" req2workload_stmt = text(req2workload_delete) session.execute(req2workload_stmt, {'request_id': request_id}) req_delete = "delete from atlas_idds.requests where request_id=:request_id" req_stmt = text(req_delete) session.execute(req_stmt, {'request_id': request_id}) except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject('Request %s cannot be found: %s' % (request_id, error))
def update_hyperparameter(self, workload_id, request_id, id, loss): """ Update hyperparameter to the Head service. :param workload_id: the workload id. :param request_id: the request. :param id: id of the hyper parameter. :param loss: the loss. :raise exceptions if it's not updated successfully. """ if not workload_id: workload_id = 'null' if not request_id: request_id = 'null' if workload_id == 'null' and request_id == 'null': raise exceptions.IDDSException( "One of workload_id and request_id should not be None or empty" ) path = self.HPO_BASEURL url = self.build_url(self.host, path=os.path.join(path, str(workload_id), str(request_id), str(id), str(loss))) r = self.get_request_response(url, type='PUT') return r
def get_rucio_client(self): try: client = RucioClient() except RucioCannotAuthenticate as error: self.logger.error(error) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(error), traceback.format_exc())) return client
def kill_processing(self, processing): try: if processing: from pandatools import Client task_id = processing.workload_id Client.killTask(task_id) except Exception as ex: msg = "Failed to check the processing (%s) status: %s" % (str( processing['processing_id']), str(ex)) raise exceptions.IDDSException(msg)
def get_rucio_download_client(self): try: from rucio.client.downloadclient import DownloadClient client = DownloadClient() except RucioCannotAuthenticate as error: self.logger.error(error) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException( '%s: %s' % (str(error), traceback.format_exc())) return client
def get_input_contents(self): """ Get all input contents from DDM. """ try: ret_files = [] return ret_files except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_external_collection(self, coll): try: if coll.status in [CollectionStatus.Closed]: if not self.terminated: self.logger.info( "Work is not terminated, reopen collection") coll.coll_metadata['is_open'] = True coll.status = CollectionStatus.Open return coll else: coll.coll_metadata['bytes'] = 0 coll.coll_metadata['total_files'] = 0 coll.coll_metadata['availability'] = True coll.coll_metadata['events'] = 0 coll.coll_metadata['is_open'] = True coll.coll_metadata['run_number'] = None coll.coll_metadata['did_type'] = 'DATASET' coll.coll_metadata['list_all_files'] = False if self.terminated: self.logger.info( "Work is terminated. Closing input dataset.") coll.coll_metadata['is_open'] = False if self.points_to_generate <= 0: self.logger.info( "points_to_generate(%s) is equal or smaller than 0. Closing input dataset." % self.points_to_generate) coll.coll_metadata['is_open'] = False if 'is_open' in coll.coll_metadata and not coll.coll_metadata[ 'is_open']: coll_status = CollectionStatus.Closed else: coll_status = CollectionStatus.Open coll.status = coll_status if 'did_type' in coll.coll_metadata: if coll.coll_metadata['did_type'] == 'DATASET': coll_type = CollectionType.Dataset elif coll.coll_metadata['did_type'] == 'CONTAINER': coll_type = CollectionType.Container else: coll_type = CollectionType.File else: coll_type = CollectionType.Dataset coll.coll_metadata['coll_type'] = coll_type return coll except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def kill_processing_force(self, processing): try: if processing: from pandaclient import Client proc = processing['processing_metadata']['processing'] task_id = proc.workload_id # task_id = processing['processing_metadata']['task_id'] Client.killTask(task_id) # Client.finishTask(task_id, soft=True) except Exception as ex: msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) raise exceptions.IDDSException(msg)
def parse_output_file(self, pfn): try: if not os.path.exists(pfn): self.logger.warn("%s doesn't exist" % pfn) return {} else: with open(pfn, 'r') as f: data = f.read() outputs = json.loads(data) return outputs except Exception as ex: # self.logger.error(ex) # self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc())) return {}
def reactivate_processing(self, processing): try: if processing: from pandaclient import Client # task_id = processing['processing_metadata']['task_id'] proc = processing['processing_metadata']['processing'] task_id = proc.workload_id # Client.retryTask(task_id) status, out = Client.retryTask(task_id, newParams={}) self.logger.warn("Retry processing(%s) with task id(%s): %s, %s" % (processing['processing_id'], task_id, status, out)) # Client.reactivateTask(task_id) # Client.resumeTask(task_id) except Exception as ex: msg = "Failed to check the processing (%s) status: %s" % (str(processing['processing_id']), str(ex)) raise exceptions.IDDSException(msg)
def get_input_contents(self, point_index=1): """ Get all input contents from DDM. """ try: if self.terminated: return [] if self.unfinished_points > 0: return [] ret_files = [] coll = self.collections[self._primary_input_collection] if self.max_points and (self.max_points - self.finished_points < self.num_points_per_iteration): self.points_to_generate = self.max_points - self.finished_points # call external processing to generate points points = self.generate_points() self.logger.info("points generated: %s" % str(points)) loss = None for point in points: ret_file = { 'coll_id': coll.coll_id, 'scope': coll.scope, 'name': str(point_index), 'bytes': 0, 'adler32': None, 'min_id': 0, 'max_id': 0, 'path': json.dumps((point, loss)), 'content_type': ContentType.File, 'content_metadata': { 'events': 0 } } ret_files.append(ret_file) point_index += 1 return ret_files except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def poll_panda_task(self, processing=None, task_name=None): try: if not self.pandamonitor: self.pandamonitor = self.load_panda_monitor() self.logger.info("panda server: %s" % self.pandamonitor) task_id = None task_info = None if processing: task_id = processing['processing_metadata']['task_id'] task_url = self.pandamonitor + '/task/?json&jeditaskid=' + str(task_id) task_info = self.download_payload_json(task_url) elif task_name: task_url = self.pandamonitor + '/tasks/?taskname=' + str(task_name) + "&json" self.logger.debug("poll_panda_task, task_url: %s" % str(task_url)) self.logger.debug("poll_panda_task, task_url: %s" % str(self.download_payload_json(task_url))) task_json = self.download_payload_json(task_url) if len(task_json) > 0: task_info = task_json[0] else: return "No status", {} if not task_id: task_id = task_info.get('jeditaskid', None) if not task_id: return "No status", {} jobs_url = self.pandamonitor + '/jobs/?json&datasets=yes&jeditaskid=' + str(task_id) jobs_list = self.download_payload_json(jobs_url) outputs_status = {} for job_info in jobs_list['jobs']: if 'jobstatus' in job_info and 'datasets' in job_info and len(job_info['datasets']) > 0: output_index = job_info['datasets'][0]['lfn'].split(':')[1] status = self.jobs_to_idd_ds_status(job_info['jobstatus']) outputs_status[output_index] = status task_status = None self.logger.debug("poll_panda_task, task_info: %s" % str(task_info)) if task_info.get("task", None) is not None: task_status = task_info["task"]["status"] return task_status, outputs_status except Exception as ex: msg = "Failed to check the panda task(%s) status: %s" % (str(task_id), str(ex)) raise exceptions.IDDSException(msg)
def poll_external_collection(self, coll): try: if coll.status in [CollectionStatus.Closed]: return coll else: client = self.get_rucio_client() did_meta = client.get_metadata(scope=coll.scope, name=coll.name) coll.coll_metadata['bytes'] = did_meta['bytes'] coll.coll_metadata['total_files'] = did_meta['length'] coll.coll_metadata['availability'] = did_meta['availability'] coll.coll_metadata['events'] = did_meta['events'] coll.coll_metadata['is_open'] = did_meta['is_open'] coll.coll_metadata['run_number'] = did_meta['run_number'] coll.coll_metadata['did_type'] = did_meta['did_type'] coll.coll_metadata['list_all_files'] = False if (('is_open' in coll.coll_metadata and not coll.coll_metadata['is_open']) or ('force_close' in coll.coll_metadata and coll.coll_metadata['force_close'])): # noqa: W503 coll_status = CollectionStatus.Closed else: coll_status = CollectionStatus.Open coll.status = coll_status if 'did_type' in coll.coll_metadata: if coll.coll_metadata['did_type'] == 'DATASET': coll_type = CollectionType.Dataset elif coll.coll_metadata['did_type'] == 'CONTAINER': coll_type = CollectionType.Container else: coll_type = CollectionType.File else: coll_type = CollectionType.Dataset coll.coll_metadata['coll_type'] = coll_type return coll except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def submit_panda_task(self, processing): try: from pandatools import Client status, tmpOut = Client.insertTaskParams(self.panda_task_paramsmap, False, True) if status == 0: tmp_status, tmp_output = tmpOut m = re.search("jediTaskID=(\d+)", tmp_output) # noqa W605 task_id = int(m.group(1)) processing.workload_id = task_id else: self.add_errors(tmpOut) raise Exception(tmpOut) except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def get_request(request_id=None, workload_id=None, session=None): """ Get a request or raise a NoObject exception. :param request_id: The id of the request. :param workload_id: The workload_id of the request. :param session: The database session in use. :raises NoObject: If no request is founded. :returns: Request. """ try: if not request_id and workload_id: request_ids = get_request_ids_by_workload_id(workload_id) if request_ids and len(request_ids) > 1: raise exceptions.IDDSException( "More than one request with the same workload_id") request_id = request_ids[0] req_select = """select request_id, scope, name, requester, request_type, transform_tag, priority, status, locking, workload_id, created_at, updated_at, accessed_at, expired_at, errors, request_metadata, processing_metadata from atlas_idds.requests where request_id=:request_id """ req_stmt = text(req_select) result = session.execute(req_stmt, {'request_id': request_id}) request = result.fetchone() if request is None: raise exceptions.NoObject( 'request request_id: %s, workload_id: %s cannot be found' % (request_id, workload_id)) request = convert_request_to_dict(request) return request except sqlalchemy.orm.exc.NoResultFound as error: raise exceptions.NoObject( 'request request_id: %s, workload_id: %s cannot be found: %s' % (request_id, workload_id, error))
def get(self, vo, auth_type='oidc'): """ Get sign url for user to approve. HTTP Success: 200 OK HTTP Error: 404 Not Found 500 InternalError :returns: dictionary with sign url. """ try: if auth_type == 'oidc': oidc = OIDCAuthentication() status, sign_url = oidc.get_oidc_sign_url(vo) if status: rets = sign_url return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets) else: raise exceptions.IDDSException( "Failed to get oidc sign url: %s" % str(sign_url)) else: raise exceptions.AuthenticationNotSupported( "auth_type %s is not supported to call this function." % str(auth_type)) except exceptions.NoObject as error: return self.generate_http_response( HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(traceback.format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
def poll_external_collection(self, coll): try: # if 'coll_metadata' in coll and 'is_open' in coll['coll_metadata'] and not coll['coll_metadata']['is_open']: if coll.status in [CollectionStatus.Closed]: return coll else: # client = self.get_rucio_client() # did_meta = client.get_metadata(scope=coll['scope'], name=coll['name']) coll.coll_metadata['bytes'] = 0 coll.coll_metadata['total_files'] = 0 coll.coll_metadata['availability'] = True coll.coll_metadata['events'] = 0 coll.coll_metadata['is_open'] = False coll.coll_metadata['run_number'] = None coll.coll_metadata['did_type'] = 'DATASET' coll.coll_metadata['list_all_files'] = False if 'is_open' in coll.coll_metadata and not coll.coll_metadata[ 'is_open']: coll_status = CollectionStatus.Closed else: coll_status = CollectionStatus.Open coll.status = coll_status if 'did_type' in coll.coll_metadata: if coll.coll_metadata['did_type'] == 'DATASET': coll_type = CollectionType.Dataset elif coll.coll_metadata['did_type'] == 'CONTAINER': coll_type = CollectionType.Container else: coll_type = CollectionType.File else: coll_type = CollectionType.Dataset coll.coll_metadata['coll_type'] = coll_type return coll except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def get(self, vo, device_code, interval=5, expires_in=60): """ Get id token. HTTP Success: 200 OK HTTP Error: 404 Not Found 500 InternalError :returns: dictionary with sign url. """ try: oidc = OIDCAuthentication() status, id_token = oidc.get_id_token(vo, device_code, interval, expires_in) if status: return self.generate_http_response(HTTP_STATUS_CODE.OK, data=id_token) else: if 'error' in id_token and 'authorization_pending' in id_token[ 'error']: raise exceptions.AuthenticationPending(str(id_token)) else: raise exceptions.IDDSException( "Failed to get oidc token: %s" % str(id_token)) except exceptions.NoObject as error: return self.generate_http_response( HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(traceback.format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
def post(self, vo): """ Refresh the token. HTTP Success: 200 OK HTTP Error: 400 Bad request 500 Internal Error """ try: parameters = self.get_request().data and json.loads( self.get_request().data) refresh_token = parameters['refresh_token'] oidc = OIDCAuthentication() status, id_token = oidc.refresh_id_token(vo, refresh_token) if status: return self.generate_http_response(HTTP_STATUS_CODE.OK, data=id_token) else: raise exceptions.IDDSException( "Failed to refresh oidc token: %s" % str(id_token)) except exceptions.NoObject as error: return self.generate_http_response( HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(traceback.format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
def get_input_contents(self): """ Get all input contents from DDM. """ try: files = self.check_dependencies() ret_files = [] coll = self.collections[self._primary_input_collection] for file in files: ret_file = {'coll_id': coll['coll_id'], 'scope': coll['scope'], 'name': file, # or a different file name from the dataset name 'bytes': 1, 'adler32': '12345678', 'min_id': 0, 'max_id': 1, 'content_type': ContentType.File, 'content_metadata': {'events': 1}} # here events is all events for eventservice, not used here. ret_files.append(ret_file) return ret_files except Exception as ex: self.logger.error(ex) self.logger.error(traceback.format_exc()) raise exceptions.IDDSException('%s: %s' % (str(ex), traceback.format_exc()))
def get_request_response(self, url, type='GET', data=None, headers=None): """ Send request to the IDDS server and get the response. :param url: http url to connection. :param type: request type(GET, PUT, POST, DEL). :param data: data to be sent to the IDDS server. :param headers: http headers. :returns: response data as json. :raises: """ result = None for retry in range(self.retries): try: if type == 'GET': result = self.session.get(url, cert=(self.client_proxy, self.client_proxy), timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return except requests.exceptions.ConnectionError as error: logging.warning('ConnectionError: ' + str(error)) if retry >= self.retries - 1: raise exceptions.ConnectionException('ConnectionError: ' + str(error)) if result is not None: # print(result.text) if result.status_code in [HTTP_STATUS_CODE.BadRequest, HTTP_STATUS_CODE.Unauthorized, HTTP_STATUS_CODE.Forbidden, HTTP_STATUS_CODE.NotFound, HTTP_STATUS_CODE.NoMethod, HTTP_STATUS_CODE.InternalError]: raise exceptions.IDDSException(result.text) elif result.status_code == HTTP_STATUS_CODE.OK: # print(result.text) if result.text: return json_loads(result.text) else: return None else: try: if result.headers and 'ExceptionClass' in result.headers: cls = getattr(exceptions, result.headers['ExceptionClass']) msg = result.headers['ExceptionMessage'] raise cls(msg) else: if result.text: data = json_loads(result.text) raise exceptions.IDDSException(**data) else: raise exceptions.IDDSException("Unknow exception: %s" % (result.text)) except AttributeError: raise exceptions.IDDSException(result.text) if result is None: raise exceptions.IDDSException('Response is None')
def __init__( self, executable=None, arguments=None, parameters=None, setup=None, work_tag='hpo', exec_type='local', sandbox=None, work_id=None, name=None, # primary_input_collection=None, other_input_collections=None, # output_collections=None, log_collections=None, logger=None, workload_id=None, agent_attributes=None, method=None, container_workdir=None, output_json=None, opt_space=None, initial_points=None, max_points=None, num_points_per_iteration=10): """ Init a work/task/transformation. :param setup: A string to setup the executable enviroment, it can be None. :param executable: The executable. :param arguments: The arguments. :param parameters: A dict with arguments needed to be replaced. :param work_type: The work type like data carousel, hyperparameteroptimization and so on. :param exec_type: The exec type like 'local', 'remote'(with remote_package set), 'docker' and so on. :param sandbox: The sandbox. :param work_id: The work/task id. :param primary_input_collection: The primary input collection. :param other_input_collections: List of the input collections. :param output_collections: List of the output collections. # :param workflow: The workflow the current work belongs to. :param method: The HPO methd to use. It can be 'nevergrad', 'container' or 'sandbox'. :param sandbox: The sandbox to be uploaded or the container path. :param executable: The executable command. :param arguments: The arguments for the executable. :param container_workdir: The working directory for container. :param opt_space: The optimization space. :param initial_points: The initial points. :param max_points: The maximum number of points. :param number_points_per_iteration: The number of points to be generated per iteration. """ if not name: if workload_id: name = 'hpo.' + str( workload_id) + "." + datetime.datetime.utcnow().strftime( "%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) else: name = 'hpo.' + datetime.datetime.utcnow().strftime( "%Y_%m_%d_%H_%M_%S_%f") + str(random.randint(1, 1000)) primary_input_collection = {'scope': 'HPO', 'name': name} other_input_collections = None output_collections = [{'scope': 'HPO', 'name': name + 'output'}] log_collections = None super(ATLASHPOWork, self).__init__(executable=executable, arguments=arguments, parameters=parameters, setup=setup, work_type=TransformType.HyperParameterOpt, exec_type=exec_type, sandbox=sandbox, work_id=work_id, primary_input_collection=primary_input_collection, other_input_collections=other_input_collections, output_collections=output_collections, log_collections=log_collections, logger=logger, agent_attributes=agent_attributes) self.method = method self.sandbox = sandbox self.executable = executable self.arguments = arguments self.container_workdir = container_workdir self.opt_space = opt_space self.initial_points = initial_points self.max_points = max_points self.num_points_per_iteration = num_points_per_iteration self.unfinished_points = 0 self.input_json = None self.output_json = output_json self.finished_points = 0 self.points_to_generate = self.num_points_per_iteration self.point_index = 0 self.terminated = False self.polling_retries = 0 if not self.num_points_per_iteration or self.num_points_per_iteration < 0: raise exceptions.IDDSException( "num_points_per_iteration must be integer bigger than 0") self.num_points_per_iteration = int(self.num_points_per_iteration) if not self.method and self.executable and 'docker' in self.executable: self.method = 'docker' # if self.agent_attributes and 'atlashpowork' in self.agent_attributes: # self.agent_attributes = self.agent_attributes['atlashpowork'] # self.logger.info("agent_attributes: %s" % self.agent_attributes) # if self.agent_attributes and 'workdir' in self.agent_attributes and self.agent_attributes['workdir']: # self.set_workdir(self.agent_attributes['workdir']) # self.logger.info("workdir: %s" % self.get_workdir()) if agent_attributes: self.set_agent_attributes(agent_attributes)
def get_request_response(self, url, type='GET', data=None, headers=None, auth_setup_step=False): """ Send request to the IDDS server and get the response. :param url: http url to connection. :param type: request type(GET, PUT, POST, DEL). :param data: data to be sent to the IDDS server. :param headers: http headers. :returns: response data as json. :raises: """ result = None if not headers: headers = {} headers['X-IDDS-Auth-Type'] = self.auth_type headers['X-IDDS-Auth-VO'] = self.vo for retry in range(self.retries): try: if self.auth_type in ['x509_proxy']: if type == 'GET': result = self.session.get(url, cert=(self.client_proxy, self.client_proxy), timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return elif self.auth_type in ['oidc']: if auth_setup_step: if type == 'GET': result = self.session.get(url, timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return else: oidc_utils = OIDCAuthenticationUtils() status, token = oidc_utils.load_token(self.oidc_token) if not status: raise exceptions.IDDSException( "Token %s cannot be loaded: %s" % (self.oidc_token, str(token))) is_expired, errors = oidc_utils.is_token_expired(token) if is_expired: raise exceptions.IDDSException( "Token is already expired: %s" % errors) headers['X-IDDS-Auth-Token'] = token['id_token'] if type == 'GET': result = self.session.get(url, timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return except requests.exceptions.ConnectionError as error: logging.warning('ConnectionError: ' + str(error)) if retry >= self.retries - 1: raise exceptions.ConnectionException('ConnectionError: ' + str(error)) if result is not None: # print(result.text) # print(result.headers) # print(result.status_code) if result.status_code == HTTP_STATUS_CODE.OK: # print(result.text) if result.text: return json_loads(result.text) else: return None elif result.headers and 'ExceptionClass' in result.headers: try: if result.headers and 'ExceptionClass' in result.headers: cls = getattr(exceptions, result.headers['ExceptionClass']) msg = result.headers['ExceptionMessage'] raise cls(msg) else: if result.text: data = json_loads(result.text) raise exceptions.IDDSException(**data) else: raise exceptions.IDDSException( "Unknow exception: %s" % (result.text)) except AttributeError: raise exceptions.IDDSException(result.text) elif result.status_code in [ HTTP_STATUS_CODE.BadRequest, HTTP_STATUS_CODE.Unauthorized, HTTP_STATUS_CODE.Forbidden, HTTP_STATUS_CODE.NotFound, HTTP_STATUS_CODE.NoMethod, HTTP_STATUS_CODE.InternalError ]: raise exceptions.IDDSException(result.text) else: try: if result.headers and 'ExceptionClass' in result.headers: cls = getattr(exceptions, result.headers['ExceptionClass']) msg = result.headers['ExceptionMessage'] raise cls(msg) else: if result.text: data = json_loads(result.text) raise exceptions.IDDSException(**data) else: raise exceptions.IDDSException( "Unknow exception: %s" % (result.text)) except AttributeError: raise exceptions.IDDSException(result.text) if result is None: raise exceptions.IDDSException('Response is None')