def process_result_value(self, value, dialect): if value is None: return value elif dialect.name == 'oracle': return json_loads(value) elif dialect.name == 'mysql': return json_loads(value) else: return json_loads(value)
def post(self, request_id, workload_id): """ Create Request. HTTP Success: 200 OK HTTP Error: 400 Bad request 500 Internal Error """ try: if request_id == 'null': request_id = None if workload_id == 'null': workload_id = None if request_id is None: raise Exception("request_id should not be None") except Exception as error: print(error) print(format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.BadRequest, exc_cls=exceptions.BadRequest.__name__, exc_msg=str(error)) try: msg = self.get_request().data and json_loads( self.get_request().data) # command = msg['command'] # parameters = msg['parameters'] add_message(msg_type=MessageType.IDDSCommunication, status=MessageStatus.New, destination=MessageDestination.Clerk, source=MessageSource.Rest, request_id=request_id, workload_id=workload_id, transform_id=None, num_contents=1, msg_content=msg) except exceptions.DuplicatedObject as error: return self.generate_http_response( HTTP_STATUS_CODE.Conflict, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error) return self.generate_http_response(HTTP_STATUS_CODE.OK, data={'request_id': request_id})
def put(self, request_id): """ Update Request properties with a given id. HTTP Success: 200 OK HTTP Error: 400 Bad request 404 Not Found 500 Internal Error """ try: request = self.get_request() parameters = request.data and json_loads(request.data) # parameters['status'] = RequestStatus.Extend except ValueError: return self.generate_http_response( HTTP_STATUS_CODE.BadRequest, exc_cls=exceptions.BadRequest.__name__, exc_msg='Cannot decode json parameter dictionary') try: # update_request(request_id, parameters) # msg = {'command': 'update_request', 'parameters': {'status': RequestStatus.ToSuspend}}) msg = {'command': 'update_request', 'parameters': parameters} add_message(msg_type=MessageType.IDDSCommunication, status=MessageStatus.New, destination=MessageDestination.Clerk, source=MessageSource.Rest, request_id=request_id, workload_id=None, transform_id=None, num_contents=1, msg_content=msg) except exceptions.NoObject as error: return self.generate_http_response( HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error) return self.generate_http_response(HTTP_STATUS_CODE.OK, data={ 'status': 0, 'message': 'update successfully' })
def post(self): """ Create Request. HTTP Success: 200 OK HTTP Error: 400 Bad request 500 Internal Error """ try: parameters = self.get_request().data and json_loads( self.get_request().data) if 'status' not in parameters: parameters['status'] = RequestStatus.New if 'priority' not in parameters: parameters['priority'] = 0 # if 'lifetime' not in parameters: # parameters['lifetime'] = 30 if 'username' not in parameters or not parameters['username']: if 'username' in self.get_request( ).environ and self.get_request().environ['username']: parameters['username'] = self.get_request( ).environ['username'] except ValueError: return self.generate_http_response( HTTP_STATUS_CODE.BadRequest, exc_cls=exceptions.BadRequest.__name__, exc_msg='Cannot decode json parameter dictionary') try: parameters = convert_old_req_2_workflow_req(parameters) request_id = add_request(**parameters) except exceptions.DuplicatedObject as error: return self.generate_http_response( HTTP_STATUS_CODE.Conflict, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error) return self.generate_http_response(HTTP_STATUS_CODE.OK, data={'request_id': request_id})
def get_request_response(self, url, type='GET', data=None, headers=None): """ Send request to the IDDS server and get the response. :param url: http url to connection. :param type: request type(GET, PUT, POST, DEL). :param data: data to be sent to the IDDS server. :param headers: http headers. :returns: response data as json. :raises: """ result = None for retry in range(self.retries): try: if type == 'GET': result = self.session.get(url, cert=(self.client_proxy, self.client_proxy), timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return except requests.exceptions.ConnectionError as error: logging.warning('ConnectionError: ' + str(error)) if retry >= self.retries - 1: raise exceptions.ConnectionException('ConnectionError: ' + str(error)) if result is not None: # print(result.text) if result.status_code in [HTTP_STATUS_CODE.BadRequest, HTTP_STATUS_CODE.Unauthorized, HTTP_STATUS_CODE.Forbidden, HTTP_STATUS_CODE.NotFound, HTTP_STATUS_CODE.NoMethod, HTTP_STATUS_CODE.InternalError]: raise exceptions.IDDSException(result.text) elif result.status_code == HTTP_STATUS_CODE.OK: # print(result.text) if result.text: return json_loads(result.text) else: return None else: try: if result.headers and 'ExceptionClass' in result.headers: cls = getattr(exceptions, result.headers['ExceptionClass']) msg = result.headers['ExceptionMessage'] raise cls(msg) else: if result.text: data = json_loads(result.text) raise exceptions.IDDSException(**data) else: raise exceptions.IDDSException("Unknow exception: %s" % (result.text)) except AttributeError: raise exceptions.IDDSException(result.text) if result is None: raise exceptions.IDDSException('Response is None')
def get_request_response(self, url, type='GET', data=None, headers=None, auth_setup_step=False): """ Send request to the IDDS server and get the response. :param url: http url to connection. :param type: request type(GET, PUT, POST, DEL). :param data: data to be sent to the IDDS server. :param headers: http headers. :returns: response data as json. :raises: """ result = None if not headers: headers = {} headers['X-IDDS-Auth-Type'] = self.auth_type headers['X-IDDS-Auth-VO'] = self.vo for retry in range(self.retries): try: if self.auth_type in ['x509_proxy']: if type == 'GET': result = self.session.get(url, cert=(self.client_proxy, self.client_proxy), timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return elif self.auth_type in ['oidc']: if auth_setup_step: if type == 'GET': result = self.session.get(url, timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return else: oidc_utils = OIDCAuthenticationUtils() status, token = oidc_utils.load_token(self.oidc_token) if not status: raise exceptions.IDDSException( "Token %s cannot be loaded: %s" % (self.oidc_token, str(token))) is_expired, errors = oidc_utils.is_token_expired(token) if is_expired: raise exceptions.IDDSException( "Token is already expired: %s" % errors) headers['X-IDDS-Auth-Token'] = token['id_token'] if type == 'GET': result = self.session.get(url, timeout=self.timeout, headers=headers, verify=False) elif type == 'PUT': result = self.session.put(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'POST': result = self.session.post(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) elif type == 'DEL': result = self.session.delete(url, data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False) else: return except requests.exceptions.ConnectionError as error: logging.warning('ConnectionError: ' + str(error)) if retry >= self.retries - 1: raise exceptions.ConnectionException('ConnectionError: ' + str(error)) if result is not None: # print(result.text) # print(result.headers) # print(result.status_code) if result.status_code == HTTP_STATUS_CODE.OK: # print(result.text) if result.text: return json_loads(result.text) else: return None elif result.headers and 'ExceptionClass' in result.headers: try: if result.headers and 'ExceptionClass' in result.headers: cls = getattr(exceptions, result.headers['ExceptionClass']) msg = result.headers['ExceptionMessage'] raise cls(msg) else: if result.text: data = json_loads(result.text) raise exceptions.IDDSException(**data) else: raise exceptions.IDDSException( "Unknow exception: %s" % (result.text)) except AttributeError: raise exceptions.IDDSException(result.text) elif result.status_code in [ HTTP_STATUS_CODE.BadRequest, HTTP_STATUS_CODE.Unauthorized, HTTP_STATUS_CODE.Forbidden, HTTP_STATUS_CODE.NotFound, HTTP_STATUS_CODE.NoMethod, HTTP_STATUS_CODE.InternalError ]: raise exceptions.IDDSException(result.text) else: try: if result.headers and 'ExceptionClass' in result.headers: cls = getattr(exceptions, result.headers['ExceptionClass']) msg = result.headers['ExceptionMessage'] raise cls(msg) else: if result.text: data = json_loads(result.text) raise exceptions.IDDSException(**data) else: raise exceptions.IDDSException( "Unknow exception: %s" % (result.text)) except AttributeError: raise exceptions.IDDSException(result.text) if result is None: raise exceptions.IDDSException('Response is None')
def test_workflow_condition_reload(self): work1 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=1) work2 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=2) work3 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=3) work4 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=4) work5 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=5) work6 = Work(executable='/bin/hostname', arguments=None, sandbox=None, work_id=6) work7 = Work(executable='echo', arguments='--in=IN_DATASET --out=OUT_DATASET', sandbox=None, work_id=7, primary_input_collection={ 'scope': 'data17', 'name': 'data17.test.raw.1' }, output_collections=[{ 'scope': 'data17', 'name': 'data17.test.work2' }]) work8 = Work(executable='echo', arguments='--in=IN_DATASET --out=OUT_DATASET', sandbox=None, work_id=8, primary_input_collection={ 'scope': 'data17', 'name': 'data17.test.work2' }, output_collections=[{ 'scope': 'data17', 'name': 'data17.test.work3' }]) workflow = Workflow() workflow.add_work(work1, initial=False) workflow.add_work(work2, initial=False) workflow.add_work(work3, initial=False) workflow.add_work(work4, initial=False) workflow.add_work(work5, initial=False) workflow.add_work(work6, initial=False) workflow.add_work(work7, initial=False) workflow.add_work(work8, initial=False) # multiple conditions cond6 = Condition(cond=work1.is_finished, true_work=work2, false_work=work3) cond7 = CompositeCondition( conditions=[work4.is_finished, work5.is_finished], true_works=[work6, cond6], false_works=work7) workflow.add_condition(cond7) workflow_str = json_dumps(workflow, sort_keys=True, indent=4) # print(workflow_str) workflow1 = json_loads(workflow_str) # print('before load_metadata') # self.print_workflow(workflow1) workflow1.load_metadata() # print('after load_metadata') # self.print_workflow(workflow1) workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4) assert (workflow_str == workflow_str1) works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) assert (works == [work7]) works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) assert (works == []) work4.status = WorkStatus.Finished work5.status = WorkStatus.Finished works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) works.sort(key=lambda x: x.work_id) assert (works == [work3, work6]) works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) assert (works == []) work1.status = WorkStatus.Finished works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) works.sort(key=lambda x: x.work_id) assert (works == [work2]) works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger) works.sort(key=lambda x: x.work_id) assert (works == []) work4.status = WorkStatus.New work5.status = WorkStatus.New work1.status = WorkStatus.New works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) assert (works == [work7]) work4.status = WorkStatus.Finished work5.status = WorkStatus.Finished works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) works.sort(key=lambda x: x.work_id) assert (works == [work3, work6]) work1.status = WorkStatus.Finished works = cond7.get_next_works(trigger=ConditionTrigger.Triggered) works.sort(key=lambda x: x.work_id) assert (works == [work2, work6]) work4.status = WorkStatus.New work5.status = WorkStatus.New work1.status = WorkStatus.New return workflow