def process_result_value(self, value, dialect):
if value is None:
return value
elif dialect.name == 'oracle':
return json_loads(value)
elif dialect.name == 'mysql':
return json_loads(value)
else:
return json_loads(value)
def post(self, request_id, workload_id):
""" Create Request.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
500 Internal Error
"""
try:
if request_id == 'null':
request_id = None
if workload_id == 'null':
workload_id = None
if request_id is None:
raise Exception("request_id should not be None")
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.BadRequest,
exc_cls=exceptions.BadRequest.__name__,
exc_msg=str(error))
try:
msg = self.get_request().data and json_loads(
self.get_request().data)
# command = msg['command']
# parameters = msg['parameters']
add_message(msg_type=MessageType.IDDSCommunication,
status=MessageStatus.New,
destination=MessageDestination.Clerk,
source=MessageSource.Rest,
request_id=request_id,
workload_id=workload_id,
transform_id=None,
num_contents=1,
msg_content=msg)
except exceptions.DuplicatedObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.Conflict,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data={'request_id': request_id})
def put(self, request_id):
""" Update Request properties with a given id.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
404 Not Found
500 Internal Error
"""
try:
request = self.get_request()
parameters = request.data and json_loads(request.data)
# parameters['status'] = RequestStatus.Extend
except ValueError:
return self.generate_http_response(
HTTP_STATUS_CODE.BadRequest,
exc_cls=exceptions.BadRequest.__name__,
exc_msg='Cannot decode json parameter dictionary')
try:
# update_request(request_id, parameters)
# msg = {'command': 'update_request', 'parameters': {'status': RequestStatus.ToSuspend}})
msg = {'command': 'update_request', 'parameters': parameters}
add_message(msg_type=MessageType.IDDSCommunication,
status=MessageStatus.New,
destination=MessageDestination.Clerk,
source=MessageSource.Rest,
request_id=request_id,
workload_id=None,
transform_id=None,
num_contents=1,
msg_content=msg)
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data={
'status': 0,
'message': 'update successfully'
})
def post(self):
""" Create Request.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
500 Internal Error
"""
try:
parameters = self.get_request().data and json_loads(
self.get_request().data)
if 'status' not in parameters:
parameters['status'] = RequestStatus.New
if 'priority' not in parameters:
parameters['priority'] = 0
# if 'lifetime' not in parameters:
# parameters['lifetime'] = 30
if 'username' not in parameters or not parameters['username']:
if 'username' in self.get_request(
).environ and self.get_request().environ['username']:
parameters['username'] = self.get_request(
).environ['username']
except ValueError:
return self.generate_http_response(
HTTP_STATUS_CODE.BadRequest,
exc_cls=exceptions.BadRequest.__name__,
exc_msg='Cannot decode json parameter dictionary')
try:
parameters = convert_old_req_2_workflow_req(parameters)
request_id = add_request(**parameters)
except exceptions.DuplicatedObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.Conflict,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK,
data={'request_id': request_id})
def get_request_response(self, url, type='GET', data=None, headers=None):
"""
Send request to the IDDS server and get the response.
:param url: http url to connection.
:param type: request type(GET, PUT, POST, DEL).
:param data: data to be sent to the IDDS server.
:param headers: http headers.
:returns: response data as json.
:raises:
"""
result = None
for retry in range(self.retries):
try:
if type == 'GET':
result = self.session.get(url, cert=(self.client_proxy, self.client_proxy), timeout=self.timeout, headers=headers, verify=False)
elif type == 'PUT':
result = self.session.put(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False)
elif type == 'POST':
result = self.session.post(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False)
elif type == 'DEL':
result = self.session.delete(url, cert=(self.client_proxy, self.client_proxy), data=json_dumps(data), timeout=self.timeout, headers=headers, verify=False)
else:
return
except requests.exceptions.ConnectionError as error:
logging.warning('ConnectionError: ' + str(error))
if retry >= self.retries - 1:
raise exceptions.ConnectionException('ConnectionError: ' + str(error))
if result is not None:
# print(result.text)
if result.status_code in [HTTP_STATUS_CODE.BadRequest,
HTTP_STATUS_CODE.Unauthorized,
HTTP_STATUS_CODE.Forbidden,
HTTP_STATUS_CODE.NotFound,
HTTP_STATUS_CODE.NoMethod,
HTTP_STATUS_CODE.InternalError]:
raise exceptions.IDDSException(result.text)
elif result.status_code == HTTP_STATUS_CODE.OK:
# print(result.text)
if result.text:
return json_loads(result.text)
else:
return None
else:
try:
if result.headers and 'ExceptionClass' in result.headers:
cls = getattr(exceptions, result.headers['ExceptionClass'])
msg = result.headers['ExceptionMessage']
raise cls(msg)
else:
if result.text:
data = json_loads(result.text)
raise exceptions.IDDSException(**data)
else:
raise exceptions.IDDSException("Unknow exception: %s" % (result.text))
except AttributeError:
raise exceptions.IDDSException(result.text)
if result is None:
raise exceptions.IDDSException('Response is None')
def get_request_response(self,
url,
type='GET',
data=None,
headers=None,
auth_setup_step=False):
"""
Send request to the IDDS server and get the response.
:param url: http url to connection.
:param type: request type(GET, PUT, POST, DEL).
:param data: data to be sent to the IDDS server.
:param headers: http headers.
:returns: response data as json.
:raises:
"""
result = None
if not headers:
headers = {}
headers['X-IDDS-Auth-Type'] = self.auth_type
headers['X-IDDS-Auth-VO'] = self.vo
for retry in range(self.retries):
try:
if self.auth_type in ['x509_proxy']:
if type == 'GET':
result = self.session.get(url,
cert=(self.client_proxy,
self.client_proxy),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
cert=(self.client_proxy,
self.client_proxy),
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
cert=(self.client_proxy,
self.client_proxy),
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
cert=(self.client_proxy,
self.client_proxy),
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
elif self.auth_type in ['oidc']:
if auth_setup_step:
if type == 'GET':
result = self.session.get(url,
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
else:
oidc_utils = OIDCAuthenticationUtils()
status, token = oidc_utils.load_token(self.oidc_token)
if not status:
raise exceptions.IDDSException(
"Token %s cannot be loaded: %s" %
(self.oidc_token, str(token)))
is_expired, errors = oidc_utils.is_token_expired(token)
if is_expired:
raise exceptions.IDDSException(
"Token is already expired: %s" % errors)
headers['X-IDDS-Auth-Token'] = token['id_token']
if type == 'GET':
result = self.session.get(url,
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'PUT':
result = self.session.put(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'POST':
result = self.session.post(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
elif type == 'DEL':
result = self.session.delete(url,
data=json_dumps(data),
timeout=self.timeout,
headers=headers,
verify=False)
else:
return
except requests.exceptions.ConnectionError as error:
logging.warning('ConnectionError: ' + str(error))
if retry >= self.retries - 1:
raise exceptions.ConnectionException('ConnectionError: ' +
str(error))
if result is not None:
# print(result.text)
# print(result.headers)
# print(result.status_code)
if result.status_code == HTTP_STATUS_CODE.OK:
# print(result.text)
if result.text:
return json_loads(result.text)
else:
return None
elif result.headers and 'ExceptionClass' in result.headers:
try:
if result.headers and 'ExceptionClass' in result.headers:
cls = getattr(exceptions,
result.headers['ExceptionClass'])
msg = result.headers['ExceptionMessage']
raise cls(msg)
else:
if result.text:
data = json_loads(result.text)
raise exceptions.IDDSException(**data)
else:
raise exceptions.IDDSException(
"Unknow exception: %s" % (result.text))
except AttributeError:
raise exceptions.IDDSException(result.text)
elif result.status_code in [
HTTP_STATUS_CODE.BadRequest,
HTTP_STATUS_CODE.Unauthorized,
HTTP_STATUS_CODE.Forbidden, HTTP_STATUS_CODE.NotFound,
HTTP_STATUS_CODE.NoMethod,
HTTP_STATUS_CODE.InternalError
]:
raise exceptions.IDDSException(result.text)
else:
try:
if result.headers and 'ExceptionClass' in result.headers:
cls = getattr(exceptions,
result.headers['ExceptionClass'])
msg = result.headers['ExceptionMessage']
raise cls(msg)
else:
if result.text:
data = json_loads(result.text)
raise exceptions.IDDSException(**data)
else:
raise exceptions.IDDSException(
"Unknow exception: %s" % (result.text))
except AttributeError:
raise exceptions.IDDSException(result.text)
if result is None:
raise exceptions.IDDSException('Response is None')
def test_workflow_condition_reload(self):
work1 = Work(executable='/bin/hostname',
arguments=None,
sandbox=None,
work_id=1)
work2 = Work(executable='/bin/hostname',
arguments=None,
sandbox=None,
work_id=2)
work3 = Work(executable='/bin/hostname',
arguments=None,
sandbox=None,
work_id=3)
work4 = Work(executable='/bin/hostname',
arguments=None,
sandbox=None,
work_id=4)
work5 = Work(executable='/bin/hostname',
arguments=None,
sandbox=None,
work_id=5)
work6 = Work(executable='/bin/hostname',
arguments=None,
sandbox=None,
work_id=6)
work7 = Work(executable='echo',
arguments='--in=IN_DATASET --out=OUT_DATASET',
sandbox=None,
work_id=7,
primary_input_collection={
'scope': 'data17',
'name': 'data17.test.raw.1'
},
output_collections=[{
'scope': 'data17',
'name': 'data17.test.work2'
}])
work8 = Work(executable='echo',
arguments='--in=IN_DATASET --out=OUT_DATASET',
sandbox=None,
work_id=8,
primary_input_collection={
'scope': 'data17',
'name': 'data17.test.work2'
},
output_collections=[{
'scope': 'data17',
'name': 'data17.test.work3'
}])
workflow = Workflow()
workflow.add_work(work1, initial=False)
workflow.add_work(work2, initial=False)
workflow.add_work(work3, initial=False)
workflow.add_work(work4, initial=False)
workflow.add_work(work5, initial=False)
workflow.add_work(work6, initial=False)
workflow.add_work(work7, initial=False)
workflow.add_work(work8, initial=False)
# multiple conditions
cond6 = Condition(cond=work1.is_finished,
true_work=work2,
false_work=work3)
cond7 = CompositeCondition(
conditions=[work4.is_finished, work5.is_finished],
true_works=[work6, cond6],
false_works=work7)
workflow.add_condition(cond7)
workflow_str = json_dumps(workflow, sort_keys=True, indent=4)
# print(workflow_str)
workflow1 = json_loads(workflow_str)
# print('before load_metadata')
# self.print_workflow(workflow1)
workflow1.load_metadata()
# print('after load_metadata')
# self.print_workflow(workflow1)
workflow_str1 = json_dumps(workflow1, sort_keys=True, indent=4)
assert (workflow_str == workflow_str1)
works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger)
assert (works == [work7])
works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger)
assert (works == [])
work4.status = WorkStatus.Finished
work5.status = WorkStatus.Finished
works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger)
works.sort(key=lambda x: x.work_id)
assert (works == [work3, work6])
works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger)
assert (works == [])
work1.status = WorkStatus.Finished
works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger)
works.sort(key=lambda x: x.work_id)
assert (works == [work2])
works = cond7.get_next_works(trigger=ConditionTrigger.ToTrigger)
works.sort(key=lambda x: x.work_id)
assert (works == [])
work4.status = WorkStatus.New
work5.status = WorkStatus.New
work1.status = WorkStatus.New
works = cond7.get_next_works(trigger=ConditionTrigger.Triggered)
assert (works == [work7])
work4.status = WorkStatus.Finished
work5.status = WorkStatus.Finished
works = cond7.get_next_works(trigger=ConditionTrigger.Triggered)
works.sort(key=lambda x: x.work_id)
assert (works == [work3, work6])
work1.status = WorkStatus.Finished
works = cond7.get_next_works(trigger=ConditionTrigger.Triggered)
works.sort(key=lambda x: x.work_id)
assert (works == [work2, work6])
work4.status = WorkStatus.New
work5.status = WorkStatus.New
work1.status = WorkStatus.New
return workflow