def get_service_info(self):
kwargs = self._request_params()
response = requests.get(
"%s/v1/tasks/service-info" % (self.url),
**kwargs)
raise_for_status(response)
return unmarshal(response.json(), ServiceInfo)
def test_unmarshal_types(self):
empty_log_dict = {
'id':
'c55qjplpsjir0oo1kdj0',
'state':
'QUEUED',
'name':
'toil-bbc72af7-e11a-4831-9392-669ea6c309a1-0',
'executors': [{
'image':
'testImage',
'command': [
'_toil_kubernetes_executor',
'gAWVGwAAAAAAAAB9lIwHY29tbWFuZJSMCnNsZWVwIDEwMDCUcy4='
]
}],
'logs': [{}],
'creation_time':
"2017-10-09T17:00:00"
}
expected = empty_log_dict.copy()
expected["creation_time"] = dateutil.parser.parse(
expected["creation_time"])
empty_log_str = json.dumps(empty_log_dict)
o1 = unmarshal(empty_log_dict, Task)
self.assertEqual(o1.as_dict(), expected)
self.assertEqual(o1.as_json(), empty_log_str)
def get_task(self, task_id, view="BASIC"):
req = GetTaskRequest(task_id, view)
payload = {"view": req.view}
kwargs = self._request_params(params=payload)
response = requests.get("%s/v1/tasks/%s" % (self.url, req.id),
**kwargs)
raise_for_status(response)
return unmarshal(response.json(), Task)
def get_task(self, task_id, view="BASIC"):
req = GetTaskRequest(task_id, view)
payload = {"view": req.view}
response = requests.get("%s/v1/tasks/%s" % (self.url, req.id),
params=payload,
timeout=self.timeout,
auth=(self.user, self.password))
raise_for_status(response)
return unmarshal(response.json(), Task)
def create_task(self, task):
if isinstance(task, Task):
msg = task.as_json()
else:
raise TypeError("Expected Task instance")
kwargs = self._request_params(data=msg)
response = requests.post("%s/v1/tasks" % (self.url), **kwargs)
raise_for_status(response)
return unmarshal(response.json(), CreateTaskResponse).id
def create_task(self, task):
if isinstance(task, Task):
msg = task.as_json()
else:
raise TypeError("Expected Task instance")
response = requests.post("%s/v1/tasks" % (self.url),
data=msg,
headers={'Content-Type': 'application/json'},
timeout=self.timeout)
raise_for_status(response)
return unmarshal(response.json(), CreateTaskResponse).id
def list_tasks(self, view="MINIMAL", page_size=None, page_token=None):
req = ListTasksRequest(view=view,
page_size=page_size,
page_token=page_token,
name_prefix=None,
project=None)
msg = req.as_dict()
kwargs = self._request_params(params=msg)
response = requests.get("%s/v1/tasks" % (self.url), **kwargs)
raise_for_status(response)
return unmarshal(response.json(), ListTasksResponse)
def list_tasks(self, view="MINIMAL", page_size=None, page_token=None):
req = ListTasksRequest(view=view,
page_size=page_size,
page_token=page_token,
name_prefix=None,
project=None)
msg = req.as_dict()
response = requests.get("%s/v1/tasks" % (self.url),
params=msg,
timeout=self.timeout,
auth=(self.user, self.password))
raise_for_status(response)
return unmarshal(response.json(), ListTasksResponse)
def test_unmarshal(self):
test_invalid_dict = {"adfasd": "bar"}
test_invalid_str = json.dumps(test_invalid_dict)
with self.assertRaises(UnmarshalError):
unmarshal(test_invalid_dict, CreateTaskResponse)
with self.assertRaises(UnmarshalError):
unmarshal(test_invalid_str, CreateTaskResponse)
test_simple_dict = {
"url": "file://test_file",
"path": "/mnt/test_file",
"type": "FILE"
}
test_simple_str = json.dumps(test_simple_dict)
o1 = unmarshal(test_simple_dict, Input)
o2 = unmarshal(test_simple_str, Input)
self.assertTrue(isinstance(o1, Input))
self.assertTrue(isinstance(o2, Input))
self.assertEqual(o1, o2)
self.assertEqual(o1.as_dict(), test_simple_dict)
self.assertEqual(o1.as_json(), test_simple_str)
test_complex_dict = {
"name":
"test",
"inputs": [{
"url": "file:///storage/inputs/test_file",
"path": "/mnt/test_file",
"type": "FILE"
}],
"outputs": [{
"url": "file:///storage/outputs/test_outputfile",
"path": "/mnt/test_outputfile",
"type": "FILE"
}],
"executors": [{
"image": "alpine",
"command": ["echo", "hello"],
"env": {
"HOME": "/home/"
}
}],
"logs": [{
"start_time":
"2017-10-09T17:05:00.0Z",
"end_time":
"2017-10-09T17:40:30.0Z",
"metadata": {
"testmeta": "testvalue"
},
"logs": [{
"start_time": "2017-10-09T17:06:30.0Z",
"end_time": "2017-10-09T17:39:50.0Z",
"exit_code": 0,
"stdout": "hello",
"stderr": "",
}],
"outputs": [{
"url": "file:///storage/outputs/test_outputfile",
"path": "/mnt/test_outputfile",
"size_bytes": "3333"
}],
"system_logs": [
"level='info' msg='Download started' \
timestamp='2018-05-04T09:12:42.391262682-07:00' \
task_attempt='0' executor_index='0' \
url='swift://biostream/protograph'"
]
}],
"creation_time":
"2017-10-09T17:00:00.0Z"
}
test_complex_str = json.dumps(test_complex_dict)
o1 = unmarshal(test_complex_dict, Task)
o2 = unmarshal(test_complex_str, Task)
self.assertTrue(isinstance(o1, Task))
self.assertTrue(isinstance(o2, Task))
self.assertEqual(o1, o2)
expected = test_complex_dict.copy()
# handle expected conversions
expected["logs"][0]["outputs"][0]["size_bytes"] = int(
expected["logs"][0]["outputs"][0]["size_bytes"])
expected["logs"][0]["start_time"] = dateutil.parser.parse(
expected["logs"][0]["start_time"])
expected["logs"][0]["end_time"] = dateutil.parser.parse(
expected["logs"][0]["end_time"])
expected["logs"][0]["logs"][0]["start_time"] = dateutil.parser.parse(
expected["logs"][0]["logs"][0]["start_time"])
expected["logs"][0]["logs"][0]["end_time"] = dateutil.parser.parse(
expected["logs"][0]["logs"][0]["end_time"])
expected["creation_time"] = dateutil.parser.parse(
expected["creation_time"])
self.assertEqual(o1.as_dict(), expected)
def get_service_info(self):
response = requests.get("%s/v1/tasks/service-info" % (self.url),
timeout=self.timeout)
raise_for_status(response)
return unmarshal(response.json(), ServiceInfo)
def test_unmarshal(self):
test_invalid_dict = {"adfasd": "bar"}
test_invalid_str = json.dumps(test_invalid_dict)
with self.assertRaises(UnmarshalError):
unmarshal(test_invalid_dict, CreateTaskResponse)
with self.assertRaises(UnmarshalError):
unmarshal(test_invalid_str, CreateTaskResponse)
test_simple_dict = {
"url": "file://test_file",
"path": "/mnt/test_file",
"type": "FILE"
}
test_simple_str = json.dumps(test_simple_dict)
o1 = unmarshal(test_simple_dict, Input)
o2 = unmarshal(test_simple_str, Input)
self.assertTrue(isinstance(o1, Input))
self.assertTrue(isinstance(o2, Input))
self.assertEqual(o1, o2)
self.assertEqual(o1.as_dict(), test_simple_dict)
self.assertEqual(o1.as_json(), test_simple_str)
test_complex_dict = {
"name":
"test",
"inputs": [{
"url": "file:///storage/inputs/test_file",
"path": "/mnt/test_file",
"type": "FILE"
}],
"outputs": [{
"url": "file:///storage/outputs/test_outputfile",
"path": "/mnt/test_outputfile",
"type": "FILE"
}],
"executors": [{
"image": "alpine",
"command": ["echo", "hello"],
"env": {
"HOME": "/home/"
}
}],
"logs": [{
"start_time":
"2017-10-09T17:05:00.0Z",
"end_time":
"2017-10-09T17:40:30.0Z",
"metadata": {
"testmeta": "testvalue"
},
"logs": [{
"start_time": "2017-10-09T17:06:30.0Z",
"end_time": "2017-10-09T17:39:50.0Z",
"exit_code": 0,
}],
"outputs": [{
"url": "file:///storage/outputs/test_outputfile",
"path": "/mnt/test_outputfile",
"size_bytes": "3333"
}]
}],
"creation_time":
"2017-10-09T17:00:00.0Z"
}
test_complex_str = json.dumps(test_complex_dict)
o1 = unmarshal(test_complex_dict, Task)
o2 = unmarshal(test_complex_str, Task)
self.assertTrue(isinstance(o1, Task))
self.assertTrue(isinstance(o2, Task))
self.assertEqual(o1, o2)
expected = test_complex_dict.copy()
# handle expected conversions
expected["logs"][0]["outputs"][0]["size_bytes"] = int(
expected["logs"][0]["outputs"][0]["size_bytes"])
expected["logs"][0]["start_time"] = dateutil.parser.parse(
expected["logs"][0]["start_time"])
expected["logs"][0]["end_time"] = dateutil.parser.parse(
expected["logs"][0]["end_time"])
expected["logs"][0]["logs"][0]["start_time"] = dateutil.parser.parse(
expected["logs"][0]["logs"][0]["start_time"])
expected["logs"][0]["logs"][0]["end_time"] = dateutil.parser.parse(
expected["logs"][0]["logs"][0]["end_time"])
expected["creation_time"] = dateutil.parser.parse(
expected["creation_time"])
self.assertEqual(o1.as_dict(), expected)
