def test_rest_client_get(self, mock_call):
"""Test get"""
auth = mock.create_autospec(Credentials)
rest_client.get(auth, "http://test", {})
mock_call.assert_called_with(auth,
'GET',
"http://test",
headers={},
params=None)
rest_client.get(auth, "http://test", {}, params={'a':1})
mock_call.assert_called_with(auth,
'GET',
"http://test",
headers={},
params={'a':1})
mock_call.return_value.json.side_effect = ValueError("Value Error!")
with self.assertRaises(RestCallException):
rest_client.get(auth, "http://test", {})
mock_call.side_effect = RestCallException(None, "Boom!", None)
with self.assertRaises(RestCallException):
rest_client.get(auth, "http://test", {})
def test_rest_client_get(self, mock_call):
"""Test get"""
auth = mock.create_autospec(Credentials)
rest_client.get(auth, "http://test", {})
mock_call.assert_called_with(auth,
'GET',
"http://test",
headers={},
params=None)
rest_client.get(auth, "http://test", {}, params={'a': 1})
mock_call.assert_called_with(auth,
'GET',
"http://test",
headers={},
params={'a': 1})
mock_call.return_value.json.side_effect = ValueError("Value Error!")
with self.assertRaises(RestCallException):
rest_client.get(auth, "http://test", {})
mock_call.side_effect = RestCallException(None, "Boom!", None)
with self.assertRaises(RestCallException):
rest_client.get(auth, "http://test", {})
def list_files(self):
"""
Lists the users files.
This refers to files loaded by the user (sometimes referred to as
assets) as distinct from task or job outputs generated during
task processing.
:Returns:
- A :class:`.Response` object containing a list of files as
dictionaries, with data:
``['id','name','lastModifiedBy','lastModifiedTime','link']``
- If the call failed or if the response is incomplete/malformed
a :class:`.Response` object with a :class:`.RestCallException`.
"""
self._log.debug("list_files, no params")
url = self.url("files")
try:
resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
if "files" not in resp or not isinstance(resp["files"], list):
return Response(False, RestCallException(KeyError, "files key not in response message", resp))
return Response(True, resp["files"])
def get_pool(self, pool_id=None, url=None):
"""
Gets information about a pool.
Pool info can be retrieved by supplying **either** the pool's ID
**or** a URL to the pool. If both are supplied, URL is used.
:Kwargs:
- pool_id (str): ID of the pool on which info is requested.
- url (str): A complete URL to the pool info.
:Returns:
- A :class:`.Response` object containing the pool details as a
dictionary, if successful. Otherwise the Response will
contain the :exc:`.RestCallException`.
- :class:`.RestCallException` if neither pool ID or URL are
supplied.
- :class:`.RestCallException` if pool details dictionary is
malformed / missing necessary keys
"""
self._log.debug("get_pool, pool_id={0}, url={1}".format(pool_id, url))
if not url and pool_id:
url = self.url("pools/{poolid}").format(poolid=pool_id)
elif not url and not pool_id:
return Response(False, RestCallException(AttributeError, "Either pool_id or url must be set", None))
try:
get_resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
else:
return Response(True, get_resp)
def list_output_files(self, job_id):
"""Lists the intermediate output files produced during a job.
:Args:
- job_id (str): The ID of the job whose outputs will be listed.
:Returns:
- A list of the outputs represented as dictionaries, each with the
'name' and 'type' of the output as well as a download 'link'.
Contained in a :class:`.Response`. If the call failed, Response
will contain the :class:`.RestCallException`.
"""
self._log.debug("list_output_files, job_id={0}".format(job_id))
url = self.url("jobs/{jobid}/outputs/files").format(jobid=job_id)
try:
get_resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
outputs = []
if not "outputs" in get_resp or not isinstance(get_resp["outputs"], list):
return Response(False, RestCallException(KeyError, "outputs key not in response message", get_resp))
for output in get_resp["outputs"]:
outputs.append(
{
"name": output.get("name"),
"link": output.get("link", {"href": None}).get("href"),
"type": output.get("kind"),
}
)
return Response(True, outputs)
def list_outputs(self, job_id):
"""
Lists the output files produced by a job.
This method will only list final outputs of the job, created by a
Merge Task.
To retrieve a list of all files created by all tasks of the job use
:meth:`.list_output_files()`.
Can be used to ascertain the output filenames before calling the
generic output download URL.
:Args:
- job_id (str): ID of the job whose outputs will be listed.
:Returns:
- A list of the outputs represented as dictionaries, each with
the 'name' and 'type' of the output as well as a download
'link'. Contained in a :class:`.Response`. If the call failed,
Response will contain the :class:`.RestCallException`.
"""
self._log.debug("list_outputs, job_id={0}".format(job_id))
url = self.url("jobs/{jobid}/outputs").format(jobid=job_id)
try:
get_resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
outputs = []
if not "jobOutputs" in get_resp or not isinstance(get_resp["jobOutputs"], list):
return Response(False, RestCallException(KeyError, "jobOutputs key not in response message", get_resp))
# Reformat output dictionary to be more manageable
for output in get_resp["jobOutputs"]:
outputs.append(
{
"name": output.get("name"),
"link": output.get("link", {"href": None}).get("href"),
"type": output.get("kind"),
}
)
return Response(True, outputs)
def get_log(self, job_id, start=None, max_lines=100):
"""
Gets log messages for a job.
These are the Batch Apps system logs, rather than those of the
application.
:Args:
- job_id (str): The ID of the job on which to download the logs.
:Kwargs:
- start (str): The start time from which the logs will be
downloaded. If not specified, the default is from the
beginning of the job.
- max_lines (int): The max number of logging messages to retrieve.
The default is 100. If set to ``None``, all messages from start
time will be retrieved.
:Returns:
- A :class:`.Response` object with a dictionary containing the
timestamp of the most recent message returned and a list of
the log messages, represented as dictionaries, with the message
text, timestamp and task id that the message applies to.
- If the call failed, the response contains the
:class:`.RestCallException`.
"""
self._log.debug("get_log, job_id={0}, start={1}, max_lines={2}".format(job_id, start, max_lines))
url = self.url("jobs/{jobid}/log").format(jobid=job_id)
get_params = {}
if start:
get_params["since"] = str(start)
if max_lines:
get_params["maxResults"] = int(max_lines)
try:
get_resp = rest_client.get(self._auth, url, self.headers, params=get_params)
except RestCallException as exp:
return Response(False, exp)
else:
# TODO: Check for specific keys here.
return Response(True, get_resp)
def list_pools(self):
"""Lists the users pools.
:Returns:
- :class:`.Response` object containing success of call. If
successful, the ``Response.result`` will contain a list of
pool dictionaries. If failed, ``Response.result`` will
hold the :exc:`.RestCallException`.
"""
self._log.debug("list_pools, no params")
url = self.url("pools")
try:
get_resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
else:
return Response(True, get_resp)
def get_job(self, job_id=None, url=None):
"""
Gets information about a job.
Job info can be retrieved by supplying **either** the job's ID
**or** a URL to the job. If both are supplied, URL is used.
:Kwargs:
- job_id (str): ID of the job on which info is requested.
- url (str): A complete URL to the job info.
:Returns:
- A :class:`.Response` object containing the job details as a
dictionary, if successful. Otherwise the Response will
contain the :exc:`.RestCallException`.
:Raises:
- :class:`.RestCallException` if neither job ID or URL are
supplied.
- :class:`.RestCallException` if job details dictionary is
malformed / missing necessary keys
"""
self._log.debug("get_job, job_id={0}, url={1}".format(job_id, url))
if not url and job_id:
url = self.url("jobs/{jobid}").format(jobid=job_id)
elif not url and not job_id:
return Response(False, RestCallException(AttributeError, "Either job_id or url must be set", None))
try:
get_resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
else:
if utils.valid_keys(get_resp, ["id", "name", "type"]):
return Response(True, get_resp)
return Response(False, RestCallException(KeyError, "incorrectly formatted job response", get_resp))
def list_task_outputs(self, job_id, task):
"""Lists the output files produced by a task.
:Args:
- job_id (str): The ID of the job the task outputs belong to.
- task (int, str): The ID of the task whose outputs will be listed.
:Returns:
- A list of the outputs represented as dictionaries, each with the
'name' and 'type' of the output as well as a download 'link'.
Contained in a :class:`.Response`. If the call failed,
Response will contain the :class:`.RestCallException`.
"""
self._log.debug("list_task_outputs, job_id={0}, " "task={1}".format(job_id, task))
url = self.url("jobs/{jobid}/tasks/{taskid}/outputs/files")
url = url.format(url, jobid=job_id, taskid=task)
try:
resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
outputs = []
if "outputs" not in resp or not isinstance(resp["outputs"], list):
return Response(False, RestCallException(KeyError, "outputs key not in response message", resp))
for output in resp["outputs"]:
outputs.append(
{
"name": output.get("name"),
"link": output.get("link", {"href": None}).get("href"),
"type": output.get("kind"),
}
)
return Response(True, outputs)
def list_jobs(self, index=0, per_call=10, name=None):
"""Lists the users jobs.
:Kwargs:
- index (int): The starting index from which the list of jobs will
be returned. The default is 0, i.e. return all jobs from the
start.
- per_call (int): The number of job entries from ``index`` to
return. The default is 10.
- name (str): Return only the jobs whose name contains the given
string. The default is None.
:Returns:
- :class:`.Response` object containing success of call. If
successful, the ``Response.result`` will contain a list of
jobs as dictionaries. If failed, ``Response.result`` will
hold the :exc:`.RestCallException`.
"""
self._log.debug("list_jobs, index={0}, per_call={1}, name={2}".format(index, per_call, name))
url = self.url("jobs")
req_set = {"maxResults": per_call, "startIndex": index}
if name:
req_set["nameContains"] = str(name)
try:
get_resp = rest_client.get(self._auth, url, self.headers, params=req_set)
except RestCallException as exp:
return Response(False, exp)
else:
if "jobs" in get_resp:
return Response(True, get_resp)
return Response(False, RestCallException(KeyError, "Key not in response message", get_resp))
def list_tasks(self, job_id=None, url=None):
"""
List the tasks of a job.
Either ``job_id`` *or* ``url`` must be set. If both are set ``url``
will be used.
:Kwargs:
- job_id (str): ID of of the job to list the tasks for.
The default is None.
- url (str): Direct URL to the task list of the job
(supplied by :meth:`.BatchAppsApi.get_job()`)
:Returns:
- A :class:`.Response` object containing the list of task
dictionaries if the call is successful.
- A :class:`.Response` object containing the
:class:`.RestCallException` is the call failed.
"""
self._log.debug("list_tasks, job_id={0}, url={1}".format(job_id, url))
if not url and job_id:
url = self.url("jobs/{jobid}/tasks").format(jobid=job_id)
elif not url and not job_id:
return Response(False, RestCallException(AttributeError, "Either job_id or url must get set", None))
try:
resp = rest_client.get(self._auth, url, self.headers)
except RestCallException as exp:
return Response(False, exp)
else:
if "tasks" not in resp or not isinstance(resp["tasks"], list):
return Response(False, RestCallException(KeyError, "tasks key not in response message", resp))
return Response(True, resp["tasks"])