def get_file(self, userfile, size, download_dir, overwrite=False, callback=None, block=4096):
"""Gets the content of a file previously uploaded by the user.
:Args:
- userfile (:class:`.UserFile`): The userfile reference for the
file to be downloaded. Could be generated from a
:meth:`.FileManager.list_files()` call or file query.
Must have 'url' attr.
- size (int): Size of the file in byte to be downloaded
(see :meth:`.props_file()`).
- download_dir (str): The full path to the destination directory.
:Kwargs:
- overwrite (bool): Whether to overwrite a destination file if it
already exists. The default is ``False``.
- callback (func): A function to be called to report download progress.
The function must take three arguments: the percent downloaded (float), the
bytes downloaded (float), and the total bytes to be downloaded (float).
- block (int): The amount of data downloaded in each block - determines
the frequency with which the callback is called. Default is 4096.
:Returns:
- :class:`.Response` with the GET response, however this is not
required if the call was successful.
- :class:`.Response` with :class:`.RestCallException` if the
download failed.
"""
if not isinstance(userfile, UserFile):
return Response(False, RestCallException(TypeError, "Not a valid UserFile", None))
self._log.debug(
"get_file, file={0}, size={1}, "
"download_dir={2}, overwrite={3}, block={4}".format(userfile, size, download_dir, overwrite, block)
)
url = userfile.url
self._log.debug("Get file url: {0}".format(url))
try:
get_resp = rest_client.download(
self._auth,
url,
self.headers,
download_dir,
size,
overwrite=overwrite,
block_size=block,
callback=callback,
)
except RestCallException as exp:
return Response(False, exp)
else:
# TODO: Reconfigure original userfile?
return Response(True, get_resp)
def test_rest_client_download(self, mock_open, mock_call, mock_path):
"""Test download"""
def _callback(progress, data, total):
self.assertEqual(progress, 0.0)
self.assertIsInstance(progress, float)
self.assertIsInstance(data, int)
self.assertIsInstance(total, int)
auth = mock.create_autospec(Credentials)
mock_path.return_value = True
val = rest_client.download(auth,
"http://host//something//test?a=b",
{},
"c:\\test",
10,
False,
block_size=12345,
callback=_callback)
self.assertFalse(mock_call.called)
self.assertTrue(val)
val = rest_client.download(auth,
"http://host//something//test?a=b",
{},
"c:\\test",
0,
True,
callback=_callback)
mock_call.assert_called_with(auth,
'GET',
"http://host//something//test?a=b",
headers={},
stream=True)
mock_path.return_value = False
val = rest_client.download(auth,
"http://host//something//test?a=b",
{},
"c:\\test",
500,
False,
ext=".jpg",
callback=_callback)
mock_call.assert_called_with(auth,
'GET',
"http://host//something//test?a=b",
headers={},
stream=True)
mock_open.assert_called_with("c:\\test\\test.jpg", "wb")
val = rest_client.download(auth,
"http://host//something//test?a=b",
{},
"c:\\test",
500,
False,
ext=".jpg",
f_name="another.png")
mock_call.assert_called_with(auth,
'GET',
"http://host//something//test?a=b",
headers={},
stream=True)
mock_open.assert_called_with("c:\\test\\another.png", "wb")
mock_open.side_effect = IOError('oops!')
with self.assertRaises(RestCallException):
rest_client.download(auth,
"http://host//something//test?a=b",
{},
"c:\\test",
0,
True)
mock_call.side_effect = RestCallException(None, "Boom!", None)
with self.assertRaises(RestCallException):
rest_client.download(auth,
"http://host//something//test?a=b",
{},
"c:\\test",
0,
True)
def test_rest_client_download(self, mock_open, mock_call, mock_path):
"""Test download"""
def _callback(progress, data, total):
self.assertEqual(progress, 0.0)
self.assertIsInstance(progress, float)
self.assertIsInstance(data, int)
self.assertIsInstance(total, int)
auth = mock.create_autospec(Credentials)
mock_path.return_value = True
val = rest_client.download(auth,
"http://host//something//test?a=b", {},
"c:\\test",
10,
False,
block_size=12345,
callback=_callback)
self.assertFalse(mock_call.called)
self.assertTrue(val)
val = rest_client.download(auth,
"http://host//something//test?a=b", {},
"c:\\test",
0,
True,
callback=_callback)
mock_call.assert_called_with(auth,
'GET',
"http://host//something//test?a=b",
headers={},
stream=True)
mock_path.return_value = False
val = rest_client.download(auth,
"http://host//something//test?a=b", {},
"c:\\test",
500,
False,
ext=".jpg",
callback=_callback)
mock_call.assert_called_with(auth,
'GET',
"http://host//something//test?a=b",
headers={},
stream=True)
mock_open.assert_called_with("c:\\test\\test.jpg", "wb")
val = rest_client.download(auth,
"http://host//something//test?a=b", {},
"c:\\test",
500,
False,
ext=".jpg",
f_name="another.png")
mock_call.assert_called_with(auth,
'GET',
"http://host//something//test?a=b",
headers={},
stream=True)
mock_open.assert_called_with("c:\\test\\another.png", "wb")
mock_open.side_effect = IOError('oops!')
with self.assertRaises(RestCallException):
rest_client.download(auth, "http://host//something//test?a=b", {},
"c:\\test", 0, True)
mock_call.side_effect = RestCallException(None, "Boom!", None)
with self.assertRaises(RestCallException):
rest_client.download(auth, "http://host//something//test?a=b", {},
"c:\\test", 0, True)
def get_output_file(
self, download_dir, size, overwrite, job_id=None, fname=None, url=None, callback=None, block=4096
):
"""
Gets the content of a file created in a job.
Either ``url``, or both ``job_id`` and ``fname`` must be set.
If all three are set, job_id & fname will be used.
:Args:
- download_dir (str): The full path to the directory where the
output will be downloaded to.
- size (int): The size in bytes of the file to be downloaded.
Used for progress reporting.
- overwrite (bool): Whether to overwrite an existing file if
present.
:Kwargs:
- job_id (str): The ID of the job whose output will be downloaded.
The default is None.
- fname (str): The name of the output file to be downloaded.
- url (str): The URL directly to the file to be downloaded.
The default is None.
- callback (func): A function to be called to report download progress.
The function must take three arguments: the percent downloaded (float), the
bytes downloaded (float), and the total bytes to be downloaded (float).
- block (int): The amount of data downloaded in each block - determines
the frequency with which the callback is called. Default is 4096.
:Returns:
- :class:`.Response` with the GET response, however this is not
required if the call was successful.
- :class:`.Response` with :exc:`AttributeError` if the correct URL
arguments are not supplied.
- :class:`.Response` with :class:`.RestCallException` if the
download failed.
"""
self._log.debug(
"get_output_file, download_dir={dd}, size={sz}, "
"overwrite={ow}, job_id={ji}, fname={fn}, "
"url={ur}, block={bl}".format(dd=download_dir, sz=size, ow=overwrite, ji=job_id, fn=fname, ur=url, bl=block)
)
name = fname if fname else None
if job_id and name:
url = self.url("jobs/{jobid}/outputs/files/{name}")
url = url.format(jobid=job_id, name=name)
elif url is None:
return Response(
False, RestCallException(AttributeError, "Either output url or job id and filename required.", None)
)
try:
get_resp = rest_client.download(
self._auth,
url,
self.headers,
download_dir,
size,
overwrite,
f_name=name,
block_size=block,
callback=callback,
)
except RestCallException as exp:
return Response(False, exp)
else:
return Response(True, get_resp)
def get_output(
self, download_dir, size, fname, overwrite, job_id=None, otype="output", url=None, callback=None, block=4096
):
"""
Gets the content of the job output or its thumbnail.
Either ``url``, or both ``job_id`` and ``otype`` must be set.
If all three are set, URL will be used.
:Args:
- download_dir (str): The full path to the directory where the
output will be downloaded to.
- size (int): The size in bytes of the file to be downloaded.
Used for progress reporting.
- fname (str): The name of the output file to be downloaded.
- overwrite (bool): Whether to overwrite an existing file if
present.
:Kwargs:
- job_id (str): The ID of the job whose output will be downloaded.
The default is None.
- otype (str): The type of output to be downloaded, must be a
string in ``['output', 'preview']``.
- url (str): The URL directly to the file to be downloaded. If
supplied, ``job_id`` and ``otype`` will not be used.
The default is None.
- callback (func): A function to be called to report download progress.
The function must take three arguments: the percent downloaded (float), the
bytes downloaded (float), and the total bytes to be downloaded (float).
- block (int): The amount of data downloaded in each block - determines
the frequency with which the callback is called. Default is 4096.
:Returns:
- :class:`.Response` with the GET response, however this is not
required if the call was successful.
- :class:`.Response` with :exc:`AttributeError` if the correct
URL arguments are not supplied.
- :class:`.Response` with :class:`.RestCallException` if the
download failed.
"""
self._log.debug(
"get_output, download_dir={dd}, size={sz}, fname={fn}, "
"overwrite={ow}, job_id={ji}, url={ur}, otype={ot}, block={bl}".format(
dd=download_dir, sz=size, fn=fname, ow=overwrite, ji=job_id, ur=url, ot=otype, bl=block
)
)
if not url and job_id:
if otype.lower() not in ["output", "preview"]:
return Response(False, RestCallException(ValueError, "output type must be 'output' or 'preview'", None))
url = self.url("jobs/{jobid}/outputs/{type}").format(jobid=job_id, type=otype)
elif not url and not job_id:
return Response(False, RestCallException(AttributeError, "Either job_id or url must be set", None))
try:
get_resp = rest_client.download(
self._auth,
url,
self.headers,
download_dir,
size,
overwrite,
f_name=fname,
block_size=block,
callback=callback,
)
except RestCallException as exp:
return Response(False, exp)
else:
return Response(True, get_resp)
