def get_file(self, userfile, size, download_dir, overwrite=False, callback=None, block=4096): """Gets the content of a file previously uploaded by the user. :Args: - userfile (:class:`.UserFile`): The userfile reference for the file to be downloaded. Could be generated from a :meth:`.FileManager.list_files()` call or file query. Must have 'url' attr. - size (int): Size of the file in byte to be downloaded (see :meth:`.props_file()`). - download_dir (str): The full path to the destination directory. :Kwargs: - overwrite (bool): Whether to overwrite a destination file if it already exists. The default is ``False``. - callback (func): A function to be called to report download progress. The function must take three arguments: the percent downloaded (float), the bytes downloaded (float), and the total bytes to be downloaded (float). - block (int): The amount of data downloaded in each block - determines the frequency with which the callback is called. Default is 4096. :Returns: - :class:`.Response` with the GET response, however this is not required if the call was successful. - :class:`.Response` with :class:`.RestCallException` if the download failed. """ if not isinstance(userfile, UserFile): return Response(False, RestCallException(TypeError, "Not a valid UserFile", None)) self._log.debug( "get_file, file={0}, size={1}, " "download_dir={2}, overwrite={3}, block={4}".format(userfile, size, download_dir, overwrite, block) ) url = userfile.url self._log.debug("Get file url: {0}".format(url)) try: get_resp = rest_client.download( self._auth, url, self.headers, download_dir, size, overwrite=overwrite, block_size=block, callback=callback, ) except RestCallException as exp: return Response(False, exp) else: # TODO: Reconfigure original userfile? return Response(True, get_resp)
def test_rest_client_download(self, mock_open, mock_call, mock_path): """Test download""" def _callback(progress, data, total): self.assertEqual(progress, 0.0) self.assertIsInstance(progress, float) self.assertIsInstance(data, int) self.assertIsInstance(total, int) auth = mock.create_autospec(Credentials) mock_path.return_value = True val = rest_client.download(auth, "http://host//something//test?a=b", {}, "c:\\test", 10, False, block_size=12345, callback=_callback) self.assertFalse(mock_call.called) self.assertTrue(val) val = rest_client.download(auth, "http://host//something//test?a=b", {}, "c:\\test", 0, True, callback=_callback) mock_call.assert_called_with(auth, 'GET', "http://host//something//test?a=b", headers={}, stream=True) mock_path.return_value = False val = rest_client.download(auth, "http://host//something//test?a=b", {}, "c:\\test", 500, False, ext=".jpg", callback=_callback) mock_call.assert_called_with(auth, 'GET', "http://host//something//test?a=b", headers={}, stream=True) mock_open.assert_called_with("c:\\test\\test.jpg", "wb") val = rest_client.download(auth, "http://host//something//test?a=b", {}, "c:\\test", 500, False, ext=".jpg", f_name="another.png") mock_call.assert_called_with(auth, 'GET', "http://host//something//test?a=b", headers={}, stream=True) mock_open.assert_called_with("c:\\test\\another.png", "wb") mock_open.side_effect = IOError('oops!') with self.assertRaises(RestCallException): rest_client.download(auth, "http://host//something//test?a=b", {}, "c:\\test", 0, True) mock_call.side_effect = RestCallException(None, "Boom!", None) with self.assertRaises(RestCallException): rest_client.download(auth, "http://host//something//test?a=b", {}, "c:\\test", 0, True)
def get_output_file( self, download_dir, size, overwrite, job_id=None, fname=None, url=None, callback=None, block=4096 ): """ Gets the content of a file created in a job. Either ``url``, or both ``job_id`` and ``fname`` must be set. If all three are set, job_id & fname will be used. :Args: - download_dir (str): The full path to the directory where the output will be downloaded to. - size (int): The size in bytes of the file to be downloaded. Used for progress reporting. - overwrite (bool): Whether to overwrite an existing file if present. :Kwargs: - job_id (str): The ID of the job whose output will be downloaded. The default is None. - fname (str): The name of the output file to be downloaded. - url (str): The URL directly to the file to be downloaded. The default is None. - callback (func): A function to be called to report download progress. The function must take three arguments: the percent downloaded (float), the bytes downloaded (float), and the total bytes to be downloaded (float). - block (int): The amount of data downloaded in each block - determines the frequency with which the callback is called. Default is 4096. :Returns: - :class:`.Response` with the GET response, however this is not required if the call was successful. - :class:`.Response` with :exc:`AttributeError` if the correct URL arguments are not supplied. - :class:`.Response` with :class:`.RestCallException` if the download failed. """ self._log.debug( "get_output_file, download_dir={dd}, size={sz}, " "overwrite={ow}, job_id={ji}, fname={fn}, " "url={ur}, block={bl}".format(dd=download_dir, sz=size, ow=overwrite, ji=job_id, fn=fname, ur=url, bl=block) ) name = fname if fname else None if job_id and name: url = self.url("jobs/{jobid}/outputs/files/{name}") url = url.format(jobid=job_id, name=name) elif url is None: return Response( False, RestCallException(AttributeError, "Either output url or job id and filename required.", None) ) try: get_resp = rest_client.download( self._auth, url, self.headers, download_dir, size, overwrite, f_name=name, block_size=block, callback=callback, ) except RestCallException as exp: return Response(False, exp) else: return Response(True, get_resp)
def get_output( self, download_dir, size, fname, overwrite, job_id=None, otype="output", url=None, callback=None, block=4096 ): """ Gets the content of the job output or its thumbnail. Either ``url``, or both ``job_id`` and ``otype`` must be set. If all three are set, URL will be used. :Args: - download_dir (str): The full path to the directory where the output will be downloaded to. - size (int): The size in bytes of the file to be downloaded. Used for progress reporting. - fname (str): The name of the output file to be downloaded. - overwrite (bool): Whether to overwrite an existing file if present. :Kwargs: - job_id (str): The ID of the job whose output will be downloaded. The default is None. - otype (str): The type of output to be downloaded, must be a string in ``['output', 'preview']``. - url (str): The URL directly to the file to be downloaded. If supplied, ``job_id`` and ``otype`` will not be used. The default is None. - callback (func): A function to be called to report download progress. The function must take three arguments: the percent downloaded (float), the bytes downloaded (float), and the total bytes to be downloaded (float). - block (int): The amount of data downloaded in each block - determines the frequency with which the callback is called. Default is 4096. :Returns: - :class:`.Response` with the GET response, however this is not required if the call was successful. - :class:`.Response` with :exc:`AttributeError` if the correct URL arguments are not supplied. - :class:`.Response` with :class:`.RestCallException` if the download failed. """ self._log.debug( "get_output, download_dir={dd}, size={sz}, fname={fn}, " "overwrite={ow}, job_id={ji}, url={ur}, otype={ot}, block={bl}".format( dd=download_dir, sz=size, fn=fname, ow=overwrite, ji=job_id, ur=url, ot=otype, bl=block ) ) if not url and job_id: if otype.lower() not in ["output", "preview"]: return Response(False, RestCallException(ValueError, "output type must be 'output' or 'preview'", None)) url = self.url("jobs/{jobid}/outputs/{type}").format(jobid=job_id, type=otype) elif not url and not job_id: return Response(False, RestCallException(AttributeError, "Either job_id or url must be set", None)) try: get_resp = rest_client.download( self._auth, url, self.headers, download_dir, size, overwrite, f_name=fname, block_size=block, callback=callback, ) except RestCallException as exp: return Response(False, exp) else: return Response(True, get_resp)