def test_write_file_locally(mock_makedirs, mock_join, mock_isdir):
mock_isdir.return_value = False
mock_join.return_value = "/User/is/fake/myfile.pdf"
with patch("builtins.open", mock_open()) as m:
write_file_locally(b"something", "/User/is/fake/myfile.pdf")
assert m.call_args_list == [call('/User/is/fake/myfile.pdf', 'wb')]
handle = m()
handle.write.assert_called_once_with(b'something')
mock_makedirs.assert_called_once_with("/User/is/fake")
def read(self, table: Union[GemTable, Tuple[str, int]], local_path: str):
"""
Read the Table file from S3.
If a Table object is not provided, retrieve it using the provided table and version ids.
"""
# NOTE: this uses the pre-signed S3 download url. If we need to download larger files,
# we have other options available (using multi-part downloads in parallel , for example).
if isinstance(table, Tuple):
table = self.get(table[0], table[1])
data_location = table.download_url
data_location = rewrite_s3_links_locally(data_location, self.session.s3_endpoint_url)
response = requests.get(data_location)
write_file_locally(response.content, local_path)
def download(self, file_link: FileLink, local_path: str):
"""
Download the file associated with a given FileLink to the local computer.
Parameters
----------
file_link: FileLink
Resource referencing the external file.
local_path: str
Path to save file on the local computer. If `local_path` is a directory,
then the filename of this FileLink object will be appended to the path.
"""
directory, filename = os.path.split(local_path)
if not filename:
filename = file_link.filename
local_path = os.path.join(directory, filename)
# The "/content-link" route returns a pre-signed url to download the file.
content_link_path = file_link.url + '/content-link'
content_link_response = self.session.get_resource(content_link_path)
pre_signed_url = content_link_response['pre_signed_read_link']
download_response = requests.get(pre_signed_url)
write_file_locally(download_response.content, local_path)
def test_write_file_locally_fails_with_no_filename():
with pytest.raises(ValueError):
write_file_locally(b"anything", "/user/is/")
def read(self, local_path):
"""[DEPRECATED] Use TableCollection.read() instead.""" # noqa: D402
data_location = self.download_url
data_location = rewrite_s3_links_locally(data_location)
response = requests.get(data_location)
write_file_locally(response.content, local_path)
def read(self, local_path):
"""Read the Table file from S3."""
data_location = self.download_url
data_location = rewrite_s3_links_locally(data_location)
response = requests.get(data_location)
write_file_locally(response.content, local_path)
