def fetch_file(self, resource_id: str) -> str:
resource = self._client.get_object(resource_id)
extension = splitext(resource_id)[1]
path = create_tempfilename() + extension
resource.download(path)
self.log_debug('fetched file %s from %s', path, resource_id)
return path
def store_objects(self,
upload: Upload,
compression: Optional[str] = None) -> Optional[str]:
compression = compression or self._compression
resource_id = f'{self._resource_id_source()}.tar.{compression}'
name, objs, encoder = upload
num_stored = 0
with removing(create_tempfilename(resource_id)) as path:
with self._open_archive(path, 'w') as archive:
with NamedTemporaryFile() as fobj:
num_bytes = 0
for obj in objs:
encoded = encoder(obj)
fobj.write(encoded)
num_bytes += len(encoded)
num_stored += 1
if num_bytes > 0:
fobj.seek(0)
archive.add(fobj.name, name)
if num_stored > 0:
self._file_storage.store_file(resource_id, path)
self.log_debug('stored %d objects at %s', num_stored, resource_id)
return resource_id if num_stored > 0 else None
def store_objects(self,
upload: Tuple[str, Iterable[dict]],
resource_id: Optional[str] = None) -> Optional[str]:
resource_id = self._to_resource_id(resource_id)
name, objs = upload
num_stored = 0
with removing(create_tempfilename()) as path:
with self._open_archive(path, 'w') as archive:
with NamedTemporaryFile() as fobj:
num_bytes = 0
for obj in objs:
serialized = to_json(obj)
encoded = serialized.encode(self._encoding)
fobj.write(encoded)
fobj.write(b'\n')
num_bytes += len(encoded) + 1
num_stored += 1
self.log_debug('stored obj %s', obj.get('_uid', ''))
if num_bytes > 0:
fobj.seek(0)
archive.add(fobj.name, name)
if num_stored > 0:
self._file_storage.store_file(resource_id, path)
self.log_debug('stored %d objects at %s', num_stored, resource_id)
return resource_id if num_stored > 0 else None
def _given_resource(self, resource_id: str, name: str, lines: bytes):
client = self._storage._file_storage._client
mode = f'w:{Path(resource_id).suffix[1:]}'
with removing(create_tempfilename(resource_id)) as buffer_path:
with tarfile_open(buffer_path, mode) as archive:
tarinfo = TarInfo(name)
tarinfo.size = len(lines)
archive.addfile(tarinfo, BytesIO(lines))
client.upload_object(buffer_path, resource_id)
def store_objects(self, objs: Iterable[dict]) -> Optional[str]:
resource_id = str(uuid4())
num_stored = 0
with removing(create_tempfilename()) as path:
with gzip_open(path, 'wb') as fobj:
for obj in objs:
serialized = to_json(obj)
encoded = serialized.encode(self._encoding)
fobj.write(encoded)
fobj.write(b'\n')
num_stored += 1
self.log_debug('stored email %s', obj.get('_uid'))
if num_stored > 0:
self._file_storage.store_file(resource_id, path)
self.log_debug('stored %d objects at %s', num_stored, resource_id)
return resource_id if num_stored > 0 else None
def test_creates_new_file(self):
filename = temporary.create_tempfilename()
self.assertFileDoesNotExist(filename)
def test_creates_different_files(self):
filename1 = temporary.create_tempfilename()
filename2 = temporary.create_tempfilename()
self.assertNotEqual(filename1, filename2)
def test_creates_new_file_handling_bad_extension(self):
filename = temporary.create_tempfilename('/foo/bar.gz')
self.assertHasExtension(filename, '.gz')
self.assertPathDoesNotContain(filename, '/foo/')
def test_creates_new_file_preserving_multi_extension(self):
filename = temporary.create_tempfilename('foo.tar.gz')
self.assertHasExtension(filename, '.tar.gz')
def when_creating_tempfile(self):
filename = temporary.create_tempfilename()
self._filenames.add(filename)
return filename
def fetch_file(self, resource_id: str) -> str:
path = create_tempfilename()
self._client.get_blob_to_path(self._container, resource_id, path)
self.log_debug('fetched file %s from %s', path, resource_id)
return path