def fetch_file(self, resource_id: str) -> str: resource = self._client.get_object(resource_id) extension = splitext(resource_id)[1] path = create_tempfilename() + extension resource.download(path) self.log_debug('fetched file %s from %s', path, resource_id) return path
def store_objects(self, upload: Upload, compression: Optional[str] = None) -> Optional[str]: compression = compression or self._compression resource_id = f'{self._resource_id_source()}.tar.{compression}' name, objs, encoder = upload num_stored = 0 with removing(create_tempfilename(resource_id)) as path: with self._open_archive(path, 'w') as archive: with NamedTemporaryFile() as fobj: num_bytes = 0 for obj in objs: encoded = encoder(obj) fobj.write(encoded) num_bytes += len(encoded) num_stored += 1 if num_bytes > 0: fobj.seek(0) archive.add(fobj.name, name) if num_stored > 0: self._file_storage.store_file(resource_id, path) self.log_debug('stored %d objects at %s', num_stored, resource_id) return resource_id if num_stored > 0 else None
def store_objects(self, upload: Tuple[str, Iterable[dict]], resource_id: Optional[str] = None) -> Optional[str]: resource_id = self._to_resource_id(resource_id) name, objs = upload num_stored = 0 with removing(create_tempfilename()) as path: with self._open_archive(path, 'w') as archive: with NamedTemporaryFile() as fobj: num_bytes = 0 for obj in objs: serialized = to_json(obj) encoded = serialized.encode(self._encoding) fobj.write(encoded) fobj.write(b'\n') num_bytes += len(encoded) + 1 num_stored += 1 self.log_debug('stored obj %s', obj.get('_uid', '')) if num_bytes > 0: fobj.seek(0) archive.add(fobj.name, name) if num_stored > 0: self._file_storage.store_file(resource_id, path) self.log_debug('stored %d objects at %s', num_stored, resource_id) return resource_id if num_stored > 0 else None
def _given_resource(self, resource_id: str, name: str, lines: bytes): client = self._storage._file_storage._client mode = f'w:{Path(resource_id).suffix[1:]}' with removing(create_tempfilename(resource_id)) as buffer_path: with tarfile_open(buffer_path, mode) as archive: tarinfo = TarInfo(name) tarinfo.size = len(lines) archive.addfile(tarinfo, BytesIO(lines)) client.upload_object(buffer_path, resource_id)
def store_objects(self, objs: Iterable[dict]) -> Optional[str]: resource_id = str(uuid4()) num_stored = 0 with removing(create_tempfilename()) as path: with gzip_open(path, 'wb') as fobj: for obj in objs: serialized = to_json(obj) encoded = serialized.encode(self._encoding) fobj.write(encoded) fobj.write(b'\n') num_stored += 1 self.log_debug('stored email %s', obj.get('_uid')) if num_stored > 0: self._file_storage.store_file(resource_id, path) self.log_debug('stored %d objects at %s', num_stored, resource_id) return resource_id if num_stored > 0 else None
def test_creates_new_file(self): filename = temporary.create_tempfilename() self.assertFileDoesNotExist(filename)
def test_creates_different_files(self): filename1 = temporary.create_tempfilename() filename2 = temporary.create_tempfilename() self.assertNotEqual(filename1, filename2)
def test_creates_new_file_handling_bad_extension(self): filename = temporary.create_tempfilename('/foo/bar.gz') self.assertHasExtension(filename, '.gz') self.assertPathDoesNotContain(filename, '/foo/')
def test_creates_new_file_preserving_multi_extension(self): filename = temporary.create_tempfilename('foo.tar.gz') self.assertHasExtension(filename, '.tar.gz')
def when_creating_tempfile(self): filename = temporary.create_tempfilename() self._filenames.add(filename) return filename
def fetch_file(self, resource_id: str) -> str: path = create_tempfilename() self._client.get_blob_to_path(self._container, resource_id, path) self.log_debug('fetched file %s from %s', path, resource_id) return path