def package_from_object(cls, obj, factory): """Create a package from a GCS object""" filename = posixpath.basename(obj.name) if obj.metadata is None: return None name = obj.metadata.get("name") version = obj.metadata.get("version") if name is None or version is None: return None metadata = Package.read_metadata(obj.metadata) return factory(name, version, filename, obj.updated, path=obj.name, **metadata)
def list(self, factory=Package): # List does not return metadata :( for blob_properties in self.container_client.list_blobs( name_starts_with=self.path_prefix): blob_client = self.container_client.get_blob_client( blob=blob_properties.name) metadata = blob_client.get_blob_properties() yield factory(metadata.metadata["name"], metadata.metadata["version"], posixpath.basename(blob_properties.name), blob_properties.last_modified, path=blob_properties.name, **Package.read_metadata(metadata.metadata))
def test_upload(self): """Uploading package sets metadata and sends to S3""" package = make_package(requires_python="3.6") datastr = b"foobar" data = BytesIO(datastr) self.storage.upload(package, data) key = list(self.bucket.objects.all())[0].Object() contents = BytesIO() key.download_fileobj(contents) self.assertEqual(contents.getvalue(), datastr) self.assertEqual(key.metadata["name"], package.name) self.assertEqual(key.metadata["version"], package.version) self.assertEqual(key.metadata["summary"], package.summary) self.assertDictContainsSubset(package.get_metadata(), Package.read_metadata(key.metadata))
def package_from_object(cls, obj, factory): """ Create a package from a S3 object """ filename = posixpath.basename(obj.key) name = obj.metadata.get("name") version = obj.metadata.get("version") metadata = Package.read_metadata(obj.metadata) # We used to not store metadata. This is for backwards # compatibility if name is None or version is None: try: name, version = parse_filename(filename) except ValueError: LOG.warning("S3 file %s has no package name", obj.key) return None return factory( name, version, filename, obj.last_modified, path=obj.key, **metadata )
def list(self, factory=Package): try: headers, objects = self.client.get_container( self.container, headers={'user-agent': USER_AGENT}) except ClientException as e: if e.http_status == 404: LOG.warning('Container was removed') create_container(self.client, self.container, self.storage_policy) return LOG.error('Error while listing container: %s', e) raise HTTPInternalServerError() for obj_info in objects: object_path = obj_info['name'] if object_path.endswith('.meta'): continue try: name, version, filename = object_path.split('/', 3) except ValueError: LOG.warning('The object is not like a package: %s', obj_info) continue last_modified = datetime.strptime(obj_info['last_modified'], '%Y-%m-%dT%H:%M:%S.%f') object_meta_path = self.path_to_meta_path(object_path) try: # Get package metadata from object headers, data = self.client.get_object( self.container, object_meta_path, headers={'user-agent': USER_AGENT}) except ClientException as e: if e.http_status != 404: LOG.warning('Can\'t get package metadata "%s": %s', object_meta_path, e) continue else: # metadata stored in old place metadata = self._get_old_metadata(object_path) else: metadata = json.loads(data) metadata = Package.read_metadata(metadata or {}) yield factory(name, version, filename, last_modified, **metadata)