def get_deleted_bundle_metadata_document(replica: Replica, key: str) -> dict: """ Build the bundle metadata document assocated with a non-existent key. """ fqid = BundleFQID.from_key(key) return dict(event_type="DELETE", bundle_info=dict(uuid=fqid.uuid, version=fqid.version))
def __iter__(self): for key in self._keys(): fqid = BundleFQID.from_key(key) if fqid.uuid != self.bundle_info['uuid']: for bundle_fqid in self._living_fqids_in_bundle_info(): yield bundle_fqid self._init_bundle_info(fqid) else: if not fqid.is_fully_qualified(): self.bundle_info['contains_unversioned_tombstone'] = True else: self.bundle_info['fqids'][fqid] = isinstance( fqid, BundleTombstoneID) for bundle_fqid in self._living_fqids_in_bundle_info(): yield bundle_fqid
def build_bundle_metadata_document(replica: Replica, key: str) -> dict: """ This returns a JSON document with bundle manifest and metadata files suitable for JMESPath filters. """ handle = Config.get_blobstore_handle(replica) manifest = json.loads(handle.get(replica.bucket, key).decode("utf-8")) fqid = BundleFQID.from_key(key) bundle_info = dict(uuid=fqid.uuid, version=fqid.version) if key.endswith(TOMBSTONE_SUFFIX): return dict(event_type="TOMBSTONE", bundle_info=bundle_info, **manifest) else: lock = threading.Lock() files: dict = defaultdict(list) def _read_file(file_metadata): blob_key = "blobs/{}.{}.{}.{}".format( file_metadata['sha256'], file_metadata['sha1'], file_metadata['s3-etag'], file_metadata['crc32c'], ) contents = handle.get(replica.bucket, blob_key).decode("utf-8") try: file_info = json.loads(contents) except json.decoder.JSONDecodeError: logging.info(f"{file_metadata['name']} not json decodable") else: # Modify name to avoid confusion with JMESPath syntax name = _dot_to_underscore_and_strip_numeric_suffix( file_metadata['name']) with lock: files[name].append(file_info) # TODO: Consider scaling parallelization with Lambda size with ThreadPoolExecutor(max_workers=4) as e: e.map(_read_file, [ file_metadata for file_metadata in manifest['files'] if file_metadata['content-type'].startswith("application/json") ]) return dict(event_type="CREATE", bundle_info=bundle_info, manifest=manifest, files=dict(files))
def dependencies_exist(source_replica: Replica, dest_replica: Replica, key: str): """ Given a source replica and manifest key, checks if all dependencies of the corresponding DSS object are present in dest_replica: - Given a file manifest key, checks if blobs exist in dest_replica. - Given a bundle manifest key, checks if file manifests exist in dest_replica. - Given a collection key, checks if all collection contents exist in dest_replica. Returns true if all dependencies exist in dest_replica, false otherwise. """ source_handle = Config.get_blobstore_handle(source_replica) dest_handle = Config.get_blobstore_handle(dest_replica) if key.endswith(TOMBSTONE_SUFFIX): return True elif key.startswith(FILE_PREFIX): file_id = FileFQID.from_key(key) file_manifest = get_json_metadata( entity_type="file", uuid=file_id.uuid, version=file_id.version, replica=source_replica, blobstore_handle=source_handle, max_metadata_size=max_syncable_metadata_size) blob_path = compose_blob_key(file_manifest) if exists(dest_replica, blob_path): return True elif key.startswith(BUNDLE_PREFIX): # head all file manifests bundle_id = BundleFQID.from_key(key) bundle_manifest = get_json_metadata( entity_type="bundle", uuid=bundle_id.uuid, version=bundle_id.version, replica=source_replica, blobstore_handle=source_handle, max_metadata_size=max_syncable_metadata_size) try: with ThreadPoolExecutor(max_workers=20) as e: futures = list() for file in bundle_manifest[BundleMetadata.FILES]: file_uuid = file[BundleFileMetadata.UUID] file_version = file[BundleFileMetadata.VERSION] futures.append( e.submit(get_json_metadata, entity_type="file", uuid=file_uuid, version=file_version, replica=dest_replica, blobstore_handle=source_handle, max_metadata_size=max_syncable_metadata_size)) for future in as_completed(futures): future.result() return True except Exception: pass elif key.startswith(COLLECTION_PREFIX): collection_id = CollectionFQID.from_key(key) collection_manifest = get_json_metadata( entity_type="collection", uuid=collection_id.uuid, version=collection_id.version, replica=source_replica, blobstore_handle=source_handle, max_metadata_size=max_syncable_metadata_size) try: verify_collection(contents=collection_manifest["contents"], replica=dest_replica, blobstore_handle=dest_handle) return True except Exception: pass else: raise NotImplementedError("Unknown prefix for key {}".format(key)) return False