def from_index(
cls,
field_types: CataloguedFieldTypes,
hit: JSON,
*,
coordinates: Optional[
DocumentCoordinates[CataloguedEntityReference]] = None
) -> 'Document':
if coordinates is None:
coordinates = DocumentCoordinates.from_hit(hit)
if 'contents' in hit['_source']:
file: JSON
content_descriptions = [
file['content_description']
for file in hit['_source']['contents']['files']
]
assert [] not in content_descriptions, 'Found empty list as content_description value'
source = cls.translate_fields(hit['_source'],
field_types[coordinates.entity.catalog],
forward=False)
# noinspection PyArgumentList
# https://youtrack.jetbrains.com/issue/PY-28506
self = cls(coordinates=coordinates,
version=hit.get('_version', 0),
contents=source.get('contents'),
**cls._from_source(source))
return self
def _transform_entity(self, entity: JSON) -> JSON:
fqid = entity['uuid'], entity['version']
return dict(size=(fqid, entity['size']),
file_format=entity['file_format'],
file_source=entity['file_source'],
is_intermediate=entity['is_intermediate'],
count=(fqid, 1),
content_description=entity['content_description'],
matrix_cell_count=(fqid, entity.get('matrix_cell_count')))
def from_json(cls,
*,
coordinates: C,
document: JSON,
version: Optional[InternalVersion],
**kwargs
) -> 'Aggregate':
self = super().from_json(coordinates=coordinates,
document=document,
version=version,
num_contributions=document['num_contributions'],
sources=map(DocumentSource.from_json, document['sources']),
bundles=document.get('bundles'))
assert isinstance(self, Aggregate)
return self
def from_json(cls,
*,
coordinates: C,
document: JSON,
version: Optional[InternalVersion],
**kwargs,
) -> 'Document':
# noinspection PyArgumentList
# https://youtrack.jetbrains.com/issue/PY-28506
self = cls(coordinates=coordinates,
version=version,
contents=document.get('contents'),
**kwargs)
assert document['entity_id'] == self.entity.entity_id
return self
def get_manifest(self, state: JSON) -> JSON:
partition = ManifestPartition.from_json(
state[self.partition_state_key])
auth = state.get('authentication')
result = self.service.get_manifest(
format_=ManifestFormat(state['format_']),
catalog=state['catalog'],
filters=Filters.from_json(state['filters']),
partition=partition,
authentication=None
if auth is None else Authentication.from_json(auth),
object_key=state['object_key'])
if isinstance(result, ManifestPartition):
assert not result.is_last, result
return {**state, self.partition_state_key: result.to_json()}
elif isinstance(result, Manifest):
return {
# The presence of this key terminates the step function loop
self.manifest_state_key:
result.to_json()
}
else:
assert False, type(result)
def _from_source(cls, source: JSON) -> Mapping[str, Any]:
return dict(super()._from_source(source),
num_contributions=source['num_contributions'],
bundles=source.get('bundles'))
def _deep_get(self, d: JSON, *path: str) -> Optional[JSON]:
if d is not None and path:
key, *path = path
return self._deep_get(d.get(key), *path)
else:
return d
def drs_path(self, manifest_entry: JSON) -> Optional[str]:
return manifest_entry.get('drs_path')