def list_bundles(self, catalog: CatalogName, source: Union[str, SourceRef],
prefix: str) -> List[SourcedBundleFQID]:
validate_uuid_prefix(prefix)
plugin = self.repository_plugin(catalog)
if isinstance(source, str):
source = plugin.resolve_source(source)
else:
assert isinstance(source, SourceRef), source
return plugin.list_bundles(source, prefix)
def __attrs_post_init__(self):
validate_uuid_prefix(self.common)
assert ':' not in self.common, self.common
if self.partition:
assert isinstance(self.partition, int), self.partition
# Version 4 UUIDs specify fixed bits in the third dash-seperated
# group. To ensure that any concatenation of common and
# partition_prefix is a valid UUID prefix, we restrict the number of
# characters from the concatenation to be within the first
# dash-seperated group.
reject(
len(self.common) + self.partition > 8,
'Invalid common prefix and partition length', self)
def list_links_ids(self, prefix: str) -> List[BundleFQID]:
validate_uuid_prefix(prefix)
current_bundles = self._query_latest_version(f'''
SELECT links_id, version
FROM {self._full_table_name('links')}
WHERE STARTS_WITH(links_id, '{prefix}')
''',
group_by='links_id')
return [
BundleFQID(uuid=row['links_id'],
version=row['version'].strftime(self.timestamp_format))
for row in current_bundles
]
def list_bundles(self, source: CannedSourceRef, prefix: str) -> List[CannedBundleFQID]:
self._assert_source(source)
prefix = source.spec.prefix.common + prefix
validate_uuid_prefix(prefix)
log.info('Listing bundles with prefix %r in source %r.', prefix, source)
bundle_fqids = []
for link in self.staging_area(source.spec).links.values():
if link.uuid.startswith(prefix):
bundle_fqids.append(SourcedBundleFQID(source=source,
uuid=link.uuid,
version=link.version))
log.info('There are %i bundle(s) with prefix %r in source %r.',
len(bundle_fqids), prefix, source)
return bundle_fqids
def partition_prefixes(self) -> Iterator[str]:
"""
>>> list(Prefix.parse('/0').partition_prefixes())
['']
>>> list(Prefix.parse('/1').partition_prefixes())
['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
>>> len(list(Prefix.parse('/2').partition_prefixes()))
256
"""
partition_prefixes = map(
''.join, product('0123456789abcdef', repeat=self.partition))
for partition_prefix in partition_prefixes:
validate_uuid_prefix(self.common + partition_prefix)
yield partition_prefix
def parse(cls, prefix: str) -> 'Prefix':
"""
>>> Prefix.parse('aa/1')
Prefix(common='aa', partition=1)
>>> p = Prefix.parse('a')
>>> print(p.partition)
None
>>> p.effective.partition == config.partition_prefix_length
True
>>> Prefix.parse('aa/')
Traceback (most recent call last):
...
azul.RequirementError: ('Prefix source cannot end in a delimiter.', 'aa/', '/')
>>> Prefix.parse('8f538f53/1').partition_prefixes() # doctest: +NORMALIZE_WHITESPACE
Traceback (most recent call last):
...
azul.RequirementError: ('Invalid common prefix and partition length',
Prefix(common='8f538f53', partition=1))
>>> list(Prefix.parse('8f538f53/0').partition_prefixes())
['']
"""
source_delimiter = '/'
reject(prefix.endswith(source_delimiter),
'Prefix source cannot end in a delimiter.', prefix,
source_delimiter)
if prefix == '':
entry = ''
partition = None
else:
try:
entry, partition = prefix.split(source_delimiter)
except ValueError:
entry = prefix
partition = None
if partition:
try:
partition = int(partition)
except ValueError:
raise ValueError(
'Partition prefix length must be an integer.',
partition)
validate_uuid_prefix(entry)
return cls(common=entry, partition=partition)
def list_bundles(self,
source: LocalSourceRef,
prefix: str
) -> List[SourcedBundleFQID[LocalSourceRef]]:
source_prefix = source.spec.prefix.common
validate_uuid_prefix(source_prefix + prefix)
directory = self.local_path / source.spec.name
files = directory.glob(f'{source_prefix}{prefix}*{self._manifest_ext}')
bundle_fqids = []
for file in files:
fqid, _, suffix = file.name.rpartition(self._manifest_ext)
assert suffix == ''
uuid, version = fqid.split('.', 1)
bundle_fqids.append(SourcedBundleFQID(uuid=uuid,
version=version,
source=source))
return bundle_fqids
def _list_links_ids(self, source: TDRSourceRef,
prefix: str) -> List[TDRBundleFQID]:
source_prefix = source.spec.prefix.common
validate_uuid_prefix(source_prefix + prefix)
current_bundles = self._query_latest_version(source.spec,
f'''
SELECT links_id, version
FROM {backtick(self._full_table_name(source.spec, 'links'))}
WHERE STARTS_WITH(links_id, '{source_prefix + prefix}')
''',
group_by='links_id')
return [
SourcedBundleFQID(source=source,
uuid=row['links_id'],
version=self.format_version(row['version']))
for row in current_bundles
]
def remote_reindex_partition(self, message: JSON) -> None:
catalog = message['catalog']
prefix = message['prefix']
validate_uuid_prefix(prefix)
source = self.repository_plugin(catalog).source_from_json(
message['source'])
bundle_fqids = self.list_bundles(catalog, source, prefix)
bundle_fqids = self.filter_obsolete_bundle_versions(bundle_fqids)
logger.info(
'After filtering obsolete versions, '
'%i bundles remain in prefix %r of source %r in catalog %r',
len(bundle_fqids), prefix, str(source.spec), catalog)
messages = (self.bundle_message(catalog, bundle_fqid)
for bundle_fqid in bundle_fqids)
num_messages = self.queue_notifications(messages)
logger.info(
'Successfully queued %i notification(s) for prefix %s of '
'source %r', num_messages, prefix, source)
def list_bundles(self, source: DSSSourceRef,
prefix: str) -> List[DSSBundleFQID]:
self._assert_source(source)
prefix = source.spec.prefix.common + prefix
validate_uuid_prefix(prefix)
log.info('Listing bundles with prefix %r in source %r.', prefix,
source)
bundle_fqids = []
response = self.dss_client.get_bundles_all.iterate(prefix=prefix,
replica='aws',
per_page=500)
for bundle in response:
bundle_fqids.append(
SourcedBundleFQID(source=source,
uuid=bundle['uuid'],
version=bundle['version']))
log.info('There are %i bundle(s) with prefix %r in source %r.',
len(bundle_fqids), prefix, source)
return bundle_fqids
def __attrs_post_init__(self):
validate_uuid_prefix(self.prefix)
