def create_fetchable_from_uri(pkg,
chksums,
ignore_missing_chksums,
ignore_unknown_mirrors,
mirrors,
default_mirrors,
common_files,
uri,
filename=None):
default_filename = os.path.basename(uri)
if filename is not None:
# log redundant renames for pkgcheck to flag
if filename == default_filename:
logger.info(f'redundant rename: {uri} -> {filename}')
else:
filename = default_filename
if not filename:
raise ValueError(f'missing filename: {uri!r}')
preexisting = common_files.get(filename)
if preexisting is None:
if filename not in chksums and not ignore_missing_chksums:
raise metadata_errors.MissingChksum(pkg, filename)
uris = fetch.uri_list(filename)
else:
uris = preexisting.uri
if filename != uri:
if preexisting is None:
if "primaryuri" not in pkg.restrict:
if default_mirrors and "mirror" not in pkg.restrict:
uris.add_mirror(default_mirrors)
if uri.startswith("mirror://"):
# mirror:// is 9 chars.
tier, remaining_uri = uri[9:].split("/", 1)
mirror = mirrors.get(tier, fetch.unknown_mirror(tier))
uris.add_mirror(mirror, sub_uri=remaining_uri)
else:
uris.add_uri(uri)
if preexisting is None and "primaryuri" in pkg.restrict:
if default_mirrors and "mirror" not in pkg.restrict:
uris.add_mirror(default_mirrors)
if preexisting is None:
common_files[filename] = fetch.fetchable(filename, uris,
chksums.get(filename))
return common_files[filename]
def _parse_config(self):
"""Load data from the repo's metadata/layout.conf file."""
path = pjoin(self.location, self.layout_offset)
data = read_dict(iter_read_bash(
readlines(path, strip_whitespace=True, swallow_missing=True)),
source_isiter=True,
strip=True,
filename=path,
ignore_errors=True)
sf = object.__setattr__
sf(self, 'repo_name', data.get('repo-name', None))
hashes = data.get('manifest-hashes', '').lower().split()
if hashes:
hashes = ['size'] + hashes
hashes = tuple(iter_stable_unique(hashes))
else:
hashes = self.default_hashes
required_hashes = data.get('manifest-required-hashes',
'').lower().split()
if required_hashes:
required_hashes = ['size'] + required_hashes
required_hashes = tuple(iter_stable_unique(required_hashes))
else:
required_hashes = self.default_required_hashes
manifest_policy = data.get('use-manifests', 'strict').lower()
d = {
'disabled': (manifest_policy == 'false'),
'strict': (manifest_policy == 'strict'),
'thin': (data.get('thin-manifests', '').lower() == 'true'),
'signed': (data.get('sign-manifests', 'true').lower() == 'true'),
'hashes': hashes,
'required_hashes': required_hashes,
}
sf(self, 'manifests', _immutable_attr_dict(d))
masters = data.get('masters')
_missing_masters = False
if masters is None:
if not self.is_empty:
logger.warning(
f"{self.repo_id} repo at {self.location!r}, doesn't "
"specify masters in metadata/layout.conf. Please explicitly "
"set masters (use \"masters =\" if the repo is standalone)."
)
_missing_masters = True
masters = ()
else:
masters = tuple(iter_stable_unique(masters.split()))
sf(self, '_missing_masters', _missing_masters)
sf(self, 'masters', masters)
aliases = data.get('aliases', '').split() + [
self.config_name, self.repo_name, self.pms_repo_name, self.location
]
sf(self, 'aliases', tuple(filter(None, iter_stable_unique(aliases))))
sf(self, 'eapis_deprecated',
tuple(iter_stable_unique(data.get('eapis-deprecated', '').split())))
sf(self, 'eapis_banned',
tuple(iter_stable_unique(data.get('eapis-banned', '').split())))
sf(
self, 'properties_allowed',
tuple(
iter_stable_unique(data.get('properties-allowed',
'').split())))
sf(self, 'restrict_allowed',
tuple(iter_stable_unique(data.get('restrict-allowed', '').split())))
v = set(data.get('cache-formats', 'md5-dict').lower().split())
if not v:
v = [None]
else:
# sort into favored order
v = [f for f in self.supported_cache_formats if f in v]
if not v:
logger.warning(
f'unknown cache format: falling back to md5-dict format')
v = ['md5-dict']
sf(self, 'cache_format', list(v)[0])
profile_formats = set(
data.get('profile-formats', 'pms').lower().split())
if not profile_formats:
logger.info(
f"{self.repo_id!r} repo at {self.location!r} has explicitly "
"unset profile-formats, defaulting to pms")
profile_formats = {'pms'}
unknown = profile_formats.difference(self.supported_profile_formats)
if unknown:
logger.info("%r repo at %r has unsupported profile format%s: %s",
self.repo_id, self.location, pluralism(unknown),
', '.join(sorted(unknown)))
profile_formats.difference_update(unknown)
profile_formats.add('pms')
sf(self, 'profile_formats', profile_formats)
def process_scope(out,
buff,
pos,
var_match,
func_match,
endchar,
envvar_callback=None,
func_callback=None,
func_level=0):
window_start = pos
window_end = None
isspace = str.isspace
end = len(buff)
while pos < end and buff[pos] != endchar:
# Wander forward to the next non space.
if window_end is not None:
if out is not None:
out.write(buff[window_start:window_end].encode('utf-8'))
window_start = pos
window_end = None
com_start = pos
ch = buff[pos]
if isspace(ch):
pos += 1
continue
# Ignore comments.
if ch == '#':
pos = walk_statement_pound(buff, pos, endchar)
continue
new_start, new_end, new_p = is_function(buff, pos)
if new_p is not None:
func_name = buff[new_start:new_end]
logger.debug(f'matched func name {func_name!r}')
new_p = process_scope(None,
buff,
new_p,
None,
None,
'}',
func_callback=func_callback,
func_level=func_level + 1)
logger.debug(f'ended processing {func_name!r}')
if func_callback is not None:
func_callback(func_level, func_name, buff[new_start:new_p])
if func_match is not None and func_match(func_name):
logger.debug(f'filtering func {func_name!r}')
window_end = com_start
pos = new_p
pos += 1
continue
# Check for env assignment.
new_start, new_end, new_p = is_envvar(buff, pos)
if new_p is None:
# Non env assignment.
pos = walk_command_complex(buff, pos, endchar, COMMAND_PARSING)
# icky icky icky icky
if pos < end and buff[pos] != endchar:
pos += 1
else:
# Env assignment.
var_name = buff[new_start:new_end]
pos = new_p
if envvar_callback:
envvar_callback(var_name)
logger.debug(f'matched env assign {var_name!r}')
if var_match is not None and var_match(var_name):
# This would be filtered.
logger.info(f"filtering var {var_name!r}")
window_end = com_start
if pos >= end:
return pos
while (pos < end and not isspace(buff[pos]) and buff[pos] != ';'):
if buff[pos] == "'":
pos = walk_statement_no_parsing(buff, pos + 1, "'") + 1
elif buff[pos] in '"`':
pos = walk_command_escaped_parsing(buff, pos + 1,
buff[pos]) + 1
elif buff[pos] == '(':
pos = walk_command_escaped_parsing(buff, pos + 1, ')') + 1
elif buff[pos] == '$':
pos += 1
if pos >= end:
continue
pos = walk_dollar_expansion(buff, pos, end, endchar)
continue
else:
# blah=cah ; single word
pos = walk_command_complex(buff, pos, ' ', SPACE_PARSING)
if out is not None:
if window_end is None:
window_end = pos
if window_end > end:
window_end = end
out.write(buff[window_start:window_end].encode('utf-8'))
return pos