def _process_wiki_entry(entry, master_parts_list, master_missing_parts,
pending_validation_entries):
"""Add valid wiki entries to the master parts list"""
# return the number of errors encountered
try:
data = yaml.safe_load(entry)
except (ScannerError, ParserError) as e:
raise errors.InvalidWikiEntryError(
'Bad wiki entry, possibly malformed YAML for entry: {}'.format(e))
try:
parts = data['parts']
except KeyError as e:
raise errors.InvalidWikiEntryError(
'"parts" missing from wiki entry: {}'.format(entry))
for part_name in parts:
if part_name and part_name in master_parts_list:
raise errors.InvalidWikiEntryError(
'Duplicate part found in the wiki: {} in entry {}'.format(
part_name, entry))
parts_list, after_parts = _process_entry(data)
known_parts = list(parts_list.keys()) + list(master_parts_list.keys())
missing_parts = missing_parts_set(after_parts, known_parts)
if not len(missing_parts):
master_parts_list.update(parts_list)
master_missing_parts -= set(parts_list.keys())
else:
pending_validation_entries.append(entry)
master_missing_parts.update(missing_parts)
logger.debug('Parts {!r} are missing'.format(",".join(missing_parts)))
def _process_entry(data):
parts_list = OrderedDict()
# Store all the parts listed in 'after' for each included part so that
# we can check later that we aren't missing any parts.
# XXX: What do we do about 'after' parts that should be looked for in
# the wiki? They should be in the master parts list.
after_parts = set()
# Get optional wiki entry fields.
origin_type = data.get('origin-type')
origin_branch = data.get('origin-branch')
origin_commit = data.get('origin-commit')
origin_tag = data.get('origin-tag')
# Get required wiki entry fields.
try:
entry_parts = data['parts']
origin = data['origin']
maintainer = data['maintainer']
description = data['description']
except KeyError as e:
raise errors.InvalidWikiEntryError(
'Missing key in wiki entry: {}'.format(e))
logger.info('Processing origin {origin!r}'.format(origin=origin))
origin_dir = os.path.join(_get_base_dir(), _encode_origin(origin))
os.makedirs(origin_dir, exist_ok=True)
source_handler = sources.get_source_handler(origin,
source_type=origin_type)
handler = source_handler(origin, source_dir=origin_dir)
repo.check_for_command(handler.command)
handler.source_branch = origin_branch
handler.source_commit = origin_commit
handler.source_tag = origin_tag
handler.pull()
try:
origin_data = _get_origin_data(origin_dir)
except MissingSnapcraftYAMLError:
raise errors.InvalidWikiEntryError(
'Origin {origin!r} is missing a snapcraft.yaml file.'.format(
origin=origin))
except BadSnapcraftYAMLError as e:
raise errors.InvalidWikiEntryError(
'snapcraft.yaml error: {}'.format(e))
origin_parts = origin_data.get('parts', {})
origin_name = origin_data.get('name')
origin_version = origin_data.get('version')
entry_parts_list, entry_parts_after_list = _process_entry_parts(
entry_parts, origin_parts, origin, maintainer, description,
origin_name, origin_version,
)
parts_list.update(entry_parts_list)
after_parts.update(entry_parts_after_list)
return parts_list, after_parts
def _get_origin_data(origin_dir):
origin_data = {}
yaml_file = project_loader.get_snapcraft_yaml(base_dir=origin_dir)
try:
with open(yaml_file) as fp:
origin_data = yaml.safe_load(fp)
except ScannerError as e:
raise errors.InvalidWikiEntryError(e) from e
return origin_data
def _process_entry(data):
parts_list = OrderedDict()
# Store all the parts listed in 'after' for each included part so that
# we can check later that we aren't missing any parts.
# XXX: What do we do about 'after' parts that should be looked for in
# the wiki? They should be in the master parts list.
after_parts = set()
# Get optional wiki entry fields.
origin_type = data.get("origin-type")
origin_branch = data.get("origin-branch")
origin_commit = data.get("origin-commit")
origin_tag = data.get("origin-tag")
# Get required wiki entry fields.
try:
entry_parts = data["parts"]
origin = data["origin"]
maintainer = data["maintainer"]
description = data["description"]
except KeyError as e:
raise errors.InvalidWikiEntryError(
"Missing key in wiki entry: {}".format(e))
logger.info("Processing origin {origin!r}".format(origin=origin))
origin_dir = os.path.join(_get_base_dir(), _encode_origin(origin))
os.makedirs(origin_dir, exist_ok=True)
source_handler = sources.get_source_handler(origin,
source_type=origin_type)
handler = source_handler(origin, source_dir=origin_dir)
repo.check_for_command(handler.command)
handler.source_branch = origin_branch
handler.source_commit = origin_commit
handler.source_tag = origin_tag
handler.pull()
try:
origin_data = _get_origin_data(origin_dir)
except project.errors.MissingSnapcraftYamlError as e:
raise errors.InvalidWikiEntryError(
"Origin {origin!r} is missing a snapcraft.yaml file.".format(
origin=origin)) from e
except errors.SnapcraftEnvironmentError as e:
raise errors.InvalidWikiEntryError(
"snapcraft.yaml error: {}".format(e)) from e
origin_parts = origin_data.get("parts", {})
origin_name = origin_data.get("name")
origin_version = origin_data.get("version")
entry_parts_list, entry_parts_after_list = _process_entry_parts(
entry_parts,
origin_parts,
origin,
maintainer,
description,
origin_name,
origin_version,
)
parts_list.update(entry_parts_list)
after_parts.update(entry_parts_after_list)
return parts_list, after_parts