def late_resolve_image(self, distgit_key):
"""Resolve image and retrive meta without adding to image_map.
Mainly for looking up parent image info."""
with Dir(self.images_dir):
meta = ImageMetadata(self, self.images_dir, distgit_key + '.yml')
return meta
def late_resolve_image(self, distgit_name, add=False):
"""Resolve image and retrieve meta, optionally adding to image_map.
If add==True and image not found, error will be thrown"""
if distgit_name in self.image_map:
return self.image_map[distgit_name]
data_obj = self.gitdata.load_data(path='images', key=distgit_name)
if add and not data_obj:
raise DoozerFatalError(
'Unable to resovle image metadata for {}'.format(distgit_name))
meta = ImageMetadata(self, data_obj)
if add:
self.image_map[distgit_name] = meta
return meta
def gen_ImageMetadata(base_dir, config_filename, force):
metadata = ImageMetadata(self, base_dir, config_filename)
if force or metadata.enabled:
self.image_map[metadata.distgit_key] = metadata
def initialize(self,
mode='images',
validate_content_sets=False,
no_group=False,
clone_source=True,
disabled=None):
if self.initialized:
return
if self.quiet and self.verbose:
click.echo("Flags --quiet and --verbose are mutually exclusive")
exit(1)
# We could mark these as required and the click library would do this for us,
# but this seems to prevent getting help from the various commands (unless you
# specify the required parameters). This can probably be solved more cleanly, but TODO
if not no_group and self.group is None:
click.echo("Group must be specified")
exit(1)
if self.working_dir is None:
self.working_dir = tempfile.mkdtemp(".tmp", "elliott-")
# This can be set to False by operations which want the working directory to be left around
self.remove_tmp_working_dir = True
atexit.register(remove_tmp_working_dir, self)
else:
self.working_dir = os.path.abspath(self.working_dir)
if not os.path.isdir(self.working_dir):
os.makedirs(self.working_dir)
if disabled is not None:
self.load_disabled = disabled
self.initialize_logging()
self.resolve_metadata()
if no_group:
return # nothing past here should be run without a group
self.record_log_path = os.path.join(self.working_dir, "record.log")
self.record_log = open(self.record_log_path, 'a')
atexit.register(close_file, self.record_log)
# Directory where brew-logs will be downloaded after a build
self.brew_logs_dir = os.path.join(self.working_dir, "brew-logs")
if not os.path.isdir(self.brew_logs_dir):
os.mkdir(self.brew_logs_dir)
# Directory for flags between invocations in the same working-dir
self.flags_dir = os.path.join(self.working_dir, "flags")
if not os.path.isdir(self.flags_dir):
os.mkdir(self.flags_dir)
self.group_dir = self.gitdata.data_dir
with Dir(self.group_dir):
self.group_config = self.get_group_config()
self.arches = self.group_config.get('arches', ['x86_64'])
if self.group_config.name != self.group:
raise IOError(
"Name in group.yml does not match group name. Someone may have copied this group without updating group.yml (make sure to check branch)"
)
if self.group_config.includes is not Missing and self.include is None:
self.include = self.group_config.includes
if self.branch is None:
if self.group_config.branch is not Missing:
self.branch = self.group_config.branch
self.logger.info("Using branch from group.yml: %s" %
self.branch)
else:
self.logger.info(
"No branch specified either in group.yml or on the command line; all included images will need to specify their own."
)
else:
self.logger.info("Using branch from command line: %s" %
self.branch)
# Flattens a list like like [ 'x', 'y,z' ] into [ 'x.yml', 'y.yml', 'z.yml' ]
# for later checking we need to remove from the lists, but they are tuples. Clone to list
def flatten_list(names):
if not names:
return []
# split csv values
result = []
for n in names:
result.append(
[x for x in n.replace(' ', ',').split(',') if x != ''])
# flatten result and remove dupes
return list(set([y for x in result for y in x]))
def filter_wip(n, d):
return d.get('mode', 'enabled') in ['wip', 'enabled']
def filter_enabled(n, d):
return d.get('mode', 'enabled') == 'enabled'
def filter_disabled(n, d):
return d.get('mode', 'enabled') in ['enabled', 'disabled']
exclude_keys = flatten_list(self.exclude)
image_keys = flatten_list(self.images)
rpm_keys = flatten_list(self.rpms)
filter_func = None
if self.load_wip and self.load_disabled:
pass # use no filter, load all
elif self.load_wip:
filter_func = filter_wip
elif self.load_disabled:
filter_func = filter_disabled
else:
filter_func = filter_enabled
image_data = self.gitdata.load_data(
path='images',
keys=image_keys,
exclude=exclude_keys,
filter_funcs=None if len(image_keys) else filter_func)
try:
rpm_data = self.gitdata.load_data(
path='rpms',
keys=rpm_keys,
exclude=exclude_keys,
filter_funcs=None if len(rpm_keys) else filter_func)
except gitdata.GitDataPathException:
# some older versions have no RPMs, that's ok.
rpm_data = {}
missed_include = set(image_keys +
rpm_keys) - set(image_data.keys() +
rpm_data.keys())
if len(missed_include) > 0:
raise ElliottFatalError(
'The following images or rpms were either missing or filtered out: {}'
.format(', '.join(missed_include)))
if mode in ['images', 'both']:
for i in image_data.itervalues():
metadata = ImageMetadata(self, i)
self.image_map[metadata.distgit_key] = metadata
if not self.image_map:
self.logger.warning(
"No image metadata directories found for given options within: {}"
.format(self.group_dir))
if mode in ['rpms', 'both']:
for r in rpm_data.itervalues():
metadata = RPMMetadata(self, r, clone_source=clone_source)
self.rpm_map[metadata.distgit_key] = metadata
if not self.rpm_map:
self.logger.warning(
"No rpm metadata directories found for given options within: {}"
.format(self.group_dir))
# Make sure that the metadata is not asking us to check out the same exact distgit & branch.
# This would almost always indicate someone has checked in duplicate metadata into a group.
no_collide_check = {}
for meta in self.rpm_map.values() + self.image_map.values():
key = '{}/{}/#{}'.format(meta.namespace, meta.name, meta.branch())
if key in no_collide_check:
raise IOError(
'Complete duplicate distgit & branch; something wrong with metadata: {} from {} and {}'
.format(key, meta.config_filename,
no_collide_check[key].config_filename))
no_collide_check[key] = meta
# Read in the streams definite for this group if one exists
streams = self.gitdata.load_data(key='streams')
if streams:
self.streams = Model(self.gitdata.load_data(key='streams').data)
def initialize(self,
mode='images',
clone_distgits=True,
validate_content_sets=False,
no_group=False,
clone_source=True,
disabled=None,
config_excludes=None):
if self.initialized:
return
if self.quiet and self.verbose:
click.echo("Flags --quiet and --verbose are mutually exclusive")
exit(1)
# We could mark these as required and the click library would do this for us,
# but this seems to prevent getting help from the various commands (unless you
# specify the required parameters). This can probably be solved more cleanly, but TODO
if not no_group and self.group is None:
click.echo("Group must be specified")
exit(1)
if self.working_dir is None:
self.working_dir = tempfile.mkdtemp(".tmp", "oit-")
# This can be set to False by operations which want the working directory to be left around
self.remove_tmp_working_dir = True
atexit.register(remove_tmp_working_dir, self)
else:
self.working_dir = os.path.abspath(
os.path.expanduser(self.working_dir))
if not os.path.isdir(self.working_dir):
os.makedirs(self.working_dir)
self.distgits_dir = os.path.join(self.working_dir, "distgits")
if not os.path.isdir(self.distgits_dir):
os.mkdir(self.distgits_dir)
self.distgits_diff_dir = os.path.join(self.working_dir,
"distgits-diffs")
if not os.path.isdir(self.distgits_diff_dir):
os.mkdir(self.distgits_diff_dir)
self.sources_dir = os.path.join(self.working_dir, "sources")
if not os.path.isdir(self.sources_dir):
os.mkdir(self.sources_dir)
if disabled is not None:
self.load_disabled = disabled
self.initialize_logging()
self.init_state()
if no_group:
return # nothing past here should be run without a group
self.resolve_metadata()
self.record_log_path = os.path.join(self.working_dir, "record.log")
self.record_log = open(self.record_log_path, 'a')
atexit.register(close_file, self.record_log)
# Directory where brew-logs will be downloaded after a build
self.brew_logs_dir = os.path.join(self.working_dir, "brew-logs")
if not os.path.isdir(self.brew_logs_dir):
os.mkdir(self.brew_logs_dir)
# Directory for flags between invocations in the same working-dir
self.flags_dir = os.path.join(self.working_dir, "flags")
if not os.path.isdir(self.flags_dir):
os.mkdir(self.flags_dir)
self.group_dir = self.gitdata.data_dir
# register the sources
# For each "--source alias path" on the command line, register its existence with
# the runtime.
for r in self.source:
self.register_source_alias(r[0], r[1])
if self.sources:
with open(self.sources, 'r') as sf:
source_dict = yaml.full_load(sf)
if not isinstance(source_dict, dict):
raise ValueError(
'--sources param must be a yaml file containing a single dict.'
)
for key, val in source_dict.items():
self.register_source_alias(key, val)
with Dir(self.group_dir):
self.group_config = self.get_group_config()
self.arches = self.group_config.get('arches', ['x86_64'])
self.repos = Repos(self.group_config.repos, self.arches)
if validate_content_sets:
self.repos.validate_content_sets()
if self.group_config.name != self.group:
raise IOError(
"Name in group.yml does not match group name. Someone may have copied this group without updating group.yml (make sure to check branch)"
)
if self.branch is None:
if self.group_config.branch is not Missing:
self.branch = self.group_config.branch
self.logger.info("Using branch from group.yml: %s" %
self.branch)
else:
self.logger.info(
"No branch specified either in group.yml or on the command line; all included images will need to specify their own."
)
else:
self.logger.info("Using branch from command line: %s" %
self.branch)
scanner = self.group_config.image_build_log_scanner
if scanner is not Missing:
# compile regexen and fail early if they don't
regexen = []
for val in scanner.matches:
try:
regexen.append(re.compile(val))
except Exception as e:
raise ValueError(
"could not compile image build log regex for group:\n{}\n{}"
.format(val, e))
scanner.matches = regexen
# Flattens a list like like [ 'x', 'y,z' ] into [ 'x.yml', 'y.yml', 'z.yml' ]
# for later checking we need to remove from the lists, but they are tuples. Clone to list
def flatten_list(names):
if not names:
return []
# split csv values
result = []
for n in names:
result.append(
[x for x in n.replace(' ', ',').split(',') if x != ''])
# flatten result and remove dupes
return list(set([y for x in result for y in x]))
def filter_wip(n, d):
return d.get('mode', 'enabled') in ['wip', 'enabled']
def filter_enabled(n, d):
return d.get('mode', 'enabled') == 'enabled'
def filter_disabled(n, d):
return d.get('mode', 'enabled') in ['enabled', 'disabled']
exclude_keys = flatten_list(self.exclude)
image_ex = list(exclude_keys)
rpm_ex = list(exclude_keys)
image_keys = flatten_list(self.images)
rpm_keys = flatten_list(self.rpms)
filter_func = None
if self.load_wip and self.load_disabled:
pass # use no filter, load all
elif self.load_wip:
filter_func = filter_wip
elif self.load_disabled:
filter_func = filter_disabled
else:
filter_func = filter_enabled
replace_vars = {}
if self.group_config.vars:
replace_vars = self.group_config.vars.primitive()
if config_excludes:
excludes = self.group_config.get(config_excludes, {})
image_ex.extend(excludes.get('images', []))
rpm_ex.extend(excludes.get('rpms', []))
# pre-load the image data to get the names for all images
# eventually we can use this to allow loading images by
# name or distgit. For now this is used elsewhere
image_name_data = self.gitdata.load_data(path='images')
for img in image_name_data.itervalues():
name = img.data.get('name')
short_name = name.split('/')[1]
self.image_name_map[name] = img.key
self.image_name_map[short_name] = img.key
image_data = self.gitdata.load_data(
path='images',
keys=image_keys,
exclude=image_ex,
replace_vars=replace_vars,
filter_funcs=None if len(image_keys) else filter_func)
try:
rpm_data = self.gitdata.load_data(
path='rpms',
keys=rpm_keys,
exclude=rpm_ex,
replace_vars=replace_vars,
filter_funcs=None if len(rpm_keys) else filter_func)
except gitdata.GitDataPathException:
# some older versions have no RPMs, that's ok.
rpm_data = {}
missed_include = set(image_keys +
rpm_keys) - set(image_data.keys() +
rpm_data.keys())
if len(missed_include) > 0:
raise DoozerFatalError(
'The following images or rpms were either missing or filtered out: {}'
.format(', '.join(missed_include)))
if mode in ['images', 'both']:
for i in image_data.itervalues():
metadata = ImageMetadata(self, i)
self.image_map[metadata.distgit_key] = metadata
if not self.image_map:
self.logger.warning(
"No image metadata directories found for given options within: {}"
.format(self.group_dir))
for image in self.image_map.itervalues():
image.resolve_parent()
# now that ancestry is defined, make sure no cyclic dependencies
for image in self.image_map.itervalues():
for child in image.children:
if image.is_ancestor(child):
raise DoozerFatalError(
'{} cannot be both a parent and dependent of {}'
.format(child.distgit_key, image.distgit_key))
self.image_tree = {}
image_lists = {0: []}
def add_child_branch(child, branch, level=1):
if level not in image_lists:
image_lists[level] = []
for sub_child in child.children:
branch[sub_child.distgit_key] = {}
image_lists[level].append(sub_child.distgit_key)
add_child_branch(sub_child,
branch[sub_child.distgit_key],
level + 1)
for image in self.image_map.itervalues():
if not image.parent:
self.image_tree[image.distgit_key] = {}
image_lists[0].append(image.distgit_key)
add_child_branch(image,
self.image_tree[image.distgit_key])
levels = image_lists.keys()
levels.sort()
self.image_order = []
for l in levels:
self.image_order.extend(image_lists[l])
if mode in ['rpms', 'both']:
for r in rpm_data.itervalues():
metadata = RPMMetadata(self, r, clone_source=clone_source)
self.rpm_map[metadata.distgit_key] = metadata
if not self.rpm_map:
self.logger.warning(
"No rpm metadata directories found for given options within: {}"
.format(self.group_dir))
# Make sure that the metadata is not asking us to check out the same exact distgit & branch.
# This would almost always indicate someone has checked in duplicate metadata into a group.
no_collide_check = {}
for meta in self.rpm_map.values() + self.image_map.values():
key = '{}/{}/#{}'.format(meta.namespace, meta.name, meta.branch())
if key in no_collide_check:
raise IOError(
'Complete duplicate distgit & branch; something wrong with metadata: {} from {} and {}'
.format(key, meta.config_filename,
no_collide_check[key].config_filename))
no_collide_check[key] = meta
# Read in the streams definite for this group if one exists
streams = self.gitdata.load_data(key='streams')
if streams:
self.streams = Model(self.gitdata.load_data(key='streams').data)
if clone_distgits:
self.clone_distgits()
self.initialized = True