def late_resolve_image(self, distgit_key): """Resolve image and retrive meta without adding to image_map. Mainly for looking up parent image info.""" with Dir(self.images_dir): meta = ImageMetadata(self, self.images_dir, distgit_key + '.yml') return meta
def late_resolve_image(self, distgit_name, add=False): """Resolve image and retrieve meta, optionally adding to image_map. If add==True and image not found, error will be thrown""" if distgit_name in self.image_map: return self.image_map[distgit_name] data_obj = self.gitdata.load_data(path='images', key=distgit_name) if add and not data_obj: raise DoozerFatalError( 'Unable to resovle image metadata for {}'.format(distgit_name)) meta = ImageMetadata(self, data_obj) if add: self.image_map[distgit_name] = meta return meta
def gen_ImageMetadata(base_dir, config_filename, force): metadata = ImageMetadata(self, base_dir, config_filename) if force or metadata.enabled: self.image_map[metadata.distgit_key] = metadata
def initialize(self, mode='images', validate_content_sets=False, no_group=False, clone_source=True, disabled=None): if self.initialized: return if self.quiet and self.verbose: click.echo("Flags --quiet and --verbose are mutually exclusive") exit(1) # We could mark these as required and the click library would do this for us, # but this seems to prevent getting help from the various commands (unless you # specify the required parameters). This can probably be solved more cleanly, but TODO if not no_group and self.group is None: click.echo("Group must be specified") exit(1) if self.working_dir is None: self.working_dir = tempfile.mkdtemp(".tmp", "elliott-") # This can be set to False by operations which want the working directory to be left around self.remove_tmp_working_dir = True atexit.register(remove_tmp_working_dir, self) else: self.working_dir = os.path.abspath(self.working_dir) if not os.path.isdir(self.working_dir): os.makedirs(self.working_dir) if disabled is not None: self.load_disabled = disabled self.initialize_logging() self.resolve_metadata() if no_group: return # nothing past here should be run without a group self.record_log_path = os.path.join(self.working_dir, "record.log") self.record_log = open(self.record_log_path, 'a') atexit.register(close_file, self.record_log) # Directory where brew-logs will be downloaded after a build self.brew_logs_dir = os.path.join(self.working_dir, "brew-logs") if not os.path.isdir(self.brew_logs_dir): os.mkdir(self.brew_logs_dir) # Directory for flags between invocations in the same working-dir self.flags_dir = os.path.join(self.working_dir, "flags") if not os.path.isdir(self.flags_dir): os.mkdir(self.flags_dir) self.group_dir = self.gitdata.data_dir with Dir(self.group_dir): self.group_config = self.get_group_config() self.arches = self.group_config.get('arches', ['x86_64']) if self.group_config.name != self.group: raise IOError( "Name in group.yml does not match group name. Someone may have copied this group without updating group.yml (make sure to check branch)" ) if self.group_config.includes is not Missing and self.include is None: self.include = self.group_config.includes if self.branch is None: if self.group_config.branch is not Missing: self.branch = self.group_config.branch self.logger.info("Using branch from group.yml: %s" % self.branch) else: self.logger.info( "No branch specified either in group.yml or on the command line; all included images will need to specify their own." ) else: self.logger.info("Using branch from command line: %s" % self.branch) # Flattens a list like like [ 'x', 'y,z' ] into [ 'x.yml', 'y.yml', 'z.yml' ] # for later checking we need to remove from the lists, but they are tuples. Clone to list def flatten_list(names): if not names: return [] # split csv values result = [] for n in names: result.append( [x for x in n.replace(' ', ',').split(',') if x != '']) # flatten result and remove dupes return list(set([y for x in result for y in x])) def filter_wip(n, d): return d.get('mode', 'enabled') in ['wip', 'enabled'] def filter_enabled(n, d): return d.get('mode', 'enabled') == 'enabled' def filter_disabled(n, d): return d.get('mode', 'enabled') in ['enabled', 'disabled'] exclude_keys = flatten_list(self.exclude) image_keys = flatten_list(self.images) rpm_keys = flatten_list(self.rpms) filter_func = None if self.load_wip and self.load_disabled: pass # use no filter, load all elif self.load_wip: filter_func = filter_wip elif self.load_disabled: filter_func = filter_disabled else: filter_func = filter_enabled image_data = self.gitdata.load_data( path='images', keys=image_keys, exclude=exclude_keys, filter_funcs=None if len(image_keys) else filter_func) try: rpm_data = self.gitdata.load_data( path='rpms', keys=rpm_keys, exclude=exclude_keys, filter_funcs=None if len(rpm_keys) else filter_func) except gitdata.GitDataPathException: # some older versions have no RPMs, that's ok. rpm_data = {} missed_include = set(image_keys + rpm_keys) - set(image_data.keys() + rpm_data.keys()) if len(missed_include) > 0: raise ElliottFatalError( 'The following images or rpms were either missing or filtered out: {}' .format(', '.join(missed_include))) if mode in ['images', 'both']: for i in image_data.itervalues(): metadata = ImageMetadata(self, i) self.image_map[metadata.distgit_key] = metadata if not self.image_map: self.logger.warning( "No image metadata directories found for given options within: {}" .format(self.group_dir)) if mode in ['rpms', 'both']: for r in rpm_data.itervalues(): metadata = RPMMetadata(self, r, clone_source=clone_source) self.rpm_map[metadata.distgit_key] = metadata if not self.rpm_map: self.logger.warning( "No rpm metadata directories found for given options within: {}" .format(self.group_dir)) # Make sure that the metadata is not asking us to check out the same exact distgit & branch. # This would almost always indicate someone has checked in duplicate metadata into a group. no_collide_check = {} for meta in self.rpm_map.values() + self.image_map.values(): key = '{}/{}/#{}'.format(meta.namespace, meta.name, meta.branch()) if key in no_collide_check: raise IOError( 'Complete duplicate distgit & branch; something wrong with metadata: {} from {} and {}' .format(key, meta.config_filename, no_collide_check[key].config_filename)) no_collide_check[key] = meta # Read in the streams definite for this group if one exists streams = self.gitdata.load_data(key='streams') if streams: self.streams = Model(self.gitdata.load_data(key='streams').data)
def initialize(self, mode='images', clone_distgits=True, validate_content_sets=False, no_group=False, clone_source=True, disabled=None, config_excludes=None): if self.initialized: return if self.quiet and self.verbose: click.echo("Flags --quiet and --verbose are mutually exclusive") exit(1) # We could mark these as required and the click library would do this for us, # but this seems to prevent getting help from the various commands (unless you # specify the required parameters). This can probably be solved more cleanly, but TODO if not no_group and self.group is None: click.echo("Group must be specified") exit(1) if self.working_dir is None: self.working_dir = tempfile.mkdtemp(".tmp", "oit-") # This can be set to False by operations which want the working directory to be left around self.remove_tmp_working_dir = True atexit.register(remove_tmp_working_dir, self) else: self.working_dir = os.path.abspath( os.path.expanduser(self.working_dir)) if not os.path.isdir(self.working_dir): os.makedirs(self.working_dir) self.distgits_dir = os.path.join(self.working_dir, "distgits") if not os.path.isdir(self.distgits_dir): os.mkdir(self.distgits_dir) self.distgits_diff_dir = os.path.join(self.working_dir, "distgits-diffs") if not os.path.isdir(self.distgits_diff_dir): os.mkdir(self.distgits_diff_dir) self.sources_dir = os.path.join(self.working_dir, "sources") if not os.path.isdir(self.sources_dir): os.mkdir(self.sources_dir) if disabled is not None: self.load_disabled = disabled self.initialize_logging() self.init_state() if no_group: return # nothing past here should be run without a group self.resolve_metadata() self.record_log_path = os.path.join(self.working_dir, "record.log") self.record_log = open(self.record_log_path, 'a') atexit.register(close_file, self.record_log) # Directory where brew-logs will be downloaded after a build self.brew_logs_dir = os.path.join(self.working_dir, "brew-logs") if not os.path.isdir(self.brew_logs_dir): os.mkdir(self.brew_logs_dir) # Directory for flags between invocations in the same working-dir self.flags_dir = os.path.join(self.working_dir, "flags") if not os.path.isdir(self.flags_dir): os.mkdir(self.flags_dir) self.group_dir = self.gitdata.data_dir # register the sources # For each "--source alias path" on the command line, register its existence with # the runtime. for r in self.source: self.register_source_alias(r[0], r[1]) if self.sources: with open(self.sources, 'r') as sf: source_dict = yaml.full_load(sf) if not isinstance(source_dict, dict): raise ValueError( '--sources param must be a yaml file containing a single dict.' ) for key, val in source_dict.items(): self.register_source_alias(key, val) with Dir(self.group_dir): self.group_config = self.get_group_config() self.arches = self.group_config.get('arches', ['x86_64']) self.repos = Repos(self.group_config.repos, self.arches) if validate_content_sets: self.repos.validate_content_sets() if self.group_config.name != self.group: raise IOError( "Name in group.yml does not match group name. Someone may have copied this group without updating group.yml (make sure to check branch)" ) if self.branch is None: if self.group_config.branch is not Missing: self.branch = self.group_config.branch self.logger.info("Using branch from group.yml: %s" % self.branch) else: self.logger.info( "No branch specified either in group.yml or on the command line; all included images will need to specify their own." ) else: self.logger.info("Using branch from command line: %s" % self.branch) scanner = self.group_config.image_build_log_scanner if scanner is not Missing: # compile regexen and fail early if they don't regexen = [] for val in scanner.matches: try: regexen.append(re.compile(val)) except Exception as e: raise ValueError( "could not compile image build log regex for group:\n{}\n{}" .format(val, e)) scanner.matches = regexen # Flattens a list like like [ 'x', 'y,z' ] into [ 'x.yml', 'y.yml', 'z.yml' ] # for later checking we need to remove from the lists, but they are tuples. Clone to list def flatten_list(names): if not names: return [] # split csv values result = [] for n in names: result.append( [x for x in n.replace(' ', ',').split(',') if x != '']) # flatten result and remove dupes return list(set([y for x in result for y in x])) def filter_wip(n, d): return d.get('mode', 'enabled') in ['wip', 'enabled'] def filter_enabled(n, d): return d.get('mode', 'enabled') == 'enabled' def filter_disabled(n, d): return d.get('mode', 'enabled') in ['enabled', 'disabled'] exclude_keys = flatten_list(self.exclude) image_ex = list(exclude_keys) rpm_ex = list(exclude_keys) image_keys = flatten_list(self.images) rpm_keys = flatten_list(self.rpms) filter_func = None if self.load_wip and self.load_disabled: pass # use no filter, load all elif self.load_wip: filter_func = filter_wip elif self.load_disabled: filter_func = filter_disabled else: filter_func = filter_enabled replace_vars = {} if self.group_config.vars: replace_vars = self.group_config.vars.primitive() if config_excludes: excludes = self.group_config.get(config_excludes, {}) image_ex.extend(excludes.get('images', [])) rpm_ex.extend(excludes.get('rpms', [])) # pre-load the image data to get the names for all images # eventually we can use this to allow loading images by # name or distgit. For now this is used elsewhere image_name_data = self.gitdata.load_data(path='images') for img in image_name_data.itervalues(): name = img.data.get('name') short_name = name.split('/')[1] self.image_name_map[name] = img.key self.image_name_map[short_name] = img.key image_data = self.gitdata.load_data( path='images', keys=image_keys, exclude=image_ex, replace_vars=replace_vars, filter_funcs=None if len(image_keys) else filter_func) try: rpm_data = self.gitdata.load_data( path='rpms', keys=rpm_keys, exclude=rpm_ex, replace_vars=replace_vars, filter_funcs=None if len(rpm_keys) else filter_func) except gitdata.GitDataPathException: # some older versions have no RPMs, that's ok. rpm_data = {} missed_include = set(image_keys + rpm_keys) - set(image_data.keys() + rpm_data.keys()) if len(missed_include) > 0: raise DoozerFatalError( 'The following images or rpms were either missing or filtered out: {}' .format(', '.join(missed_include))) if mode in ['images', 'both']: for i in image_data.itervalues(): metadata = ImageMetadata(self, i) self.image_map[metadata.distgit_key] = metadata if not self.image_map: self.logger.warning( "No image metadata directories found for given options within: {}" .format(self.group_dir)) for image in self.image_map.itervalues(): image.resolve_parent() # now that ancestry is defined, make sure no cyclic dependencies for image in self.image_map.itervalues(): for child in image.children: if image.is_ancestor(child): raise DoozerFatalError( '{} cannot be both a parent and dependent of {}' .format(child.distgit_key, image.distgit_key)) self.image_tree = {} image_lists = {0: []} def add_child_branch(child, branch, level=1): if level not in image_lists: image_lists[level] = [] for sub_child in child.children: branch[sub_child.distgit_key] = {} image_lists[level].append(sub_child.distgit_key) add_child_branch(sub_child, branch[sub_child.distgit_key], level + 1) for image in self.image_map.itervalues(): if not image.parent: self.image_tree[image.distgit_key] = {} image_lists[0].append(image.distgit_key) add_child_branch(image, self.image_tree[image.distgit_key]) levels = image_lists.keys() levels.sort() self.image_order = [] for l in levels: self.image_order.extend(image_lists[l]) if mode in ['rpms', 'both']: for r in rpm_data.itervalues(): metadata = RPMMetadata(self, r, clone_source=clone_source) self.rpm_map[metadata.distgit_key] = metadata if not self.rpm_map: self.logger.warning( "No rpm metadata directories found for given options within: {}" .format(self.group_dir)) # Make sure that the metadata is not asking us to check out the same exact distgit & branch. # This would almost always indicate someone has checked in duplicate metadata into a group. no_collide_check = {} for meta in self.rpm_map.values() + self.image_map.values(): key = '{}/{}/#{}'.format(meta.namespace, meta.name, meta.branch()) if key in no_collide_check: raise IOError( 'Complete duplicate distgit & branch; something wrong with metadata: {} from {} and {}' .format(key, meta.config_filename, no_collide_check[key].config_filename)) no_collide_check[key] = meta # Read in the streams definite for this group if one exists streams = self.gitdata.load_data(key='streams') if streams: self.streams = Model(self.gitdata.load_data(key='streams').data) if clone_distgits: self.clone_distgits() self.initialized = True