def from_cfg_dir(cfg_dir: str, cfg_types_file='config_types.yaml'): cfg_dir = existing_dir(os.path.abspath(cfg_dir)) cfg_types_dict = parse_yaml_file(os.path.join(cfg_dir, cfg_types_file)) raw = {} raw[ConfigFactory.CFG_TYPES] = cfg_types_dict def parse_cfg(cfg_type): # assume for now that there is exactly one cfg source (file) cfg_sources = list(cfg_type.sources()) if not len(cfg_sources) == 1: raise ValueError( 'currently, only exactly one cfg file is supported per type' ) cfg_file = cfg_sources[0].file() parsed_cfg = parse_yaml_file(os.path.join(cfg_dir, cfg_file)) return parsed_cfg # parse all configurations for cfg_type in map(ConfigType, cfg_types_dict.values()): cfg_name = cfg_type.cfg_type_name() raw[cfg_name] = parse_cfg(cfg_type) return ConfigFactory(raw_dict=raw)
def download_dependencies( component_descriptor: CliHints.existing_file(), out_dir: str, ): if not os.path.isdir(out_dir): os.mkdir(out_dir) component_descriptor = ComponentDescriptor.from_dict( parse_yaml_file(component_descriptor)) image_references = [ container_image.image_reference() for _, container_image in _enumerate_effective_images( component_descriptor=component_descriptor) ] def mangled_outfile_name(image_reference): mangled_fname = image_reference.replace(':', '_').replace('/', '_') return os.path.join(out_dir, mangled_fname + '.tar') for image_ref in image_references: fname = mangled_outfile_name(image_ref) with open(fname, 'wb') as f: container.registry.retrieve_container_image( image_reference=image_ref, outfileobj=f, ) print(fname)
def enumerate_definition_descriptors(self): info('enumerating explicitly specified definition file') try: definitions = parse_yaml_file(self.definition_file) yield from self._wrap_into_descriptors( repo_path=self.repo_path, repo_hostname=self.repo_host, branch=self.repo_branch, raw_definitions=definitions, ) except BaseException as e: yield DefinitionDescriptor( pipeline_name='<invalid YAML>', pipeline_definition={}, main_repo={ 'path': self.repo_path, 'branch': self.repo_branch, 'hostname': self.repo_host, }, concourse_target_cfg=self.cfg_set.concourse(), concourse_target_team=self.job_mapping.team_name(), override_definitions=(), exception=e, )
def upload_grouped_product_images( protecode_cfg_name: str, product_cfg_file: CliHints.existing_file(), processing_mode: CliHint( choices=list(ProcessingMode), type=ProcessingMode, )=ProcessingMode.RESCAN, protecode_group_id: int=5, parallel_jobs: int=4, cve_threshold: int=7, ignore_if_triaged: bool=True, reference_group_ids: [int]=[], ): cfg_factory = ctx().cfg_factory() protecode_cfg = cfg_factory.protecode(protecode_cfg_name) component_descriptor = ComponentDescriptor.from_dict( raw_dict=parse_yaml_file(product_cfg_file) ) upload_results, license_report = upload_grouped_images( protecode_cfg=protecode_cfg, component_descriptor=component_descriptor, protecode_group_id=protecode_group_id, parallel_jobs=parallel_jobs, cve_threshold=cve_threshold, ignore_if_triaged=ignore_if_triaged, processing_mode=processing_mode, reference_group_ids=reference_group_ids, )
def parse_cfg(cfg_type): # assume for now that there is exactly one cfg source (file) cfg_sources = list(cfg_type.sources()) if not len(cfg_sources) == 1: raise ValueError('currently, only exactly one cfg file is supported per type') cfg_file = cfg_sources[0].file() parsed_cfg = parse_yaml_file(os.path.join(cfg_dir, cfg_file)) return parsed_cfg
def resolve_component_descriptor( component_descriptor: CliHints.existing_file(), ): cfg_factory = ctx().cfg_factory() resolver = ComponentDescriptorResolver(cfg_factory=cfg_factory, ) component_descriptor = ComponentDescriptor.from_dict( parse_yaml_file(component_descriptor)) resolved_descriptor = resolver.resolve_component_references( product=component_descriptor) print(yaml.dump(resolved_descriptor.raw))
def add_dependencies( descriptor_src_file: CliHints.existing_file(), component_name: str, component_version: str, descriptor_out_file: str=None, component_dependencies: CliHint(action='append')=[], container_image_dependencies: CliHint(action='append')=[], web_dependencies: CliHint(action='append')=[], generic_dependencies: CliHint(action='append')=[], validation_policies: CliHint( type=ValidationPolicy, typehint=[ValidationPolicy], choices=[policy for policy in ValidationPolicy], )=[], ): product = ComponentDescriptor.from_dict(parse_yaml_file(descriptor_src_file)) component = product.component( ComponentReference.create(name=component_name, version=component_version) ) if not component: fail('component {c}:{v} was not found in {f}'.format( c=component_name, v=component_version, f=descriptor_src_file )) # maintain old behaviour if not validation_policies: validation_policies = [ValidationPolicy.FORBID_EXTRA_ATTRIBUTES] dependencies = _parse_dependencies( component_dependencies=component_dependencies, container_image_dependencies=container_image_dependencies, web_dependencies=web_dependencies, generic_dependencies=generic_dependencies, validation_policies=validation_policies, ) component.add_dependencies(dependencies) product_dict = {'components': [component.raw]} print(yaml.dump(product_dict, indent=2)) product_dict = json.loads(json.dumps({'components': [component.raw]})) if not descriptor_out_file: print(yaml.dump(product_dict, indent=2)) else: with open(descriptor_out_file, 'w') as f: yaml.dump(product_dict, f, indent=2)
def component_descriptor_to_xml( component_descriptor: CliHints.existing_file(), out_file: str, ): component_descriptor = ComponentDescriptor.from_dict( parse_yaml_file(component_descriptor)) image_references = [ container_image for _, container_image in _enumerate_effective_images( component_descriptor=component_descriptor) ] result_xml = product.xml.container_image_refs_to_xml(image_references, ) result_xml.write(out_file)
def _from_cfg_dir( cfg_dir: str, disable_cfg_element_lookup: bool, cfg_types_file='config_types.yaml', cfg_src_types=None, lookup_cfg_factory=None, ): cfg_dir = existing_dir(os.path.abspath(cfg_dir)) cfg_types_dict = parse_yaml_file(os.path.join(cfg_dir, cfg_types_file)) raw = {} raw[ConfigFactory.CFG_TYPES] = cfg_types_dict def retrieve_cfg(cfg_type): cfg_dict = {} for cfg_src in cfg_type.sources(): if cfg_src_types and type(cfg_src) not in cfg_src_types: continue if isinstance(cfg_src, LocalFileCfgSrc): parsed_cfg = ConfigFactory._parse_local_file( cfg_dir=cfg_dir, cfg_src=cfg_src, ) elif isinstance(cfg_src, GithubRepoFileSrc): if disable_cfg_element_lookup: continue parsed_cfg = ConfigFactory._parse_repo_file( cfg_src=cfg_src, lookup_cfg_factory=lookup_cfg_factory, ) else: raise NotImplementedError(cfg_src) for k, v in parsed_cfg.items(): if k in cfg_dict and cfg_dict[k] != v: raise ValueError( f'conflicting definition for {k=} in src {cfg_src}' ) cfg_dict[k] = v return cfg_dict return ConfigFactory( raw_dict=raw, retrieve_cfg=retrieve_cfg, )
def add_dependencies( descriptor_src_file: CliHints.existing_file(), component_name: str, component_version: str, descriptor_out_file: str = None, component_dependencies: CliHint(typehint=_parse_component_deps, action='append') = [], container_image_dependencies: CliHint(typehint=_parse_container_image_deps, action='append') = [], web_dependencies: CliHint(typehint=_parse_web_deps, action='append') = [], generic_dependencies: CliHint(typehint=_parse_generic_deps, action='append') = [], ): product = ComponentDescriptor.from_dict( parse_yaml_file(descriptor_src_file)) component = product.component( ComponentReference.create(name=component_name, version=component_version)) if not component: fail('component {c}:{v} was not found in {f}'.format( c=component_name, v=component_version, f=descriptor_src_file)) component_deps = component.dependencies() for component_ref in component_dependencies: component_deps.add_component_dependency(component_ref) for image_dep in container_image_dependencies: component_deps.add_container_image_dependency(image_dep) for web_dep in web_dependencies: component_deps.add_web_dependency(web_dep) for generic_dep in generic_dependencies: component_deps.add_generic_dependency(generic_dep) product_dict = json.loads(json.dumps({'components': [component.raw]})) if not descriptor_out_file: print(yaml.dump(product_dict, indent=2)) else: with open(descriptor_out_file, 'w') as f: yaml.dump(product_dict, f, indent=2)
def _parse_local_file(cfg_dir: str, cfg_src: LocalFileCfgSrc): cfg_file = cfg_src.file return parse_yaml_file(os.path.join(cfg_dir, cfg_file))
def parse_product_file(f): return ComponentDescriptor.from_dict(parse_yaml_file(f))
def _from_cfg_dir( cfg_dir: str, cfg_types_file='config_types.yaml', cfg_src_types=None, lookup_cfg_factory=None, ): cfg_dir = existing_dir(os.path.abspath(cfg_dir)) cfg_types_dict = parse_yaml_file(os.path.join(cfg_dir, cfg_types_file)) raw = {} raw[ConfigFactory.CFG_TYPES] = cfg_types_dict def parse_cfg(cfg_type): cfg_dict = {} def parse_local_file(cfg_src: LocalFileCfgSrc): cfg_file = cfg_src.file return parse_yaml_file(os.path.join(cfg_dir, cfg_file)) def parse_repo_file(cfg_src: GithubRepoFileSrc): import ccc.github repo_url = cfg_src.repository_url if not '://' in repo_url: repo_url = 'https://' + repo_url repo_url = urllib.parse.urlparse(repo_url) if not lookup_cfg_factory: raise RuntimeError( 'cannot resolve non-local cfg w/o bootstrap-cfg-factory' ) gh_api = ccc.github.github_api( ccc.github.github_cfg_for_hostname( repo_url.hostname, cfg_factory=lookup_cfg_factory, ), cfg_factory=lookup_cfg_factory, ) org, repo = repo_url.path.strip('/').split('/') gh_repo = gh_api.repository(org, repo) file_contents = gh_repo.file_contents( path=cfg_src.relpath, ref=gh_repo.default_branch, ).decoded.decode('utf-8') return yaml.safe_load(file_contents) for cfg_src in cfg_type.sources(): if cfg_src_types and type(cfg_src) not in cfg_src_types: continue if isinstance(cfg_src, LocalFileCfgSrc): parsed_cfg = parse_local_file(cfg_src=cfg_src) elif isinstance(cfg_src, GithubRepoFileSrc): parsed_cfg = parse_repo_file(cfg_src=cfg_src) else: raise NotImplementedError(cfg_src) for k, v in parsed_cfg.items(): if k in cfg_dict and cfg_dict[k] != v: raise ValueError(f'conflicting definition for {k=}') cfg_dict[k] = v return cfg_dict # parse all configurations for cfg_type in map(ConfigType, cfg_types_dict.values()): cfg_name = cfg_type.cfg_type_name() raw[cfg_name] = parse_cfg(cfg_type) return ConfigFactory(raw_dict=raw)