def from_cfg_dir(cfg_dir: str, cfg_types_file='config_types.yaml'):
cfg_dir = existing_dir(os.path.abspath(cfg_dir))
cfg_types_dict = parse_yaml_file(os.path.join(cfg_dir, cfg_types_file))
raw = {}
raw[ConfigFactory.CFG_TYPES] = cfg_types_dict
def parse_cfg(cfg_type):
# assume for now that there is exactly one cfg source (file)
cfg_sources = list(cfg_type.sources())
if not len(cfg_sources) == 1:
raise ValueError(
'currently, only exactly one cfg file is supported per type'
)
cfg_file = cfg_sources[0].file()
parsed_cfg = parse_yaml_file(os.path.join(cfg_dir, cfg_file))
return parsed_cfg
# parse all configurations
for cfg_type in map(ConfigType, cfg_types_dict.values()):
cfg_name = cfg_type.cfg_type_name()
raw[cfg_name] = parse_cfg(cfg_type)
return ConfigFactory(raw_dict=raw)
def download_dependencies(
component_descriptor: CliHints.existing_file(),
out_dir: str,
):
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
component_descriptor = ComponentDescriptor.from_dict(
parse_yaml_file(component_descriptor))
image_references = [
container_image.image_reference()
for _, container_image in _enumerate_effective_images(
component_descriptor=component_descriptor)
]
def mangled_outfile_name(image_reference):
mangled_fname = image_reference.replace(':', '_').replace('/', '_')
return os.path.join(out_dir, mangled_fname + '.tar')
for image_ref in image_references:
fname = mangled_outfile_name(image_ref)
with open(fname, 'wb') as f:
container.registry.retrieve_container_image(
image_reference=image_ref,
outfileobj=f,
)
print(fname)
def enumerate_definition_descriptors(self):
info('enumerating explicitly specified definition file')
try:
definitions = parse_yaml_file(self.definition_file)
yield from self._wrap_into_descriptors(
repo_path=self.repo_path,
repo_hostname=self.repo_host,
branch=self.repo_branch,
raw_definitions=definitions,
)
except BaseException as e:
yield DefinitionDescriptor(
pipeline_name='<invalid YAML>',
pipeline_definition={},
main_repo={
'path': self.repo_path,
'branch': self.repo_branch,
'hostname': self.repo_host,
},
concourse_target_cfg=self.cfg_set.concourse(),
concourse_target_team=self.job_mapping.team_name(),
override_definitions=(),
exception=e,
)
def upload_grouped_product_images(
protecode_cfg_name: str,
product_cfg_file: CliHints.existing_file(),
processing_mode: CliHint(
choices=list(ProcessingMode),
type=ProcessingMode,
)=ProcessingMode.RESCAN,
protecode_group_id: int=5,
parallel_jobs: int=4,
cve_threshold: int=7,
ignore_if_triaged: bool=True,
reference_group_ids: [int]=[],
):
cfg_factory = ctx().cfg_factory()
protecode_cfg = cfg_factory.protecode(protecode_cfg_name)
component_descriptor = ComponentDescriptor.from_dict(
raw_dict=parse_yaml_file(product_cfg_file)
)
upload_results, license_report = upload_grouped_images(
protecode_cfg=protecode_cfg,
component_descriptor=component_descriptor,
protecode_group_id=protecode_group_id,
parallel_jobs=parallel_jobs,
cve_threshold=cve_threshold,
ignore_if_triaged=ignore_if_triaged,
processing_mode=processing_mode,
reference_group_ids=reference_group_ids,
)
def parse_cfg(cfg_type):
# assume for now that there is exactly one cfg source (file)
cfg_sources = list(cfg_type.sources())
if not len(cfg_sources) == 1:
raise ValueError('currently, only exactly one cfg file is supported per type')
cfg_file = cfg_sources[0].file()
parsed_cfg = parse_yaml_file(os.path.join(cfg_dir, cfg_file))
return parsed_cfg
def resolve_component_descriptor(
component_descriptor: CliHints.existing_file(), ):
cfg_factory = ctx().cfg_factory()
resolver = ComponentDescriptorResolver(cfg_factory=cfg_factory, )
component_descriptor = ComponentDescriptor.from_dict(
parse_yaml_file(component_descriptor))
resolved_descriptor = resolver.resolve_component_references(
product=component_descriptor)
print(yaml.dump(resolved_descriptor.raw))
def add_dependencies(
descriptor_src_file: CliHints.existing_file(),
component_name: str,
component_version: str,
descriptor_out_file: str=None,
component_dependencies: CliHint(action='append')=[],
container_image_dependencies: CliHint(action='append')=[],
web_dependencies: CliHint(action='append')=[],
generic_dependencies: CliHint(action='append')=[],
validation_policies: CliHint(
type=ValidationPolicy,
typehint=[ValidationPolicy],
choices=[policy for policy in ValidationPolicy],
)=[],
):
product = ComponentDescriptor.from_dict(parse_yaml_file(descriptor_src_file))
component = product.component(
ComponentReference.create(name=component_name, version=component_version)
)
if not component:
fail('component {c}:{v} was not found in {f}'.format(
c=component_name,
v=component_version,
f=descriptor_src_file
))
# maintain old behaviour
if not validation_policies:
validation_policies = [ValidationPolicy.FORBID_EXTRA_ATTRIBUTES]
dependencies = _parse_dependencies(
component_dependencies=component_dependencies,
container_image_dependencies=container_image_dependencies,
web_dependencies=web_dependencies,
generic_dependencies=generic_dependencies,
validation_policies=validation_policies,
)
component.add_dependencies(dependencies)
product_dict = {'components': [component.raw]}
print(yaml.dump(product_dict, indent=2))
product_dict = json.loads(json.dumps({'components': [component.raw]}))
if not descriptor_out_file:
print(yaml.dump(product_dict, indent=2))
else:
with open(descriptor_out_file, 'w') as f:
yaml.dump(product_dict, f, indent=2)
def component_descriptor_to_xml(
component_descriptor: CliHints.existing_file(),
out_file: str,
):
component_descriptor = ComponentDescriptor.from_dict(
parse_yaml_file(component_descriptor))
image_references = [
container_image for _, container_image in _enumerate_effective_images(
component_descriptor=component_descriptor)
]
result_xml = product.xml.container_image_refs_to_xml(image_references, )
result_xml.write(out_file)
def _from_cfg_dir(
cfg_dir: str,
disable_cfg_element_lookup: bool,
cfg_types_file='config_types.yaml',
cfg_src_types=None,
lookup_cfg_factory=None,
):
cfg_dir = existing_dir(os.path.abspath(cfg_dir))
cfg_types_dict = parse_yaml_file(os.path.join(cfg_dir, cfg_types_file))
raw = {}
raw[ConfigFactory.CFG_TYPES] = cfg_types_dict
def retrieve_cfg(cfg_type):
cfg_dict = {}
for cfg_src in cfg_type.sources():
if cfg_src_types and type(cfg_src) not in cfg_src_types:
continue
if isinstance(cfg_src, LocalFileCfgSrc):
parsed_cfg = ConfigFactory._parse_local_file(
cfg_dir=cfg_dir,
cfg_src=cfg_src,
)
elif isinstance(cfg_src, GithubRepoFileSrc):
if disable_cfg_element_lookup:
continue
parsed_cfg = ConfigFactory._parse_repo_file(
cfg_src=cfg_src,
lookup_cfg_factory=lookup_cfg_factory,
)
else:
raise NotImplementedError(cfg_src)
for k, v in parsed_cfg.items():
if k in cfg_dict and cfg_dict[k] != v:
raise ValueError(
f'conflicting definition for {k=} in src {cfg_src}'
)
cfg_dict[k] = v
return cfg_dict
return ConfigFactory(
raw_dict=raw,
retrieve_cfg=retrieve_cfg,
)
def add_dependencies(
descriptor_src_file: CliHints.existing_file(),
component_name: str,
component_version: str,
descriptor_out_file: str = None,
component_dependencies: CliHint(typehint=_parse_component_deps,
action='append') = [],
container_image_dependencies: CliHint(typehint=_parse_container_image_deps,
action='append') = [],
web_dependencies: CliHint(typehint=_parse_web_deps, action='append') = [],
generic_dependencies: CliHint(typehint=_parse_generic_deps,
action='append') = [],
):
product = ComponentDescriptor.from_dict(
parse_yaml_file(descriptor_src_file))
component = product.component(
ComponentReference.create(name=component_name,
version=component_version))
if not component:
fail('component {c}:{v} was not found in {f}'.format(
c=component_name, v=component_version, f=descriptor_src_file))
component_deps = component.dependencies()
for component_ref in component_dependencies:
component_deps.add_component_dependency(component_ref)
for image_dep in container_image_dependencies:
component_deps.add_container_image_dependency(image_dep)
for web_dep in web_dependencies:
component_deps.add_web_dependency(web_dep)
for generic_dep in generic_dependencies:
component_deps.add_generic_dependency(generic_dep)
product_dict = json.loads(json.dumps({'components': [component.raw]}))
if not descriptor_out_file:
print(yaml.dump(product_dict, indent=2))
else:
with open(descriptor_out_file, 'w') as f:
yaml.dump(product_dict, f, indent=2)
def _parse_local_file(cfg_dir: str, cfg_src: LocalFileCfgSrc):
cfg_file = cfg_src.file
return parse_yaml_file(os.path.join(cfg_dir, cfg_file))
def parse_product_file(f):
return ComponentDescriptor.from_dict(parse_yaml_file(f))
def _from_cfg_dir(
cfg_dir: str,
cfg_types_file='config_types.yaml',
cfg_src_types=None,
lookup_cfg_factory=None,
):
cfg_dir = existing_dir(os.path.abspath(cfg_dir))
cfg_types_dict = parse_yaml_file(os.path.join(cfg_dir, cfg_types_file))
raw = {}
raw[ConfigFactory.CFG_TYPES] = cfg_types_dict
def parse_cfg(cfg_type):
cfg_dict = {}
def parse_local_file(cfg_src: LocalFileCfgSrc):
cfg_file = cfg_src.file
return parse_yaml_file(os.path.join(cfg_dir, cfg_file))
def parse_repo_file(cfg_src: GithubRepoFileSrc):
import ccc.github
repo_url = cfg_src.repository_url
if not '://' in repo_url:
repo_url = 'https://' + repo_url
repo_url = urllib.parse.urlparse(repo_url)
if not lookup_cfg_factory:
raise RuntimeError(
'cannot resolve non-local cfg w/o bootstrap-cfg-factory'
)
gh_api = ccc.github.github_api(
ccc.github.github_cfg_for_hostname(
repo_url.hostname,
cfg_factory=lookup_cfg_factory,
),
cfg_factory=lookup_cfg_factory,
)
org, repo = repo_url.path.strip('/').split('/')
gh_repo = gh_api.repository(org, repo)
file_contents = gh_repo.file_contents(
path=cfg_src.relpath,
ref=gh_repo.default_branch,
).decoded.decode('utf-8')
return yaml.safe_load(file_contents)
for cfg_src in cfg_type.sources():
if cfg_src_types and type(cfg_src) not in cfg_src_types:
continue
if isinstance(cfg_src, LocalFileCfgSrc):
parsed_cfg = parse_local_file(cfg_src=cfg_src)
elif isinstance(cfg_src, GithubRepoFileSrc):
parsed_cfg = parse_repo_file(cfg_src=cfg_src)
else:
raise NotImplementedError(cfg_src)
for k, v in parsed_cfg.items():
if k in cfg_dict and cfg_dict[k] != v:
raise ValueError(f'conflicting definition for {k=}')
cfg_dict[k] = v
return cfg_dict
# parse all configurations
for cfg_type in map(ConfigType, cfg_types_dict.values()):
cfg_name = cfg_type.cfg_type_name()
raw[cfg_name] = parse_cfg(cfg_type)
return ConfigFactory(raw_dict=raw)
