def validate_celery_config(conf, **kwargs):
"""
Perform basic validatation on the Celery configuration when the worker is initialized.
:param celery.app.utils.Settings conf: the Celery application configuration to validate
:raises iib.exceptions.ConfigError: if the configuration is invalid
"""
if not conf.get('iib_registry'):
raise ConfigError('iib_registry must be set to the destination container registry')
if not conf.get('iib_api_url'):
raise ConfigError('iib_api_url must be set')
if not isinstance(conf['iib_required_labels'], dict):
raise ConfigError('iib_required_labels must be a dictionary')
_validate_iib_org_customizations(conf['iib_organization_customizations'])
iib_request_logs_dir = conf.get('iib_request_logs_dir')
if iib_request_logs_dir:
if not os.path.isdir(iib_request_logs_dir):
raise ConfigError(
f'iib_request_logs_dir, {iib_request_logs_dir}, must exist and be a directory'
)
if not os.access(iib_request_logs_dir, os.W_OK):
raise ConfigError(f'iib_request_logs_dir, {iib_request_logs_dir}, is not writable!')
def validate_celery_config(conf, **kwargs):
"""
Perform basic validatation on the Celery configuration when the worker is initialized.
:param celery.app.utils.Settings conf: the Celery application configuration to validate
:raises iib.exceptions.ConfigError: if the configuration is invalid
"""
if not conf.get('iib_registry'):
raise ConfigError('iib_registry must be set to the destination container registry')
if not conf.get('iib_api_url'):
raise ConfigError('iib_api_url must be set')
if not isinstance(conf['iib_required_labels'], dict):
raise ConfigError('iib_required_labels must be a dictionary')
def validate_api_config(config):
"""
Determine if the configuration is valid.
:param dict config: the dict containing the IIB REST API config
:raises ConfigError: if the config is invalid
"""
if config['IIB_GREENWAVE_CONFIG']:
defined_queue_names = set(config['IIB_USER_TO_QUEUE'].values())
invalid_greenwave_queues = set(
config['IIB_GREENWAVE_CONFIG'].keys()) - defined_queue_names
# The queue_name `None` is the configuration for the default Celery queue
invalid_greenwave_queues.discard(None)
if invalid_greenwave_queues:
raise ConfigError(
f'The following queues are invalid in "IIB_GREENWAVE_CONFIG"'
f': {", ".join(invalid_greenwave_queues)}')
required_params = {
'decision_context', 'product_version', 'subject_type'
}
for queue_name, greenwave_config in config[
'IIB_GREENWAVE_CONFIG'].items():
defined_params = set(greenwave_config.keys())
missing_params = required_params - defined_params
if missing_params:
raise ConfigError(
f'Missing required params {", ".join(missing_params)} for queue {queue_name} '
'in "IIB_GREENWAVE_CONFIG"')
invalid_params = defined_params - required_params
if invalid_params:
raise ConfigError(
f'Invalid params {", ".join(invalid_params)} for queue {queue_name} '
'in "IIB_GREENWAVE_CONFIG"')
if greenwave_config['subject_type'] != 'koji_build':
raise ConfigError(
'IIB only supports gating for subject_type "koji_build". Invalid subject_type '
f'{greenwave_config["subject_type"]} defined for queue '
f'{queue_name} in "IIB_GREENWAVE_CONFIG"')
def validate_celery_config(conf, **kwargs):
"""
Perform basic validatation on the Celery configuration when the worker is initialized.
:param celery.app.utils.Settings conf: the Celery application configuration to validate
:raises iib.exceptions.ConfigError: if the configuration is invalid
"""
if not conf.get('iib_registry'):
raise ConfigError('iib_registry must be set to the destination container registry')
if not conf.get('iib_api_url'):
raise ConfigError('iib_api_url must be set')
if not isinstance(conf['iib_required_labels'], dict):
raise ConfigError('iib_required_labels must be a dictionary')
_validate_iib_org_customizations(conf['iib_organization_customizations'])
if conf.get('iib_aws_s3_bucket_name'):
if not isinstance(conf['iib_aws_s3_bucket_name'], str):
raise ConfigError(
'"iib_aws_s3_bucket_name" must be set to a valid string. '
'This is used for read/write access to the s3 bucket by IIB'
)
if not conf.get('iib_request_logs_dir') or not conf.get('iib_request_related_bundles_dir'):
raise ConfigError(
'"iib_request_logs_dir" and "iib_request_related_bundles_dir" '
'must be set when iib_aws_s3_bucket_name is set.'
)
if (
not os.getenv('AWS_ACCESS_KEY_ID')
or not os.getenv('AWS_SECRET_ACCESS_KEY')
or not os.getenv('AWS_DEFAULT_REGION')
):
raise ConfigError(
'"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" and "AWS_DEFAULT_REGION" '
'environment variables must be set to valid strings when'
'"iib_aws_s3_bucket_name" is set. '
'These are used for read/write access to the s3 bucket by IIB'
)
for directory in ('iib_request_logs_dir', 'iib_request_related_bundles_dir'):
iib_request_temp_data_dir = conf.get(directory)
if iib_request_temp_data_dir:
if not os.path.isdir(iib_request_temp_data_dir):
raise ConfigError(f'{directory} must exist and be a directory')
if not os.access(iib_request_temp_data_dir, os.W_OK):
raise ConfigError(f'{directory}, is not writable!')
def validate_celery_config(conf, **kwargs):
"""
Perform basic validatation on the Celery configuration when the worker is initialized.
:param celery.app.utils.Settings conf: the Celery application configuration to validate
:raises iib.exceptions.ConfigError: if the configuration is invalid
"""
if not conf.get('iib_registry'):
raise ConfigError(
'iib_registry must be set to the destination container registry')
if not conf.get('iib_api_url'):
raise ConfigError('iib_api_url must be set')
if not isinstance(conf['iib_required_labels'], dict):
raise ConfigError('iib_required_labels must be a dictionary')
if not isinstance(conf['iib_organization_customizations'], dict):
raise ConfigError(
'iib_organization_customizations must be a dictionary')
for org, org_config in conf['iib_organization_customizations'].items():
if not isinstance(org, str):
raise ConfigError(
'The keys in iib_organization_customizations must be strings')
if not isinstance(org_config, dict):
raise ConfigError(
'The values in iib_organization_customizations must be dictionaries'
)
invalid_keys = org_config.keys() - {
'csv_annotations',
'package_name_suffix',
'registry_replacements',
}
if invalid_keys:
raise ConfigError(
'The following keys set on iib_organization_customizations are '
f'invalid: {", ".join(sorted(invalid_keys))}')
for key in ('csv_annotations', 'registry_replacements'):
value = org_config.get(key)
if not value:
continue
for k, v in value.items():
if not isinstance(k, str):
raise ConfigError(
f'The keys in iib_organization_customizations.{org}.{key} must be strings'
)
if not isinstance(v, str):
raise ConfigError(
f'The values in iib_organization_customizations.{org}.{key} must be strings'
)
if not isinstance(org_config.get('package_name_suffix', ''), str):
raise ConfigError(
f'The value of iib_organization_customizations.{org}.package_name_suffix '
'must be a string')
iib_request_logs_dir = conf.get('iib_request_logs_dir')
if iib_request_logs_dir:
if not os.path.isdir(iib_request_logs_dir):
raise ConfigError(
f'iib_request_logs_dir, {iib_request_logs_dir}, must exist and be a directory'
)
if not os.access(iib_request_logs_dir, os.W_OK):
raise ConfigError(
f'iib_request_logs_dir, {iib_request_logs_dir}, is not writable!'
)
def _validate_iib_org_customizations(iib_org_customizations):
"""
Validate ``iib_organization_customizations`` celery config variable.
:param dict iib_org_customizations: the value of iib_organization_customizations config
variable
:raises iib.exceptions.ConfigError: if the configuration is invalid
"""
if not isinstance(iib_org_customizations, dict):
raise ConfigError('iib_organization_customizations must be a dictionary')
valid_customizations = {
'csv_annotations': {'annotations'},
'package_name_suffix': {'suffix'},
'registry_replacements': {'replacements'},
'image_name_from_labels': {'template'},
'enclose_repo': {'enclosure_glue', 'namespace'},
}
for org, org_config in iib_org_customizations.items():
if not isinstance(org, str):
raise ConfigError('The org keys in iib_organization_customizations must be strings')
if not isinstance(org_config, list):
raise ConfigError('The org values in iib_organization_customizations must be a list')
for customization in org_config:
if not isinstance(customization, dict):
raise ConfigError(
'Every customization for an org in '
'iib_organization_customizations must be dictionary'
)
customization_type = customization.get('type')
if customization_type not in valid_customizations.keys():
raise ConfigError(
f'Invalid customization in iib_organization_customizations {customization}'
)
invalid_customization_keys = (
customization.keys() - valid_customizations[customization_type] - {'type'}
)
if invalid_customization_keys:
raise ConfigError(
f'The keys {invalid_customization_keys} in iib_organization_customizations'
f'.{org}[{org_config.index(customization)}] are invalid.'
)
if customization_type in ('csv_annotations', 'registry_replacements'):
for valid_key in valid_customizations[customization_type]:
if not customization[valid_key]:
continue
for k, v in customization[valid_key].items():
if not isinstance(k, str):
raise ConfigError(
f'The keys in iib_organization_customizations.{org}'
f'[{org_config.index(customization)}].{valid_key} must be strings'
)
if not isinstance(v, str):
raise ConfigError(
f'The values in iib_organization_customizations.{org}'
f'[{org_config.index(customization)}].{valid_key} must be strings'
)
if customization_type in (
'package_name_suffix',
'image_name_from_labels',
'enclose_repo',
):
for valid_key in valid_customizations[customization_type]:
if not isinstance(customization[valid_key], str):
raise ConfigError(
f'The value of iib_organization_customizations.{org}'
f'[{org_config.index(customization)}].{valid_key} must be a string'
)
def validate_api_config(config):
"""
Determine if the configuration is valid.
:param dict config: the dict containing the IIB REST API config
:raises ConfigError: if the config is invalid
"""
if config['IIB_GREENWAVE_CONFIG']:
defined_queue_names = set(config['IIB_USER_TO_QUEUE'].values())
invalid_greenwave_queues = set(
config['IIB_GREENWAVE_CONFIG'].keys()) - defined_queue_names
# The queue_name `None` is the configuration for the default Celery queue
invalid_greenwave_queues.discard(None)
if invalid_greenwave_queues:
raise ConfigError(
f'The following queues are invalid in "IIB_GREENWAVE_CONFIG"'
f': {", ".join(invalid_greenwave_queues)}')
required_params = {
'decision_context', 'product_version', 'subject_type'
}
for queue_name, greenwave_config in config[
'IIB_GREENWAVE_CONFIG'].items():
defined_params = set(greenwave_config.keys())
missing_params = required_params - defined_params
if missing_params:
raise ConfigError(
f'Missing required params {", ".join(missing_params)} for queue {queue_name} '
'in "IIB_GREENWAVE_CONFIG"')
invalid_params = defined_params - required_params
if invalid_params:
raise ConfigError(
f'Invalid params {", ".join(invalid_params)} for queue {queue_name} '
'in "IIB_GREENWAVE_CONFIG"')
if greenwave_config['subject_type'] != 'koji_build':
raise ConfigError(
'IIB only supports gating for subject_type "koji_build". Invalid subject_type '
f'{greenwave_config["subject_type"]} defined for queue '
f'{queue_name} in "IIB_GREENWAVE_CONFIG"')
if config['IIB_BINARY_IMAGE_CONFIG']:
if not isinstance(config['IIB_BINARY_IMAGE_CONFIG'], dict):
raise ConfigError(
'IIB_BINARY_IMAGE_CONFIG must be a dict mapping distribution_scope to '
'another dict mapping ocp_version to binary_image')
for distribution_scope, value_dict in config[
'IIB_BINARY_IMAGE_CONFIG'].items():
if not isinstance(distribution_scope,
str) or distribution_scope not in (
'dev',
'stage',
'prod',
):
raise ConfigError(
'distribution_scope values must be one of the following'
' "prod", "stage" or "dev" strings.')
if not isinstance(value_dict, dict):
raise ConfigError(
'Value for distribution_scope keys must be a dict mapping'
' ocp_version to binary_image')
for ocp_version, binary_image_value in value_dict.items():
if not isinstance(ocp_version, str) or not isinstance(
binary_image_value, str):
raise ConfigError(
'All ocp_version and binary_image values must be strings.'
)
def validate_api_config(config):
"""
Determine if the configuration is valid.
:param dict config: the dict containing the IIB REST API config
:raises ConfigError: if the config is invalid
"""
if config['IIB_GREENWAVE_CONFIG']:
defined_queue_names = set(config['IIB_USER_TO_QUEUE'].values())
invalid_greenwave_queues = set(
config['IIB_GREENWAVE_CONFIG'].keys()) - defined_queue_names
# The queue_name `None` is the configuration for the default Celery queue
invalid_greenwave_queues.discard(None)
if invalid_greenwave_queues:
raise ConfigError(
f'The following queues are invalid in "IIB_GREENWAVE_CONFIG"'
f': {", ".join(invalid_greenwave_queues)}')
required_params = {
'decision_context', 'product_version', 'subject_type'
}
for queue_name, greenwave_config in config[
'IIB_GREENWAVE_CONFIG'].items():
defined_params = set(greenwave_config.keys())
missing_params = required_params - defined_params
if missing_params:
raise ConfigError(
f'Missing required params {", ".join(missing_params)} for queue {queue_name} '
'in "IIB_GREENWAVE_CONFIG"')
invalid_params = defined_params - required_params
if invalid_params:
raise ConfigError(
f'Invalid params {", ".join(invalid_params)} for queue {queue_name} '
'in "IIB_GREENWAVE_CONFIG"')
if greenwave_config['subject_type'] != 'koji_build':
raise ConfigError(
'IIB only supports gating for subject_type "koji_build". Invalid subject_type '
f'{greenwave_config["subject_type"]} defined for queue '
f'{queue_name} in "IIB_GREENWAVE_CONFIG"')
if config['IIB_BINARY_IMAGE_CONFIG']:
if not isinstance(config['IIB_BINARY_IMAGE_CONFIG'], dict):
raise ConfigError(
'IIB_BINARY_IMAGE_CONFIG must be a dict mapping distribution_scope to '
'another dict mapping ocp_version to binary_image')
for distribution_scope, value_dict in config[
'IIB_BINARY_IMAGE_CONFIG'].items():
if not isinstance(distribution_scope,
str) or distribution_scope not in (
'dev',
'stage',
'prod',
):
raise ConfigError(
'distribution_scope values must be one of the following'
' "prod", "stage" or "dev" strings.')
if not isinstance(value_dict, dict):
raise ConfigError(
'Value for distribution_scope keys must be a dict mapping'
' ocp_version to binary_image')
for ocp_version, binary_image_value in value_dict.items():
if not isinstance(ocp_version, str) or not isinstance(
binary_image_value, str):
raise ConfigError(
'All ocp_version and binary_image values must be strings.'
)
if config['IIB_AWS_S3_BUCKET_NAME'] and (
config['IIB_REQUEST_LOGS_DIR']
or config['IIB_REQUEST_RELATED_BUNDLES_DIR']):
raise ConfigError(
'S3 bucket and local artifacts directories cannot be set together.'
' Either S3 bucket should be configured or "IIB_REQUEST_LOGS_DIR" and '
'"IIB_REQUEST_RELATED_BUNDLES_DIR" must be set. Or "IIB_AWS_S3_BUCKET_NAME"'
'"IIB_REQUEST_LOGS_DIR" and "IIB_REQUEST_RELATED_BUNDLES_DIR" must not be set'
)
if config['IIB_AWS_S3_BUCKET_NAME']:
if not isinstance(config['IIB_AWS_S3_BUCKET_NAME'], str):
raise ConfigError(
'"IIB_AWS_S3_BUCKET_NAME" must be set to a valid string. '
'This is used for read/write access to the s3 bucket by IIB')
if (not os.getenv('AWS_ACCESS_KEY_ID')
or not os.getenv('AWS_SECRET_ACCESS_KEY')
or not os.getenv('AWS_DEFAULT_REGION')):
raise ConfigError(
'"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" and "AWS_DEFAULT_REGION" '
'environment variables must be set to valid strings when'
'"IIB_AWS_S3_BUCKET_NAME" is set. '
'These are used for read/write access to the s3 bucket by IIB')