def version_requirement_validation(service_requirement, requirement_key_list,
manifest_versions, current_validation):
"""
Validates version matches the requirement for a specific service
Args:
service_requirement: service name and its version requirement
requirement_key_list: list of service name which need validation
manifest_versions: versions block from manifest
current_validation (str): service name and version that are currently being validated
Return:
ok(bool): whether the validation succeeded.
"""
ok = True
for required_service in requirement_key_list:
actual_version = get_manifest_version(manifest_versions,
required_service)
if not actual_version:
logger.error(
'Service "{}" not in manifest but required to validate "{}" with "{}"'
.format(required_service, current_validation,
service_requirement))
ok = False
continue
if version_is_branch(actual_version):
# ignoring service on branch - user was already informed of this
# by the log in `manifest_version()`
continue
if ("min" not in service_requirement[required_service]
and "max" not in service_requirement[required_service]):
ok = (assert_and_log(
version.parse(
service_requirement[required_service]) <= actual_version,
'Service "{}" version "{}" does not respect requirement "{}" for "{}"'
.format(
required_service,
actual_version,
service_requirement,
current_validation,
),
) and ok)
elif ("min" in service_requirement[required_service]
and "max" in service_requirement[required_service]):
ok = (assert_and_log(
version.parse(service_requirement[required_service]["min"]) <=
actual_version < version.parse(
service_requirement[required_service]["max"]),
'Service "{}" version "{}" does not respect requirement "{}" for "{}"'
.format(
required_service,
actual_version,
service_requirement,
current_validation,
),
) and ok)
return ok
def validate_against_dictionary(gitops, data_dictionary):
"""
Validates gitops.json configuration against a data dictionary
Args:
gitops (dict): gitops.json config
data_dictionary (str): url of data dictionary
Returns:
ok(bool): whether the validation succeeded.
"""
_, model = init_dictionary(data_dictionary)
schema = model.dictionary.schema
ok = True
graphql = gitops["graphql"]
for item in graphql["boardCounts"]:
node_count = item["graphql"]
# assumes form _{node}_count
idx = node_count.rfind("_")
node = node_count[1:idx]
ok = ok and assert_and_log(
schema.get(node) is not None,
"Node: {} in graphql.boardCounts not found in dictionary".format(
node),
)
for item in graphql["chartCounts"]:
node_count = item["graphql"]
# assumes form _{node}_count
idx = node_count.rfind("_")
node = node_count[1:idx]
ok = ok and assert_and_log(
schema.get(node) is not None,
"Node: {} in graphql.chartCounts not found in dictionary".format(
node),
)
for item in graphql.get("homepageChartNodes", []):
node = item["node"]
ok = ok and assert_and_log(
schema.get(node) is not None,
"Node: {} in graphql.homepageChartNodes not found in dictionary".
format(node),
)
return ok
def versions_validation(manifest_versions, versions_requirements):
"""
Validates versions in cdis-manifest
Arg:
manifest_versions: manifest.json "versions" section
versions_requirements: the "versions" requirement under validation_config.yaml
Return:
ok(bool): whether the validation succeeded.
"""
ok = True
for versions_requirement in versions_requirements:
requirement_list = versions_requirement["needs"]
requirement_key_list = list(requirement_list.keys())
requirement_key = list(versions_requirement)[0]
actual_version = get_manifest_version(manifest_versions,
requirement_key)
if requirement_key in manifest_versions and not version_is_branch(
actual_version):
required_version = versions_requirement[requirement_key]
# If the first service set to * under validation_config versions, other services should be in the manifest
# The second condition is ignoring branch on sevice. WHICH IS NOT GOO. Added a warning in the log
if required_version == "*":
for required_service in requirement_key_list:
ok = (assert_and_log(
required_service in manifest_versions,
required_service + " is missing in manifest.json",
) and ok)
elif ("min" not in required_version
and "max" not in required_version
and version.parse(required_version) <= actual_version):
# If the first service set to a specific version in validation_config, other services should matches the version requirements
ok = (version_requirement_validation(
requirement_list,
requirement_key_list,
manifest_versions,
"{} {}".format(requirement_key, actual_version),
) and ok)
elif ("min" in required_version and "max" in required_version
and version.parse(required_version["min"]) <= actual_version
< version.parse(required_version["max"])):
# if service is min_requirement <= service_version < max_requirement, other services should matches the version requirements
ok = (version_requirement_validation(
requirement_list,
requirement_key_list,
manifest_versions,
"{} {}".format(requirement_key, actual_version),
) and ok)
return ok
def validate_manifest_block(manifest, blocks_requirements):
"""
Validates blocks in cdis-manifest.
Args:
manifest (dict): Contents of manifest.json file.
blocks_requirements (dict): the "block" requirement under
validation_config.yaml. The keys of the dict are the service names
and the values are the requirements for the block associated to
the service.
Return:
ok(bool): whether the validation succeeded.
"""
ok = True
for service_name, block in blocks_requirements.items():
if service_name in manifest["versions"]:
# Validation for all services has requirement in validation_config.
block_requirement_version = get_manifest_version(
manifest["versions"], service_name)
is_branch = version_is_branch(block_requirement_version)
should_check_has = type(
block) == dict and "has" in block and not is_branch
if type(block) == dict and "version" in block and not is_branch:
# If we have version requirement for a service, min or max is required.
# We are not validating branch or master branch
# min: Version in manifest is equal or greater than the verson in validation_config
# max: Version in manifest is smaller than the verson in validation_config
min_version = block["version"].get("min")
max_version = block["version"].get("max")
if min_version:
min_version = version.parse(min_version)
if max_version:
max_version = version.parse(max_version)
if min_version and max_version:
should_check_has = (
min_version <= block_requirement_version < max_version)
elif min_version:
should_check_has = block_requirement_version >= min_version
elif max_version:
should_check_has = block_requirement_version < max_version
if should_check_has:
# Validation to check if a service has a specific key in its block in cdis-manfiest
error_msg = "{} is missing in {} block or {} block is missing".format(
block["has"], service_name, service_name)
ok = (assert_and_log(
service_name in manifest
and block["has"] in manifest[service_name]
or service_name not in manifest and block.get("optional"),
error_msg,
) and ok)
if block is True:
# Validation to check if a block exists in cdis-manifest
ok = (assert_and_log(
service_name in manifest,
service_name + " block is missing in cdis-manifest",
) and ok)
return ok
def validate_gitops_syntax(gitops):
"""
Validates the syntax of gitops.json by checking for required fields
Args:
gitops (dict): gitops.json config
Returns:
Error: Returns the error if it exists, else returns None.
TODO
Currently only checks syntax needed for etl mapping and dictionary validation.
"""
graphql = gitops.get("graphql")
ok = assert_and_log(graphql, FieldSyntaxError("graphql"))
if graphql:
ok = ok and assert_and_log("boardCounts" in graphql,
FieldSyntaxError("graphql.boardCounts"))
boardcounts = graphql.get("boardCounts", [])
if boardcounts:
for item in boardcounts:
checks = ["graphql", "name", "plural"]
ok = check_required_fields("graphql.boardCounts", checks, item,
ok)
ok = ok and assert_and_log("chartCounts" in graphql,
FieldSyntaxError("graphql.chartCounts"))
components = gitops.get("components")
ok = ok and assert_and_log(components, FieldSyntaxError("components"))
if components:
index = components.get("index")
ok = ok and assert_and_log(index, FieldSyntaxError("components.index"))
if index:
homepage = index.get("homepageChartNodes", [])
ok = ok and assert_and_log(
index, FieldSyntaxError("components.homepageChartNodes"))
for item in homepage:
checks = ["node", "name"]
ok = check_required_fields("components.index.homepage", checks,
item, ok)
# footerLogos is optional, but when present, it must be an array (list)
footerLogos = components.get("footerLogos", [])
ok = ok and assert_and_log(
isinstance(footerLogos, list),
FieldError(
"footerLogos must be an array if presents in the config"),
)
explorer_enabled = gitops.get("featureFlags", {}).get("explorer", True)
explorerconfig = gitops.get("explorerConfig")
configs = []
if not explorerconfig:
dataConfig = gitops.get("dataExplorerConfig")
fileConfig = gitops.get("fileExplorerConfig")
ok = ok and assert_and_log(dataConfig or not explorer_enabled,
FieldSyntaxError("(data)explorerConfig"))
if dataConfig:
configs.append(dataConfig)
if fileConfig:
configs.append(fileConfig)
else:
configs = explorerconfig
for exp_config in configs:
filters = exp_config.get("filters")
ok = ok and assert_and_log(filters,
FieldSyntaxError("explorerConfig.filters"))
if filters:
tabs = filters.get("tabs", [])
ok = ok and assert_and_log(
tabs, FieldSyntaxError("explorerConfig.filters.tabs"))
for tab in tabs:
checks = ["title", "fields"]
ok = check_required_fields("explorerConfig.filters.tab",
checks, tab, ok)
guppy = exp_config.get("guppyConfig")
ok = ok and assert_and_log(
guppy, FieldSyntaxError("explorerConfig.guppyConfig"))
buttons = exp_config.get("buttons", [])
manifest_mapping = None
if guppy:
ok = ok and assert_and_log(
guppy.get("dataType"),
FieldSyntaxError("explorerConfig.guppyConfig.dataType"),
)
manifest_mapping = guppy.get("manifestMapping")
val_mapping = False
for button in buttons:
if button.get("enabled") and button.get("type") == "manifest":
val_mapping = True
if manifest_mapping and val_mapping:
checks = [
"resourceIndexType",
"resourceIdField",
"referenceIdFieldInResourceIndex",
"referenceIdFieldInDataIndex",
]
ok = check_required_fields(
"explorerConfig.guppyConfig.manifestMapping",
checks,
manifest_mapping,
ok,
)
study_viewer = gitops.get("studyViewerConfig")
if study_viewer:
checks = [
"dataType",
"listItemConfig",
"rowAccessor",
]
for viewer in study_viewer:
ok = check_required_fields("studyViewerConfig", checks, viewer, ok)
return ok
def check_required_fields(path, checks, field, ok):
for check in checks:
ok = ok and assert_and_log(field.get(check),
FieldSyntaxError(f"{path}.{check}"))
return ok