def run(self):
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context,
self._file_name, entry)
if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \
entry:
self._summary_collector.add_error(
"Missing git-url, git-path, git-branch or target-file")
self._success = False
continue
clone_location = IndexProjectsValidator.update_git_url(
self._context.environment.repo_dump, entry["git-url"],
entry["git-branch"])
if not clone_location:
self._summary_collector.add_error(
"Unable to clone specified git-url or find specified git-branch"
)
self._success = False
continue
validation_path = clone_location + "/" + str(
entry["git-path"]) + "/" + entry["target-file"]
if not path.exists(validation_path):
self._summary_collector.add_error(
"Invalid git-path or target-file specified")
self._success = False
if self._success:
DependencyValidationUpdater(context=self._context,
index_file=self._index_file).run()
return self._success
class LightWeightValidator(IndexValidator):
"""Light weight validator does minimum validation, and focuses mostly on building dependency graph"""
def __init__(self, context, index_file):
IndexValidator.__init__(self, context, index_file)
def run(self):
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \
entry:
self._summary_collector.add_error("Missing git-url, git-path, git-branch or target-file")
self._success = False
continue
clone_location = IndexProjectsValidator.update_git_url(self._context.environment.repo_dump,
entry["git-url"], entry["git-branch"])
if not clone_location:
self._summary_collector.add_error("Unable to clone specified git-url or find specified git-branch")
self._success = False
continue
validation_path = clone_location + "/" + str(entry["git-path"]) + "/" + entry["target-file"]
if not path.exists(validation_path):
self._summary_collector.add_error("Invalid git-path or target-file specified")
self._success = False
if self._success:
DependencyValidationUpdater(context=self._context, index_file=self._index_file).run()
return self._success
class LightWeightValidator(IndexValidator):
"""Light weight validator does minimum validation, and focuses mostly on building dependency graph"""
def __init__(self, context, index_file):
IndexValidator.__init__(self, context, index_file)
def run(self):
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \
entry:
self._summary_collector.add_error("Missing git-url, git-path, git-branch or target-file")
self._success = False
continue
clone_location = IndexProjectsValidator.update_git_url(self._context.environment.repo_dump,
entry["git-url"], entry["git-branch"])
if not clone_location:
self._summary_collector.add_error("Unable to clone specified git-url or find specified git-branch")
self._success = False
continue
validation_path = clone_location + "/" + entry["git-path"] + "/" + entry["target-file"]
if not path.exists(validation_path):
self._summary_collector.add_error("Invalid git-path or target-file specified")
self._success = False
if self._success:
DependencyValidationUpdater(context=self._context, index_file=self._index_file).run()
return self._success
def run(self):
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \
entry:
self._summary_collector.add_error("Missing git-url, git-path, git-branch or target-file")
self._success = False
continue
clone_location = IndexProjectsValidator.update_git_url(self._context.environment.repo_dump,
entry["git-url"], entry["git-branch"])
if not clone_location:
self._summary_collector.add_error("Unable to clone specified git-url or find specified git-branch")
self._success = False
continue
validation_path = clone_location + "/" + entry["git-path"] + "/" + entry["target-file"]
if not path.exists(validation_path):
self._summary_collector.add_error("Invalid git-path or target-file specified")
self._success = False
if self._success:
DependencyValidationUpdater(context=self._context, index_file=self._index_file).run()
return self._success
def run(self):
if not self._success:
return False, self._status_list
id_list = []
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
self._mark_entry_valid(entry)
# Check if id field exists
if "id" not in entry or ("id" in entry and entry["id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing id")
else:
# Check if id has not already been passed
if entry["id"] in id_list:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("id field must be unique in the file")
else:
id_list.append(entry["id"])
# Check for pre-build script and pre-build context
if constants.PREBUILD_SCRIPT in entry and entry[constants.PREBUILD_SCRIPT] is not None:
prebuild_path = entry.get(constants.PREBUILD_CONTEXT)
if not prebuild_path:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("If pre-build script is specified,"
" then prebuild-context should also "
"be specified")
# Checking app-id field
if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing app-id")
else:
if entry["app-id"] != self._file_name.split(".")[0]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("app-id should be same as first part of the file name")
if "_" in entry["app-id"] or "/" in entry["app-id"] or "." in entry["app-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("app-id cannot contain _, / or . character.")
# Checking job-id field
if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing job-id field")
else:
try:
int(entry["job-id"])
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Job id must be a string")
except ValueError:
pass
if "_" in entry["job-id"] or "/" in entry["job-id"] or "." in entry["job-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("job-id cannot contain _, / or . character.")
# Check for git-url
if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-url")
else:
if "gitlab." in entry["git-url"] and not entry["git-url"].endswith(".git"):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Git urls from gitlab must end with .git, try {0}.git".format(entry["git-url"]))
# Checking git-path
if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-path")
# Check git-branch
if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-branch")
# Check target-file
if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing target-file")
# Check desired-tag
if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing desired-tag")
# Check for build-context
# Ideally, build-context will be a compulsory field but for now im just checking its None
# TODO : Need to update this to make it compulsory
if constants.BUILD_CONTEXT not in entry and not (constants.BUILD_CONTEXT in entry and
entry[constants.BUILD_CONTEXT] is None):
pass
# Check notify-email
if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing notify-email")
# Check depends-on
if "depends-on" not in entry:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing depends-on")
elif entry["depends-on"]:
depends_on = entry["depends-on"]
if not isinstance(depends_on, list):
depends_on = [depends_on]
matcher = re.compile("^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}"
"[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$")
for item in depends_on:
if not matcher.search(str(item)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Depends on entry pattern mismatch found {0} must be"
" <string>/<string>:<string>, ".format(str(item)))
return self._success, self._status_list
def run(self):
if not self._success:
return False, self._status_list
container_names = {}
for entry in self._index_data:
self._mark_entry_valid(entry)
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"])
if clone_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Could not clone specified git-url or could not find specified branch")
continue
# Else clone was success, check the git path
git_path = clone_path + "/" + str(entry["git-path"])
# * Check for pre-build script
# TODO : Make a better implementation of pre-build script checking
# TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None
prebuild_exists = False
if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry:
prebuild_exists = True
prebuild_script = entry.get(constants.PREBUILD_SCRIPT)
prebuild_context = entry.get(constants.PREBUILD_CONTEXT)
if (prebuild_script and not path.exists(path.join(git_path, prebuild_script))
and prebuild_context and not path.exists(path.join(git_path, prebuild_context))):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Invalid pre-build script or path specified")
# * Check for existence of target-file
if not prebuild_exists and not path.exists(git_path + "/" + entry["target-file"]):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified target-file does not exist at the git-path")
# Check if specified path exists
if not prebuild_exists and not path.exists(git_path):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified git-path does not exist in git repo.")
continue
# Else, continue with remaining checks
cccp_yml_path = None
# * Check if cccp.yml file exists
for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]:
check_path = git_path + "/" + item
if path.exists(check_path):
cccp_yml_path = check_path
break
if not prebuild_exists and cccp_yml_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing cccp yml file, please check your git-path")
continue
# * Check for duplicate entry for same container name
container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"])
if container_name in container_names:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Duplicate entry exists at ids : " + str(container_names[container_name]))
else:
container_names[container_name] = []
container_names[container_name].append(entry["id"])
# * Check for build-context
if constants.BUILD_CONTEXT in entry:
build_context = entry.get(constants.BUILD_CONTEXT)
if not prebuild_exists and build_context and not path.exists(path.join(clone_path, git_path, build_context)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Specified build context does not exist.")
# * Validate the cccp yml file
if not prebuild_exists:
self._cccp_yml_check(git_path, cccp_yml_path, entry)
return self._success, self._status_list
class IndexProjectsValidator(IndexValidator):
"""Does deeper analysis of index, checking for correctness of provided values."""
def __init__(self, context, index_file):
IndexValidator.__init__(self, context, index_file)
@staticmethod
def update_git_url(repo_dump, git_url, git_branch):
clone_path = None
# Work out the path to clone repo to
clone_to = git_url
if ":" in clone_to:
clone_to = clone_to.split(":")[1]
clone_to = repo_dump + "/" + clone_to
# If the path doesnt already exist, attempt to clone repo
if not path.exists(clone_to):
cmd = ["git", "clone", git_url, clone_to]
if not execute_command(cmd):
return None
# Update repo
get_back = getcwd()
chdir(clone_to)
cmd = "git branch -r | grep -v '\->' | while read remote; do git branch --track \"${remote#origin/}\"" \
" \"$remote\" &> /dev/null; done"
# Get all the branches
system(cmd)
# fetch the branches
cmd = ["git", "fetch", "--all"]
execute_command(cmd)
# Pull for update
cmd = ["git", "pull", "--all"]
execute_command(cmd)
# Checkout required branch
cmd = ["git", "checkout", "origin/" + git_branch]
if execute_command(cmd):
clone_path = clone_to
chdir(get_back)
return clone_path
def run(self):
if not self._success:
return False, self._status_list
container_names = {}
for entry in self._index_data:
self._mark_entry_valid(entry)
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"])
if clone_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Could not clone specified git-url or could not find specified branch")
continue
# Else clone was success, check the git path
git_path = clone_path + "/" + str(entry["git-path"])
# * Check for pre-build script
# TODO : Make a better implementation of pre-build script checking
# TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None
prebuild_exists = False
if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry:
prebuild_exists = True
prebuild_script = entry.get(constants.PREBUILD_SCRIPT)
prebuild_context = entry.get(constants.PREBUILD_CONTEXT)
if (prebuild_script and not path.exists(path.join(git_path, prebuild_script))
and prebuild_context and not path.exists(path.join(git_path, prebuild_context))):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Invalid pre-build script or path specified")
# * Check for existence of target-file
if not prebuild_exists and not path.exists(git_path + "/" + entry["target-file"]):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified target-file does not exist at the git-path")
# Check if specified path exists
if not prebuild_exists and not path.exists(git_path):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified git-path does not exist in git repo.")
continue
# Else, continue with remaining checks
cccp_yml_path = None
# * Check if cccp.yml file exists
for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]:
check_path = git_path + "/" + item
if path.exists(check_path):
cccp_yml_path = check_path
break
if not prebuild_exists and cccp_yml_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing cccp yml file, please check your git-path")
continue
# * Check for duplicate entry for same container name
container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"])
if container_name in container_names:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Duplicate entry exists at ids : " + str(container_names[container_name]))
else:
container_names[container_name] = []
container_names[container_name].append(entry["id"])
# * Check for build-context
if constants.BUILD_CONTEXT in entry:
build_context = entry.get(constants.BUILD_CONTEXT)
if not prebuild_exists and build_context and not path.exists(path.join(clone_path, git_path, build_context)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Specified build context does not exist.")
# * Validate the cccp yml file
if not prebuild_exists:
self._cccp_yml_check(git_path, cccp_yml_path, entry)
return self._success, self._status_list
def _cccp_yml_check(self, git_path, cccp_yaml_path, entry):
"""Validates the cccp yaml file"""
temp_cccp, ex = self._load_yml(cccp_yaml_path)
if not temp_cccp:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Malformed cccp yml : " + str(ex))
cccp_yaml = temp_cccp
self._entry_valid = True
get_back = getcwd()
chdir(git_path)
# * Check for job-entry_id
if "job-id" not in cccp_yaml:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing job-id field in cccp yaml")
# * Check for test-skip
if "test-skip" in cccp_yaml:
value = cccp_yaml["test-skip"]
try:
if value and (value is True or value is False):
pass
if value is None:
self._summary_collector.add_warning("Optional test-skip is set None, which means its value will be"
" ignored")
except Exception as ex:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("test-skip should either be True or False as it is a flag")
# * Check test-script
if "test-script" in cccp_yaml:
self._summary_collector.add_warning("Custom test-script has been specified")
value = cccp_yaml["test-script"]
if value and not path.exists(str(value)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified test-script does not exist")
if value is None:
self._summary_collector.add_warning("Optional test-script has a value of None, which means it will be"
" ignored")
# * Check build-script
if "build-script" in cccp_yaml:
self._summary_collector.add_warning("Custom build-script has been specified")
value = cccp_yaml["build-script"]
if value and not path.exists(str(value)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified build-script does not exist")
if value is None:
self._summary_collector.add_warning("Optional build-script has has a value None, which means it will be"
" ignored")
# * Check delivery-script
if "delivery-script" in cccp_yaml:
self._summary_collector.add_warning("Custom delivery-script has been specified.")
value = cccp_yaml["delivery-script"]
if value and not path.exists(str(value)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified delivery-script does not exist")
if value is None:
self._summary_collector.add_warning("Optional delivery script has value None, which means it will be"
" ignored")
chdir(get_back)
def run(self):
if not self._success:
return False, self._status_list
id_list = []
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context,
self._file_name, entry)
self._mark_entry_valid(entry)
# Check if id field exists
if "id" not in entry or ("id" in entry and entry["id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing id")
else:
# Check if id has not already been passed
if entry["id"] in id_list:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"id field must be unique in the file")
else:
id_list.append(entry["id"])
# Check for pre-build script and pre-build context
if constants.PREBUILD_SCRIPT in entry and entry[
constants.PREBUILD_SCRIPT] is not None:
prebuild_path = entry.get(constants.PREBUILD_CONTEXT)
if not prebuild_path:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"If pre-build script is specified,"
" then prebuild-context should also "
"be specified")
# Checking app-id field
if "app-id" not in entry or ("app-id" in entry
and entry["app-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing app-id")
else:
if entry["app-id"] != self._file_name.split(".")[0]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"app-id should be same as first part of the file name")
if "_" in entry["app-id"] or "/" in entry[
"app-id"] or "." in entry["app-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"app-id cannot contain _, / or . character.")
# Checking job-id field
if "job-id" not in entry or ("job-id" in entry
and entry["job-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing job-id field")
else:
try:
int(entry["job-id"])
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Job id must be a string")
except ValueError:
pass
if "_" in entry["job-id"] or "/" in entry[
"job-id"] or "." in entry["job-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"job-id cannot contain _, / or . character.")
# Check for git-url
if "git-url" not in entry or ("git-url" in entry
and entry["git-url"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-url")
else:
if "gitlab." in entry[
"git-url"] and not entry["git-url"].endswith(".git"):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Git urls from gitlab must end with .git, try {0}.git".
format(entry["git-url"]))
# Checking git-path
if "git-path" not in entry or ("git-path" in entry
and entry["git-path"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-path")
# Check git-branch
if "git-branch" not in entry or ("git-branch" in entry
and entry["git-branch"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-branch")
# Check target-file
if "target-file" not in entry or ("target-file" in entry and
entry["target-file"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing target-file")
# Check desired-tag
if "desired-tag" not in entry or ("desired-tag" in entry and
entry["desired-tag"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing desired-tag")
# Check for build-context
# Ideally, build-context will be a compulsory field but for now im just checking its None
# TODO : Need to update this to make it compulsory
if constants.BUILD_CONTEXT not in entry and not (
constants.BUILD_CONTEXT in entry
and entry[constants.BUILD_CONTEXT] is None):
pass
# Check notify-email
if "notify-email" not in entry or ("notify-email" in entry and
entry["notify-email"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing notify-email")
# Check depends-on
if "depends-on" not in entry:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing depends-on")
elif entry["depends-on"]:
depends_on = entry["depends-on"]
if not isinstance(depends_on, list):
depends_on = [depends_on]
matcher = re.compile(
"^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}"
"[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$")
for item in depends_on:
if not matcher.search(str(item)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Depends on entry pattern mismatch found {0} must be"
" <string>/<string>:<string>, ".format(str(item)))
return self._success, self._status_list
def run(self):
if not self._success:
return False, self._status_list
container_names = {}
for entry in self._index_data:
self._mark_entry_valid(entry)
self._summary_collector = SummaryCollector(self._context,
self._file_name, entry)
clone_path = self.update_git_url(
self._context.environment.repo_dump, entry["git-url"],
entry["git-branch"])
if clone_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Could not clone specified git-url or could not find specified branch"
)
continue
# Else clone was success, check the git path
git_path = clone_path + "/" + str(entry["git-path"])
# * Check for pre-build script
# TODO : Make a better implementation of pre-build script checking
# TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None
prebuild_exists = False
if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry:
prebuild_exists = True
prebuild_script = entry.get(constants.PREBUILD_SCRIPT)
prebuild_context = entry.get(constants.PREBUILD_CONTEXT)
if (prebuild_script and
not path.exists(path.join(git_path, prebuild_script))
and prebuild_context and not path.exists(
path.join(git_path, prebuild_context))):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Invalid pre-build script or path specified")
# * Check for existence of target-file
if not prebuild_exists and not path.exists(git_path + "/" +
entry["target-file"]):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified target-file does not exist at the git-path")
# Check if specified path exists
if not prebuild_exists and not path.exists(git_path):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified git-path does not exist in git repo.")
continue
# Else, continue with remaining checks
cccp_yml_path = None
# * Check if cccp.yml file exists
for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]:
check_path = git_path + "/" + item
if path.exists(check_path):
cccp_yml_path = check_path
break
if not prebuild_exists and cccp_yml_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Missing cccp yml file, please check your git-path")
continue
# * Check for duplicate entry for same container name
container_name = entry["app-id"] + "/" + entry[
"job-id"] + ":" + str(entry["desired-tag"])
if container_name in container_names:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Duplicate entry exists at ids : " +
str(container_names[container_name]))
else:
container_names[container_name] = []
container_names[container_name].append(entry["id"])
# * Check for build-context
if constants.BUILD_CONTEXT in entry:
build_context = entry.get(constants.BUILD_CONTEXT)
if not prebuild_exists and build_context and not path.exists(
path.join(clone_path, git_path, build_context)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Specified build context does not exist.")
# * Validate the cccp yml file
if not prebuild_exists:
self._cccp_yml_check(git_path, cccp_yml_path, entry)
return self._success, self._status_list
class IndexProjectsValidator(IndexValidator):
"""Does deeper analysis of index, checking for correctness of provided values."""
def __init__(self, context, index_file):
IndexValidator.__init__(self, context, index_file)
@staticmethod
def update_git_url(repo_dump, git_url, git_branch):
clone_path = None
# Work out the path to clone repo to
clone_to = git_url
if ":" in clone_to:
clone_to = clone_to.split(":")[1]
clone_to = repo_dump + "/" + clone_to
# If the path doesnt already exist, attempt to clone repo
if not path.exists(clone_to):
cmd = ["git", "clone", git_url, clone_to]
if not execute_command(cmd):
return None
# Update repo
get_back = getcwd()
chdir(clone_to)
cmd = "git branch -r | grep -v '\->' | while read remote; do git branch --track \"${remote#origin/}\"" \
" \"$remote\" &> /dev/null; done"
# Get all the branches
system(cmd)
# fetch the branches
cmd = ["git", "fetch", "--all"]
execute_command(cmd)
# Pull for update
cmd = ["git", "pull", "--all"]
execute_command(cmd)
# Checkout required branch
cmd = ["git", "checkout", "origin/" + git_branch]
if execute_command(cmd):
clone_path = clone_to
chdir(get_back)
return clone_path
def run(self):
if not self._success:
return False, self._status_list
container_names = {}
for entry in self._index_data:
self._mark_entry_valid(entry)
self._summary_collector = SummaryCollector(self._context,
self._file_name, entry)
clone_path = self.update_git_url(
self._context.environment.repo_dump, entry["git-url"],
entry["git-branch"])
if clone_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Could not clone specified git-url or could not find specified branch"
)
continue
# Else clone was success, check the git path
git_path = clone_path + "/" + str(entry["git-path"])
# * Check for pre-build script
# TODO : Make a better implementation of pre-build script checking
# TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None
prebuild_exists = False
if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry:
prebuild_exists = True
prebuild_script = entry.get(constants.PREBUILD_SCRIPT)
prebuild_context = entry.get(constants.PREBUILD_CONTEXT)
if (prebuild_script and
not path.exists(path.join(git_path, prebuild_script))
and prebuild_context and not path.exists(
path.join(git_path, prebuild_context))):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Invalid pre-build script or path specified")
# * Check for existence of target-file
if not prebuild_exists and not path.exists(git_path + "/" +
entry["target-file"]):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified target-file does not exist at the git-path")
# Check if specified path exists
if not prebuild_exists and not path.exists(git_path):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified git-path does not exist in git repo.")
continue
# Else, continue with remaining checks
cccp_yml_path = None
# * Check if cccp.yml file exists
for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]:
check_path = git_path + "/" + item
if path.exists(check_path):
cccp_yml_path = check_path
break
if not prebuild_exists and cccp_yml_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Missing cccp yml file, please check your git-path")
continue
# * Check for duplicate entry for same container name
container_name = entry["app-id"] + "/" + entry[
"job-id"] + ":" + str(entry["desired-tag"])
if container_name in container_names:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Duplicate entry exists at ids : " +
str(container_names[container_name]))
else:
container_names[container_name] = []
container_names[container_name].append(entry["id"])
# * Check for build-context
if constants.BUILD_CONTEXT in entry:
build_context = entry.get(constants.BUILD_CONTEXT)
if not prebuild_exists and build_context and not path.exists(
path.join(clone_path, git_path, build_context)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Specified build context does not exist.")
# * Validate the cccp yml file
if not prebuild_exists:
self._cccp_yml_check(git_path, cccp_yml_path, entry)
return self._success, self._status_list
def _cccp_yml_check(self, git_path, cccp_yaml_path, entry):
"""Validates the cccp yaml file"""
temp_cccp, ex = self._load_yml(cccp_yaml_path)
if not temp_cccp:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Malformed cccp yml : " +
str(ex))
cccp_yaml = temp_cccp
self._entry_valid = True
get_back = getcwd()
chdir(git_path)
# * Check for job-entry_id
if "job-id" not in cccp_yaml:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Missing job-id field in cccp yaml")
# * Check for test-skip
if "test-skip" in cccp_yaml:
value = cccp_yaml["test-skip"]
try:
if value and (value is True or value is False):
pass
if value is None:
self._summary_collector.add_warning(
"Optional test-skip is set None, which means its value will be"
" ignored")
except Exception as ex:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"test-skip should either be True or False as it is a flag")
# * Check test-script
if "test-script" in cccp_yaml:
self._summary_collector.add_warning(
"Custom test-script has been specified")
value = cccp_yaml["test-script"]
if value and not path.exists(str(value)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified test-script does not exist")
if value is None:
self._summary_collector.add_warning(
"Optional test-script has a value of None, which means it will be"
" ignored")
# * Check build-script
if "build-script" in cccp_yaml:
self._summary_collector.add_warning(
"Custom build-script has been specified")
value = cccp_yaml["build-script"]
if value and not path.exists(str(value)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified build-script does not exist")
if value is None:
self._summary_collector.add_warning(
"Optional build-script has has a value None, which means it will be"
" ignored")
# * Check delivery-script
if "delivery-script" in cccp_yaml:
self._summary_collector.add_warning(
"Custom delivery-script has been specified.")
value = cccp_yaml["delivery-script"]
if value and not path.exists(str(value)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"The specified delivery-script does not exist")
if value is None:
self._summary_collector.add_warning(
"Optional delivery script has value None, which means it will be"
" ignored")
chdir(get_back)
class IndexFormatValidator(IndexValidator):
"""Checks the format of index files"""
def __init__(self, context, index_file):
IndexValidator.__init__(self, context, index_file)
def run(self):
if not self._success:
return False, self._status_list
id_list = []
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
self._mark_entry_valid(entry)
# Check if id field exists
if "id" not in entry or ("id" in entry and entry["id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing id")
else:
# Check if id has not already been passed
if entry["id"] in id_list:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("id field must be unique in the file")
else:
id_list.append(entry["id"])
# Checking app-id field
if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing app-id")
else:
if entry["app-id"] != self._file_name.split(".")[0]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("app-id should be same as first part of the file name")
if "_" in entry["app-id"] or "/" in entry["app-id"] or "." in entry["app-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("app-id cannot contain _, / or . character.")
# Checking job-id field
if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing job-id field")
else:
try:
int(entry["job-id"])
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Job id must be a string")
except ValueError:
pass
if "_" in entry["job-id"] or "/" in entry["job-id"] or "." in entry["job-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("job-id cannot contain _, / or . character.")
# Check for git-url
if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-url")
else:
if "gitlab." in entry["git-url"] and not entry["git-url"].endswith(".git"):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Git urls from gitlab must end with .git, try {0}.git".format(entry["git-url"]))
# Checking git-path
if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-path")
# Check git-branch
if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-branch")
# Check target-file
if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing target-file")
# Check desired-tag
if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing desired-tag")
# Check notify-email
if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing notify-email")
# Check depends-on
if "depends-on" not in entry:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing depends-on")
elif entry["depends-on"]:
depends_on = entry["depends-on"]
if not isinstance(depends_on, list):
depends_on = [depends_on]
matcher = re.compile("^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}"
"[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$")
for item in depends_on:
if not matcher.search(str(item)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Depends on entry pattern mismatch found {0} must be"
" <string>/<string>:<string>, ".format(str(item)))
return self._success, self._status_list
def run(self):
if not self._success:
return False, self._status_list
container_names = {}
for entry in self._index_data:
self._mark_entry_valid(entry)
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"])
if clone_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Could not clone specified git-url or could not find specified branch")
continue
# Else clone was success, check the git path
git_path = clone_path + "/" + entry["git-path"]
# Check if specified path exists
if not path.exists(git_path):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified git-path does not exist in git repo.")
continue
# Else, continue with remaining checks
cccp_yml_path = None
# * Check if cccp.yml file exists
for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]:
check_path = git_path + "/" + item
if path.exists(check_path):
cccp_yml_path = check_path
break
if cccp_yml_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing cccp yml file, please check your git-path")
continue
# * Check for duplicate entry for same container name
container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"])
if container_name in container_names:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Duplicate entry exists at ids : " + str(container_names[container_name]))
else:
container_names[container_name] = []
container_names[container_name].append(entry["id"])
# * Check for existence of target-file
if not path.exists(git_path + "/" + entry["target-file"]):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified target-file does not exist at the git-path")
# * Validate the cccp yml file
self._cccp_yml_check(git_path, cccp_yml_path, entry)
return self._success, self._status_list
def run(self):
if not self._success:
return False, self._status_list
id_list = []
for entry in self._index_data:
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
self._mark_entry_valid(entry)
# Check if id field exists
if "id" not in entry or ("id" in entry and entry["id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing id")
else:
# Check if id has not already been passed
if entry["id"] in id_list:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("id field must be unique in the file")
else:
id_list.append(entry["id"])
# Checking app-id field
if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing app-id")
else:
if entry["app-id"] != self._file_name.split(".")[0]:
self._summary_collector.add_warning("app-id should be same as first part of the file name")
if "_" in entry["app-id"] or "/" in entry["app-id"] or "." in entry["app-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("app-id cannot contain _, / or . character.")
# Checking job-id field
if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing job-id field")
else:
try:
int(entry["job-id"])
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Job id must be a string")
except ValueError:
pass
if "_" in entry["job-id"] or "/" in entry["job-id"] or "." in entry["job-id"]:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("job-id cannot contain _, / or . character.")
# Check for git-url
if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-url")
# Checking git-path
if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-path")
# Check git-branch
if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing git-branch")
# Check target-file
if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing target-file")
# Check desired-tag
if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing desired-tag")
# Check notify-email
if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing notify-email")
# Check depends-on
if "depends-on" not in entry:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing depends-on")
elif entry["depends-on"]:
depends_on = entry["depends-on"]
if not isinstance(depends_on, list):
depends_on = [depends_on]
matcher = re.compile("^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}"
"[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$")
for item in depends_on:
if not matcher.search(str(item)):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Depends on entry pattern mismatch found {0} must be"
" <string>/<string>:<string>, ".format(str(item)))
return self._success, self._status_list
def run(self):
if not self._success:
return False, self._status_list
container_names = {}
for entry in self._index_data:
self._mark_entry_valid(entry)
self._summary_collector = SummaryCollector(self._context, self._file_name, entry)
clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"])
if clone_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Could not clone specified git-url or could not find specified branch")
continue
# Else clone was success, check the git path
git_path = clone_path + "/" + entry["git-path"]
# Check if specified path exists
if not path.exists(git_path):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified git-path does not exist in git repo.")
continue
# Else, continue with remaining checks
cccp_yml_path = None
# * Check if cccp.yml file exists
for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]:
check_path = git_path + "/" + item
if path.exists(check_path):
cccp_yml_path = check_path
break
if cccp_yml_path is None:
self._mark_entry_invalid(entry)
self._summary_collector.add_error("Missing cccp yml file, please check your git-path")
continue
# * Check for duplicate entry for same container name
container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"])
if container_name in container_names:
self._mark_entry_invalid(entry)
self._summary_collector.add_error(
"Duplicate entry exists at ids : " + str(container_names[container_name]))
else:
container_names[container_name] = []
container_names[container_name].append(entry["id"])
# * Check for existence of target-file
if not path.exists(git_path + "/" + entry["target-file"]):
self._mark_entry_invalid(entry)
self._summary_collector.add_error("The specified target-file does not exist at the git-path")
# * Validate the cccp yml file
self._cccp_yml_check(git_path, cccp_yml_path, entry)
return self._success, self._status_list
