def run(self): for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \ entry: self._summary_collector.add_error( "Missing git-url, git-path, git-branch or target-file") self._success = False continue clone_location = IndexProjectsValidator.update_git_url( self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if not clone_location: self._summary_collector.add_error( "Unable to clone specified git-url or find specified git-branch" ) self._success = False continue validation_path = clone_location + "/" + str( entry["git-path"]) + "/" + entry["target-file"] if not path.exists(validation_path): self._summary_collector.add_error( "Invalid git-path or target-file specified") self._success = False if self._success: DependencyValidationUpdater(context=self._context, index_file=self._index_file).run() return self._success
class LightWeightValidator(IndexValidator): """Light weight validator does minimum validation, and focuses mostly on building dependency graph""" def __init__(self, context, index_file): IndexValidator.__init__(self, context, index_file) def run(self): for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \ entry: self._summary_collector.add_error("Missing git-url, git-path, git-branch or target-file") self._success = False continue clone_location = IndexProjectsValidator.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if not clone_location: self._summary_collector.add_error("Unable to clone specified git-url or find specified git-branch") self._success = False continue validation_path = clone_location + "/" + str(entry["git-path"]) + "/" + entry["target-file"] if not path.exists(validation_path): self._summary_collector.add_error("Invalid git-path or target-file specified") self._success = False if self._success: DependencyValidationUpdater(context=self._context, index_file=self._index_file).run() return self._success
class LightWeightValidator(IndexValidator): """Light weight validator does minimum validation, and focuses mostly on building dependency graph""" def __init__(self, context, index_file): IndexValidator.__init__(self, context, index_file) def run(self): for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \ entry: self._summary_collector.add_error("Missing git-url, git-path, git-branch or target-file") self._success = False continue clone_location = IndexProjectsValidator.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if not clone_location: self._summary_collector.add_error("Unable to clone specified git-url or find specified git-branch") self._success = False continue validation_path = clone_location + "/" + entry["git-path"] + "/" + entry["target-file"] if not path.exists(validation_path): self._summary_collector.add_error("Invalid git-path or target-file specified") self._success = False if self._success: DependencyValidationUpdater(context=self._context, index_file=self._index_file).run() return self._success
def run(self): for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) if "git-url" not in entry or "git-branch" not in entry or "git-path" not in entry or "target-file" not in \ entry: self._summary_collector.add_error("Missing git-url, git-path, git-branch or target-file") self._success = False continue clone_location = IndexProjectsValidator.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if not clone_location: self._summary_collector.add_error("Unable to clone specified git-url or find specified git-branch") self._success = False continue validation_path = clone_location + "/" + entry["git-path"] + "/" + entry["target-file"] if not path.exists(validation_path): self._summary_collector.add_error("Invalid git-path or target-file specified") self._success = False if self._success: DependencyValidationUpdater(context=self._context, index_file=self._index_file).run() return self._success
def run(self): if not self._success: return False, self._status_list id_list = [] for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) self._mark_entry_valid(entry) # Check if id field exists if "id" not in entry or ("id" in entry and entry["id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing id") else: # Check if id has not already been passed if entry["id"] in id_list: self._mark_entry_invalid(entry) self._summary_collector.add_error("id field must be unique in the file") else: id_list.append(entry["id"]) # Check for pre-build script and pre-build context if constants.PREBUILD_SCRIPT in entry and entry[constants.PREBUILD_SCRIPT] is not None: prebuild_path = entry.get(constants.PREBUILD_CONTEXT) if not prebuild_path: self._mark_entry_invalid(entry) self._summary_collector.add_error("If pre-build script is specified," " then prebuild-context should also " "be specified") # Checking app-id field if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing app-id") else: if entry["app-id"] != self._file_name.split(".")[0]: self._mark_entry_invalid(entry) self._summary_collector.add_error("app-id should be same as first part of the file name") if "_" in entry["app-id"] or "/" in entry["app-id"] or "." in entry["app-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error("app-id cannot contain _, / or . character.") # Checking job-id field if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing job-id field") else: try: int(entry["job-id"]) self._mark_entry_invalid(entry) self._summary_collector.add_error("Job id must be a string") except ValueError: pass if "_" in entry["job-id"] or "/" in entry["job-id"] or "." in entry["job-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error("job-id cannot contain _, / or . character.") # Check for git-url if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-url") else: if "gitlab." in entry["git-url"] and not entry["git-url"].endswith(".git"): self._mark_entry_invalid(entry) self._summary_collector.add_error("Git urls from gitlab must end with .git, try {0}.git".format(entry["git-url"])) # Checking git-path if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-path") # Check git-branch if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-branch") # Check target-file if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing target-file") # Check desired-tag if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing desired-tag") # Check for build-context # Ideally, build-context will be a compulsory field but for now im just checking its None # TODO : Need to update this to make it compulsory if constants.BUILD_CONTEXT not in entry and not (constants.BUILD_CONTEXT in entry and entry[constants.BUILD_CONTEXT] is None): pass # Check notify-email if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing notify-email") # Check depends-on if "depends-on" not in entry: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing depends-on") elif entry["depends-on"]: depends_on = entry["depends-on"] if not isinstance(depends_on, list): depends_on = [depends_on] matcher = re.compile("^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}" "[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$") for item in depends_on: if not matcher.search(str(item)): self._mark_entry_invalid(entry) self._summary_collector.add_error("Depends on entry pattern mismatch found {0} must be" " <string>/<string>:<string>, ".format(str(item))) return self._success, self._status_list
def run(self): if not self._success: return False, self._status_list container_names = {} for entry in self._index_data: self._mark_entry_valid(entry) self._summary_collector = SummaryCollector(self._context, self._file_name, entry) clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if clone_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Could not clone specified git-url or could not find specified branch") continue # Else clone was success, check the git path git_path = clone_path + "/" + str(entry["git-path"]) # * Check for pre-build script # TODO : Make a better implementation of pre-build script checking # TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None prebuild_exists = False if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry: prebuild_exists = True prebuild_script = entry.get(constants.PREBUILD_SCRIPT) prebuild_context = entry.get(constants.PREBUILD_CONTEXT) if (prebuild_script and not path.exists(path.join(git_path, prebuild_script)) and prebuild_context and not path.exists(path.join(git_path, prebuild_context))): self._mark_entry_invalid(entry) self._summary_collector.add_error("Invalid pre-build script or path specified") # * Check for existence of target-file if not prebuild_exists and not path.exists(git_path + "/" + entry["target-file"]): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified target-file does not exist at the git-path") # Check if specified path exists if not prebuild_exists and not path.exists(git_path): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified git-path does not exist in git repo.") continue # Else, continue with remaining checks cccp_yml_path = None # * Check if cccp.yml file exists for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]: check_path = git_path + "/" + item if path.exists(check_path): cccp_yml_path = check_path break if not prebuild_exists and cccp_yml_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing cccp yml file, please check your git-path") continue # * Check for duplicate entry for same container name container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"]) if container_name in container_names: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Duplicate entry exists at ids : " + str(container_names[container_name])) else: container_names[container_name] = [] container_names[container_name].append(entry["id"]) # * Check for build-context if constants.BUILD_CONTEXT in entry: build_context = entry.get(constants.BUILD_CONTEXT) if not prebuild_exists and build_context and not path.exists(path.join(clone_path, git_path, build_context)): self._mark_entry_invalid(entry) self._summary_collector.add_error("Specified build context does not exist.") # * Validate the cccp yml file if not prebuild_exists: self._cccp_yml_check(git_path, cccp_yml_path, entry) return self._success, self._status_list
class IndexProjectsValidator(IndexValidator): """Does deeper analysis of index, checking for correctness of provided values.""" def __init__(self, context, index_file): IndexValidator.__init__(self, context, index_file) @staticmethod def update_git_url(repo_dump, git_url, git_branch): clone_path = None # Work out the path to clone repo to clone_to = git_url if ":" in clone_to: clone_to = clone_to.split(":")[1] clone_to = repo_dump + "/" + clone_to # If the path doesnt already exist, attempt to clone repo if not path.exists(clone_to): cmd = ["git", "clone", git_url, clone_to] if not execute_command(cmd): return None # Update repo get_back = getcwd() chdir(clone_to) cmd = "git branch -r | grep -v '\->' | while read remote; do git branch --track \"${remote#origin/}\"" \ " \"$remote\" &> /dev/null; done" # Get all the branches system(cmd) # fetch the branches cmd = ["git", "fetch", "--all"] execute_command(cmd) # Pull for update cmd = ["git", "pull", "--all"] execute_command(cmd) # Checkout required branch cmd = ["git", "checkout", "origin/" + git_branch] if execute_command(cmd): clone_path = clone_to chdir(get_back) return clone_path def run(self): if not self._success: return False, self._status_list container_names = {} for entry in self._index_data: self._mark_entry_valid(entry) self._summary_collector = SummaryCollector(self._context, self._file_name, entry) clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if clone_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Could not clone specified git-url or could not find specified branch") continue # Else clone was success, check the git path git_path = clone_path + "/" + str(entry["git-path"]) # * Check for pre-build script # TODO : Make a better implementation of pre-build script checking # TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None prebuild_exists = False if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry: prebuild_exists = True prebuild_script = entry.get(constants.PREBUILD_SCRIPT) prebuild_context = entry.get(constants.PREBUILD_CONTEXT) if (prebuild_script and not path.exists(path.join(git_path, prebuild_script)) and prebuild_context and not path.exists(path.join(git_path, prebuild_context))): self._mark_entry_invalid(entry) self._summary_collector.add_error("Invalid pre-build script or path specified") # * Check for existence of target-file if not prebuild_exists and not path.exists(git_path + "/" + entry["target-file"]): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified target-file does not exist at the git-path") # Check if specified path exists if not prebuild_exists and not path.exists(git_path): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified git-path does not exist in git repo.") continue # Else, continue with remaining checks cccp_yml_path = None # * Check if cccp.yml file exists for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]: check_path = git_path + "/" + item if path.exists(check_path): cccp_yml_path = check_path break if not prebuild_exists and cccp_yml_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing cccp yml file, please check your git-path") continue # * Check for duplicate entry for same container name container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"]) if container_name in container_names: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Duplicate entry exists at ids : " + str(container_names[container_name])) else: container_names[container_name] = [] container_names[container_name].append(entry["id"]) # * Check for build-context if constants.BUILD_CONTEXT in entry: build_context = entry.get(constants.BUILD_CONTEXT) if not prebuild_exists and build_context and not path.exists(path.join(clone_path, git_path, build_context)): self._mark_entry_invalid(entry) self._summary_collector.add_error("Specified build context does not exist.") # * Validate the cccp yml file if not prebuild_exists: self._cccp_yml_check(git_path, cccp_yml_path, entry) return self._success, self._status_list def _cccp_yml_check(self, git_path, cccp_yaml_path, entry): """Validates the cccp yaml file""" temp_cccp, ex = self._load_yml(cccp_yaml_path) if not temp_cccp: self._mark_entry_invalid(entry) self._summary_collector.add_error("Malformed cccp yml : " + str(ex)) cccp_yaml = temp_cccp self._entry_valid = True get_back = getcwd() chdir(git_path) # * Check for job-entry_id if "job-id" not in cccp_yaml: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing job-id field in cccp yaml") # * Check for test-skip if "test-skip" in cccp_yaml: value = cccp_yaml["test-skip"] try: if value and (value is True or value is False): pass if value is None: self._summary_collector.add_warning("Optional test-skip is set None, which means its value will be" " ignored") except Exception as ex: self._mark_entry_invalid(entry) self._summary_collector.add_error("test-skip should either be True or False as it is a flag") # * Check test-script if "test-script" in cccp_yaml: self._summary_collector.add_warning("Custom test-script has been specified") value = cccp_yaml["test-script"] if value and not path.exists(str(value)): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified test-script does not exist") if value is None: self._summary_collector.add_warning("Optional test-script has a value of None, which means it will be" " ignored") # * Check build-script if "build-script" in cccp_yaml: self._summary_collector.add_warning("Custom build-script has been specified") value = cccp_yaml["build-script"] if value and not path.exists(str(value)): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified build-script does not exist") if value is None: self._summary_collector.add_warning("Optional build-script has has a value None, which means it will be" " ignored") # * Check delivery-script if "delivery-script" in cccp_yaml: self._summary_collector.add_warning("Custom delivery-script has been specified.") value = cccp_yaml["delivery-script"] if value and not path.exists(str(value)): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified delivery-script does not exist") if value is None: self._summary_collector.add_warning("Optional delivery script has value None, which means it will be" " ignored") chdir(get_back)
def run(self): if not self._success: return False, self._status_list id_list = [] for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) self._mark_entry_valid(entry) # Check if id field exists if "id" not in entry or ("id" in entry and entry["id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing id") else: # Check if id has not already been passed if entry["id"] in id_list: self._mark_entry_invalid(entry) self._summary_collector.add_error( "id field must be unique in the file") else: id_list.append(entry["id"]) # Check for pre-build script and pre-build context if constants.PREBUILD_SCRIPT in entry and entry[ constants.PREBUILD_SCRIPT] is not None: prebuild_path = entry.get(constants.PREBUILD_CONTEXT) if not prebuild_path: self._mark_entry_invalid(entry) self._summary_collector.add_error( "If pre-build script is specified," " then prebuild-context should also " "be specified") # Checking app-id field if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing app-id") else: if entry["app-id"] != self._file_name.split(".")[0]: self._mark_entry_invalid(entry) self._summary_collector.add_error( "app-id should be same as first part of the file name") if "_" in entry["app-id"] or "/" in entry[ "app-id"] or "." in entry["app-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error( "app-id cannot contain _, / or . character.") # Checking job-id field if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing job-id field") else: try: int(entry["job-id"]) self._mark_entry_invalid(entry) self._summary_collector.add_error( "Job id must be a string") except ValueError: pass if "_" in entry["job-id"] or "/" in entry[ "job-id"] or "." in entry["job-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error( "job-id cannot contain _, / or . character.") # Check for git-url if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-url") else: if "gitlab." in entry[ "git-url"] and not entry["git-url"].endswith(".git"): self._mark_entry_invalid(entry) self._summary_collector.add_error( "Git urls from gitlab must end with .git, try {0}.git". format(entry["git-url"])) # Checking git-path if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-path") # Check git-branch if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-branch") # Check target-file if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing target-file") # Check desired-tag if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing desired-tag") # Check for build-context # Ideally, build-context will be a compulsory field but for now im just checking its None # TODO : Need to update this to make it compulsory if constants.BUILD_CONTEXT not in entry and not ( constants.BUILD_CONTEXT in entry and entry[constants.BUILD_CONTEXT] is None): pass # Check notify-email if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing notify-email") # Check depends-on if "depends-on" not in entry: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing depends-on") elif entry["depends-on"]: depends_on = entry["depends-on"] if not isinstance(depends_on, list): depends_on = [depends_on] matcher = re.compile( "^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}" "[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$") for item in depends_on: if not matcher.search(str(item)): self._mark_entry_invalid(entry) self._summary_collector.add_error( "Depends on entry pattern mismatch found {0} must be" " <string>/<string>:<string>, ".format(str(item))) return self._success, self._status_list
def run(self): if not self._success: return False, self._status_list container_names = {} for entry in self._index_data: self._mark_entry_valid(entry) self._summary_collector = SummaryCollector(self._context, self._file_name, entry) clone_path = self.update_git_url( self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if clone_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Could not clone specified git-url or could not find specified branch" ) continue # Else clone was success, check the git path git_path = clone_path + "/" + str(entry["git-path"]) # * Check for pre-build script # TODO : Make a better implementation of pre-build script checking # TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None prebuild_exists = False if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry: prebuild_exists = True prebuild_script = entry.get(constants.PREBUILD_SCRIPT) prebuild_context = entry.get(constants.PREBUILD_CONTEXT) if (prebuild_script and not path.exists(path.join(git_path, prebuild_script)) and prebuild_context and not path.exists( path.join(git_path, prebuild_context))): self._mark_entry_invalid(entry) self._summary_collector.add_error( "Invalid pre-build script or path specified") # * Check for existence of target-file if not prebuild_exists and not path.exists(git_path + "/" + entry["target-file"]): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified target-file does not exist at the git-path") # Check if specified path exists if not prebuild_exists and not path.exists(git_path): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified git-path does not exist in git repo.") continue # Else, continue with remaining checks cccp_yml_path = None # * Check if cccp.yml file exists for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]: check_path = git_path + "/" + item if path.exists(check_path): cccp_yml_path = check_path break if not prebuild_exists and cccp_yml_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Missing cccp yml file, please check your git-path") continue # * Check for duplicate entry for same container name container_name = entry["app-id"] + "/" + entry[ "job-id"] + ":" + str(entry["desired-tag"]) if container_name in container_names: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Duplicate entry exists at ids : " + str(container_names[container_name])) else: container_names[container_name] = [] container_names[container_name].append(entry["id"]) # * Check for build-context if constants.BUILD_CONTEXT in entry: build_context = entry.get(constants.BUILD_CONTEXT) if not prebuild_exists and build_context and not path.exists( path.join(clone_path, git_path, build_context)): self._mark_entry_invalid(entry) self._summary_collector.add_error( "Specified build context does not exist.") # * Validate the cccp yml file if not prebuild_exists: self._cccp_yml_check(git_path, cccp_yml_path, entry) return self._success, self._status_list
class IndexProjectsValidator(IndexValidator): """Does deeper analysis of index, checking for correctness of provided values.""" def __init__(self, context, index_file): IndexValidator.__init__(self, context, index_file) @staticmethod def update_git_url(repo_dump, git_url, git_branch): clone_path = None # Work out the path to clone repo to clone_to = git_url if ":" in clone_to: clone_to = clone_to.split(":")[1] clone_to = repo_dump + "/" + clone_to # If the path doesnt already exist, attempt to clone repo if not path.exists(clone_to): cmd = ["git", "clone", git_url, clone_to] if not execute_command(cmd): return None # Update repo get_back = getcwd() chdir(clone_to) cmd = "git branch -r | grep -v '\->' | while read remote; do git branch --track \"${remote#origin/}\"" \ " \"$remote\" &> /dev/null; done" # Get all the branches system(cmd) # fetch the branches cmd = ["git", "fetch", "--all"] execute_command(cmd) # Pull for update cmd = ["git", "pull", "--all"] execute_command(cmd) # Checkout required branch cmd = ["git", "checkout", "origin/" + git_branch] if execute_command(cmd): clone_path = clone_to chdir(get_back) return clone_path def run(self): if not self._success: return False, self._status_list container_names = {} for entry in self._index_data: self._mark_entry_valid(entry) self._summary_collector = SummaryCollector(self._context, self._file_name, entry) clone_path = self.update_git_url( self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if clone_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Could not clone specified git-url or could not find specified branch" ) continue # Else clone was success, check the git path git_path = clone_path + "/" + str(entry["git-path"]) # * Check for pre-build script # TODO : Make a better implementation of pre-build script checking # TODO : Ideally, if prebuild is not in entry, it wont reach here and this should happen if it is not None prebuild_exists = False if constants.PREBUILD_SCRIPT in entry and constants.PREBUILD_CONTEXT in entry: prebuild_exists = True prebuild_script = entry.get(constants.PREBUILD_SCRIPT) prebuild_context = entry.get(constants.PREBUILD_CONTEXT) if (prebuild_script and not path.exists(path.join(git_path, prebuild_script)) and prebuild_context and not path.exists( path.join(git_path, prebuild_context))): self._mark_entry_invalid(entry) self._summary_collector.add_error( "Invalid pre-build script or path specified") # * Check for existence of target-file if not prebuild_exists and not path.exists(git_path + "/" + entry["target-file"]): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified target-file does not exist at the git-path") # Check if specified path exists if not prebuild_exists and not path.exists(git_path): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified git-path does not exist in git repo.") continue # Else, continue with remaining checks cccp_yml_path = None # * Check if cccp.yml file exists for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]: check_path = git_path + "/" + item if path.exists(check_path): cccp_yml_path = check_path break if not prebuild_exists and cccp_yml_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Missing cccp yml file, please check your git-path") continue # * Check for duplicate entry for same container name container_name = entry["app-id"] + "/" + entry[ "job-id"] + ":" + str(entry["desired-tag"]) if container_name in container_names: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Duplicate entry exists at ids : " + str(container_names[container_name])) else: container_names[container_name] = [] container_names[container_name].append(entry["id"]) # * Check for build-context if constants.BUILD_CONTEXT in entry: build_context = entry.get(constants.BUILD_CONTEXT) if not prebuild_exists and build_context and not path.exists( path.join(clone_path, git_path, build_context)): self._mark_entry_invalid(entry) self._summary_collector.add_error( "Specified build context does not exist.") # * Validate the cccp yml file if not prebuild_exists: self._cccp_yml_check(git_path, cccp_yml_path, entry) return self._success, self._status_list def _cccp_yml_check(self, git_path, cccp_yaml_path, entry): """Validates the cccp yaml file""" temp_cccp, ex = self._load_yml(cccp_yaml_path) if not temp_cccp: self._mark_entry_invalid(entry) self._summary_collector.add_error("Malformed cccp yml : " + str(ex)) cccp_yaml = temp_cccp self._entry_valid = True get_back = getcwd() chdir(git_path) # * Check for job-entry_id if "job-id" not in cccp_yaml: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Missing job-id field in cccp yaml") # * Check for test-skip if "test-skip" in cccp_yaml: value = cccp_yaml["test-skip"] try: if value and (value is True or value is False): pass if value is None: self._summary_collector.add_warning( "Optional test-skip is set None, which means its value will be" " ignored") except Exception as ex: self._mark_entry_invalid(entry) self._summary_collector.add_error( "test-skip should either be True or False as it is a flag") # * Check test-script if "test-script" in cccp_yaml: self._summary_collector.add_warning( "Custom test-script has been specified") value = cccp_yaml["test-script"] if value and not path.exists(str(value)): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified test-script does not exist") if value is None: self._summary_collector.add_warning( "Optional test-script has a value of None, which means it will be" " ignored") # * Check build-script if "build-script" in cccp_yaml: self._summary_collector.add_warning( "Custom build-script has been specified") value = cccp_yaml["build-script"] if value and not path.exists(str(value)): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified build-script does not exist") if value is None: self._summary_collector.add_warning( "Optional build-script has has a value None, which means it will be" " ignored") # * Check delivery-script if "delivery-script" in cccp_yaml: self._summary_collector.add_warning( "Custom delivery-script has been specified.") value = cccp_yaml["delivery-script"] if value and not path.exists(str(value)): self._mark_entry_invalid(entry) self._summary_collector.add_error( "The specified delivery-script does not exist") if value is None: self._summary_collector.add_warning( "Optional delivery script has value None, which means it will be" " ignored") chdir(get_back)
class IndexFormatValidator(IndexValidator): """Checks the format of index files""" def __init__(self, context, index_file): IndexValidator.__init__(self, context, index_file) def run(self): if not self._success: return False, self._status_list id_list = [] for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) self._mark_entry_valid(entry) # Check if id field exists if "id" not in entry or ("id" in entry and entry["id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing id") else: # Check if id has not already been passed if entry["id"] in id_list: self._mark_entry_invalid(entry) self._summary_collector.add_error("id field must be unique in the file") else: id_list.append(entry["id"]) # Checking app-id field if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing app-id") else: if entry["app-id"] != self._file_name.split(".")[0]: self._mark_entry_invalid(entry) self._summary_collector.add_error("app-id should be same as first part of the file name") if "_" in entry["app-id"] or "/" in entry["app-id"] or "." in entry["app-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error("app-id cannot contain _, / or . character.") # Checking job-id field if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing job-id field") else: try: int(entry["job-id"]) self._mark_entry_invalid(entry) self._summary_collector.add_error("Job id must be a string") except ValueError: pass if "_" in entry["job-id"] or "/" in entry["job-id"] or "." in entry["job-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error("job-id cannot contain _, / or . character.") # Check for git-url if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-url") else: if "gitlab." in entry["git-url"] and not entry["git-url"].endswith(".git"): self._mark_entry_invalid(entry) self._summary_collector.add_error("Git urls from gitlab must end with .git, try {0}.git".format(entry["git-url"])) # Checking git-path if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-path") # Check git-branch if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-branch") # Check target-file if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing target-file") # Check desired-tag if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing desired-tag") # Check notify-email if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing notify-email") # Check depends-on if "depends-on" not in entry: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing depends-on") elif entry["depends-on"]: depends_on = entry["depends-on"] if not isinstance(depends_on, list): depends_on = [depends_on] matcher = re.compile("^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}" "[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$") for item in depends_on: if not matcher.search(str(item)): self._mark_entry_invalid(entry) self._summary_collector.add_error("Depends on entry pattern mismatch found {0} must be" " <string>/<string>:<string>, ".format(str(item))) return self._success, self._status_list
def run(self): if not self._success: return False, self._status_list container_names = {} for entry in self._index_data: self._mark_entry_valid(entry) self._summary_collector = SummaryCollector(self._context, self._file_name, entry) clone_path = self.update_git_url(self._context.environment.repo_dump, entry["git-url"], entry["git-branch"]) if clone_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Could not clone specified git-url or could not find specified branch") continue # Else clone was success, check the git path git_path = clone_path + "/" + entry["git-path"] # Check if specified path exists if not path.exists(git_path): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified git-path does not exist in git repo.") continue # Else, continue with remaining checks cccp_yml_path = None # * Check if cccp.yml file exists for item in ["cccp.yml", ".cccp.yml", "cccp.yaml", ".cccp.yaml"]: check_path = git_path + "/" + item if path.exists(check_path): cccp_yml_path = check_path break if cccp_yml_path is None: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing cccp yml file, please check your git-path") continue # * Check for duplicate entry for same container name container_name = entry["app-id"] + "/" + entry["job-id"] + ":" + str(entry["desired-tag"]) if container_name in container_names: self._mark_entry_invalid(entry) self._summary_collector.add_error( "Duplicate entry exists at ids : " + str(container_names[container_name])) else: container_names[container_name] = [] container_names[container_name].append(entry["id"]) # * Check for existence of target-file if not path.exists(git_path + "/" + entry["target-file"]): self._mark_entry_invalid(entry) self._summary_collector.add_error("The specified target-file does not exist at the git-path") # * Validate the cccp yml file self._cccp_yml_check(git_path, cccp_yml_path, entry) return self._success, self._status_list
def run(self): if not self._success: return False, self._status_list id_list = [] for entry in self._index_data: self._summary_collector = SummaryCollector(self._context, self._file_name, entry) self._mark_entry_valid(entry) # Check if id field exists if "id" not in entry or ("id" in entry and entry["id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing id") else: # Check if id has not already been passed if entry["id"] in id_list: self._mark_entry_invalid(entry) self._summary_collector.add_error("id field must be unique in the file") else: id_list.append(entry["id"]) # Checking app-id field if "app-id" not in entry or ("app-id" in entry and entry["app-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing app-id") else: if entry["app-id"] != self._file_name.split(".")[0]: self._summary_collector.add_warning("app-id should be same as first part of the file name") if "_" in entry["app-id"] or "/" in entry["app-id"] or "." in entry["app-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error("app-id cannot contain _, / or . character.") # Checking job-id field if "job-id" not in entry or ("job-id" in entry and entry["job-id"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing job-id field") else: try: int(entry["job-id"]) self._mark_entry_invalid(entry) self._summary_collector.add_error("Job id must be a string") except ValueError: pass if "_" in entry["job-id"] or "/" in entry["job-id"] or "." in entry["job-id"]: self._mark_entry_invalid(entry) self._summary_collector.add_error("job-id cannot contain _, / or . character.") # Check for git-url if "git-url" not in entry or ("git-url" in entry and entry["git-url"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-url") # Checking git-path if "git-path" not in entry or ("git-path" in entry and entry["git-path"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-path") # Check git-branch if "git-branch" not in entry or ("git-branch" in entry and entry["git-branch"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing git-branch") # Check target-file if "target-file" not in entry or ("target-file" in entry and entry["target-file"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing target-file") # Check desired-tag if "desired-tag" not in entry or ("desired-tag" in entry and entry["desired-tag"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing desired-tag") # Check notify-email if "notify-email" not in entry or ("notify-email" in entry and entry["notify-email"] is None): self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing notify-email") # Check depends-on if "depends-on" not in entry: self._mark_entry_invalid(entry) self._summary_collector.add_error("Missing depends-on") elif entry["depends-on"]: depends_on = entry["depends-on"] if not isinstance(depends_on, list): depends_on = [depends_on] matcher = re.compile("^(([0-9a-zA-Z_-]+[.]{1})*([0-9a-zA-Z_-]+){1}[/]{1})?[0-9a-zA-Z_-]+[/]{1}" "[0-9a-zA-Z_-]+[:]{1}([0-9a-zA-Z_-]+\.?)+$") for item in depends_on: if not matcher.search(str(item)): self._mark_entry_invalid(entry) self._summary_collector.add_error("Depends on entry pattern mismatch found {0} must be" " <string>/<string>:<string>, ".format(str(item))) return self._success, self._status_list