def scan_projects(project_out): log = task.logger config = GlobalConfig(task.conf) paths = PathsConfig(config) projects_path = paths.projects_path() log.info("Scanning projects ...") count = 0 sent_projects = [] for path, project in list_projects(log, projects_path): project_path = os.path.dirname(path) if "id" not in project: log.warn("Discarding project that doesn't have 'id': {0}".format(path)) continue project_id = project["id"] if "name" in project: log.info("--- [{0}: {1}] ---------------------------------".format(project_id, project["name"])) else: log.info("--- [{0}] ---------------------------------".format(project_id)) if "db" not in project: project["db"] = os.path.join(project_path, "project.db.gz") elif not os.path.isabs(project["db"]): project["db"] = os.path.join(project_path, project["db"]) if not os.path.exists(project["db"]): log.error("Project database not found at {0}".format(os.path.relpath(project["db"], project_path))) continue if project["db"].endswith(".gz"): log.info("Uncompressing project database ...") retcode = subprocess.call("gunzip -fc {0} >{1}".format(project["db"], project["db"][:-3]), shell=True) if retcode != 0: log.error("Unexpected error while uncompressing the project database at {0}".format(os.path.relpath(project["db"], projects_path))) continue project["db"] = project["db"][:-3] temp_path = paths.project_temp_path(project_path) if not os.path.exists(temp_path): os.makedirs(temp_path) project["path"] = project_path project["temp_path"] = temp_path sent_projects += [project_id] project_out.send(project) count += 1 log.info("Found {0} projects:\n - {1}".format(count, "\n - ".join(sent_projects)))
def load(self): log = get_logger(__name__) log.info("Loading project data definitions ...") log.debug("Paths:\n{0}".format("\n ".join(self.paths))) log.debug("Includes:\n{0}".format("\n ".join(self.includes))) log.debug("Excludes:\n{0}".format("\n ".join(self.excludes))) # compile regular expressions includes = [re.compile(inc) for inc in self.includes] excludes = [re.compile(exc) for exc in self.excludes] # scan paths project_ids = set() projects = [] for scan_path in self.paths: for path, project in list_projects(log, scan_path): if "id" not in project: log.warn("Discarding project that doesn't have 'id': {0}".format(path)) continue if "files" not in project: log.warn("Discarding project that doesn't have 'files': {0}".format(path)) continue project_id = project["id"] project_name = ": " + project["name"] if "name" in project else "" if match_id(project_id, includes) and not match_id(project_id, excludes): if project_id in project_ids: msg = "Duplicated project id at {0}".format(path) if self.ignore_duplicates: log.warn(msg) continue else: raise Exception(msg) log.info("Included {0}{1}".format(project_id, project_name)) self.__format_project(log, project, base_path=os.path.dirname(path)) projects += [project] else: log.info("Excluded {0}{1}".format(project_id, project_name)) return projects
def execute(self): # Gather scan paths from arguments scan_paths = [] for scan_path in self.args.paths: if not os.path.isabs(scan_path): scan_path = os.path.join(os.getcwd(), scan_path) if not os.path.exists(scan_path): self.log.error("Path not found: {}".format(scan_path)) exit(-1) scan_paths += [scan_path] # Gather includes and excludes from options includes = [] if self.args.include is not None: for inc in self.args.include: includes += ["^{0}$".format(re.escape(inc))] if self.args.include_regex is not None: for inc_regex in self.args.include_regex: includes += [inc_regex] if self.args.include_from is not None: for file in self.args.include_from: with open(file, "r") as f: for line in f: line = line.strip() if line.startswith("#") or len(line) == 0: continue includes += ["^{0}$".format(re.escape(line))] if len(includes) == 0: includes = ["^.*$"] excludes = [] if self.args.exclude is not None: for exc in self.args.exclude: excludes += ["^{0}$".format(re.escape(exc))] if self.args.exclude_regex is not None: for exc_regex in self.args.exclude_regex: excludes += [exc_regex] if self.args.exclude_from is not None: for file in self.args.exclude_from: with open(file, "r") as f: for line in f: line = line.strip() if line.startswith("#") or len(line) == 0: continue excludes += ["^{0}$".format(re.escape(line))] # compile regular expressions includes = [re.compile(inc) for inc in includes] excludes = [re.compile(exc) for exc in excludes] # scan paths self.projects = [] project_ids = set() file_object = {} self.log.info("Looking for data projects ...") for scan_path in scan_paths: for path, project in list_projects(self.log, scan_path): if "id" not in project: self.log.warn("Discarding project missing 'id': {0}".format(path)) continue if "files" not in project: self.log.warn("Discarding project missing 'files': {0}".format(path)) continue project["id"] = normalize_id(project["id"]) project_id = project["id"] if "name" in project: project_name = ": " + project["name"] else: project_name = "" if match_id(project_id, includes) and not match_id(project_id, excludes): if project_id in project_ids: self.log.error("Duplicated project id at {0}".format(path)) exit(-1) self.log.info(" {0}{1} (included)".format(project_id, project_name)) project = init_project_files(project, os.path.dirname(path), file_object) self.projects += [project] else: self.log.info(" {0}{1} (excluded)".format(project_id, project_name)) # Create the wok engine and the workflow instance self.case_conf_builder.add_value("projects", self.projects) self._wok_run(MUTATIONS_FLOW_NAME, container="{}-{}".format(self.user_id, self.workspace))
def multiple_projects(projects_port): log = task.logger conf = task.conf log.info("Scanning multiple projects ...") scan_paths = conf.get("scan_paths") if scan_paths is None or not Data.is_list(scan_paths): raise InternalError("Unexpected scan_paths type: {0}".format(type(scan_paths))) # Prepare includes and excludes includes = conf.get("includes") if includes is not None and not Data.is_list(includes): raise InternalError("Unexpected includes type: {0}".format(type(includes))) if includes is None or len(includes) == 0: includes = ["^.*$"] excludes = conf.get("excludes") if excludes is not None and not Data.is_list(excludes): raise InternalError("Unexpected excludes type: {0}".format(type(excludes))) if excludes is None: excludes = [] log.debug("scan_paths:\n{0}".format("\n ".join(conf["scan_paths"]))) log.debug("Includes:\n{0}".format("\n ".join(includes))) log.debug("Excludes:\n{0}".format("\n ".join(excludes))) # compile regular expressions includes = [re.compile(inc) for inc in includes] excludes = [re.compile(exc) for exc in excludes] # scan paths project_ids = set() for scan_path in scan_paths: for path, project in list_projects(log, scan_path): if "id" not in project: log.warn("Discarding project that doesn't have 'id': {0}".format(path)) continue if "files" not in project: log.warn("Discarding project that doesn't have 'files': {0}".format(path)) continue project_id = project["id"] if "name" in project: project_name = ": " + project["name"] else: project_name = "" if match_id(project_id, includes) and not match_id(project_id, excludes): if project_id in project_ids: raise Exception("Duplicated project id at {0}".format(path)) log.info("Included {0}{1}".format(project_id, project_name)) make_project(log, conf, project, base_path=os.path.dirname(path)) projects_port.send(project) else: log.info("Excluded {0}{1}".format(project_id, project_name))