Example #1
0
def scan_projects(project_out):
	log = task.logger

	config = GlobalConfig(task.conf)
	paths = PathsConfig(config)

	projects_path = paths.projects_path()

	log.info("Scanning projects ...")

	count = 0

	sent_projects = []

	for path, project in list_projects(log, projects_path):
		project_path = os.path.dirname(path)

		if "id" not in project:
			log.warn("Discarding project that doesn't have 'id': {0}".format(path))
			continue

		project_id = project["id"]

		if "name" in project:
			log.info("--- [{0}: {1}] ---------------------------------".format(project_id, project["name"]))
		else:
			log.info("--- [{0}] ---------------------------------".format(project_id))

		if "db" not in project:
			project["db"] = os.path.join(project_path, "project.db.gz")
		elif not os.path.isabs(project["db"]):
			project["db"] = os.path.join(project_path, project["db"])

		if not os.path.exists(project["db"]):
			log.error("Project database not found at {0}".format(os.path.relpath(project["db"], project_path)))
			continue

		if project["db"].endswith(".gz"):
			log.info("Uncompressing project database ...")
			retcode = subprocess.call("gunzip -fc {0} >{1}".format(project["db"], project["db"][:-3]), shell=True)
			if retcode != 0:
				log.error("Unexpected error while uncompressing the project database at {0}".format(os.path.relpath(project["db"], projects_path)))
				continue

			project["db"] = project["db"][:-3]

		temp_path = paths.project_temp_path(project_path)
		if not os.path.exists(temp_path):
			os.makedirs(temp_path)

		project["path"] = project_path
		project["temp_path"] = temp_path

		sent_projects += [project_id]
		project_out.send(project)

		count += 1

	log.info("Found {0} projects:\n  - {1}".format(count, "\n  - ".join(sent_projects)))
Example #2
0
	def load(self):
		log = get_logger(__name__)

		log.info("Loading project data definitions ...")
		log.debug("Paths:\n{0}".format("\n  ".join(self.paths)))
		log.debug("Includes:\n{0}".format("\n  ".join(self.includes)))
		log.debug("Excludes:\n{0}".format("\n  ".join(self.excludes)))

		# compile regular expressions

		includes = [re.compile(inc) for inc in self.includes]
		excludes = [re.compile(exc) for exc in self.excludes]

		# scan paths

		project_ids = set()
		projects = []

		for scan_path in self.paths:
			for path, project in list_projects(log, scan_path):
				if "id" not in project:
					log.warn("Discarding project that doesn't have 'id': {0}".format(path))
					continue
				if "files" not in project:
					log.warn("Discarding project that doesn't have 'files': {0}".format(path))
					continue

				project_id = project["id"]

				project_name = ": " + project["name"] if "name" in project else ""

				if match_id(project_id, includes) and not match_id(project_id, excludes):
					if project_id in project_ids:
						msg = "Duplicated project id at {0}".format(path)
						if self.ignore_duplicates:
							log.warn(msg)
							continue
						else:
							raise Exception(msg)

					log.info("Included {0}{1}".format(project_id, project_name))

					self.__format_project(log, project, base_path=os.path.dirname(path))

					projects += [project]
				else:
					log.info("Excluded {0}{1}".format(project_id, project_name))

		return projects
Example #3
0
	def execute(self):

		# Gather scan paths from arguments

		scan_paths = []

		for scan_path in self.args.paths:
			if not os.path.isabs(scan_path):
				scan_path = os.path.join(os.getcwd(), scan_path)

			if not os.path.exists(scan_path):
				self.log.error("Path not found: {}".format(scan_path))
				exit(-1)

			scan_paths += [scan_path]

		# Gather includes and excludes from options

		includes = []

		if self.args.include is not None:
			for inc in self.args.include:
				includes += ["^{0}$".format(re.escape(inc))]
		if self.args.include_regex is not None:
			for inc_regex in self.args.include_regex:
				includes += [inc_regex]
		if self.args.include_from is not None:
			for file in self.args.include_from:
				with open(file, "r") as f:
					for line in f:
						line = line.strip()
						if line.startswith("#") or len(line) == 0:
							continue
						includes += ["^{0}$".format(re.escape(line))]

		if len(includes) == 0:
			includes = ["^.*$"]

		excludes = []

		if self.args.exclude is not None:
			for exc in self.args.exclude:
				excludes += ["^{0}$".format(re.escape(exc))]
		if self.args.exclude_regex is not None:
			for exc_regex in self.args.exclude_regex:
				excludes += [exc_regex]
		if self.args.exclude_from is not None:
			for file in self.args.exclude_from:
				with open(file, "r") as f:
					for line in f:
						line = line.strip()
						if line.startswith("#") or len(line) == 0:
							continue
						excludes += ["^{0}$".format(re.escape(line))]

		# compile regular expressions

		includes = [re.compile(inc) for inc in includes]
		excludes = [re.compile(exc) for exc in excludes]

		# scan paths

		self.projects = []
		project_ids = set()
		file_object = {}

		self.log.info("Looking for data projects ...")

		for scan_path in scan_paths:
			for path, project in list_projects(self.log, scan_path):
				if "id" not in project:
					self.log.warn("Discarding project missing 'id': {0}".format(path))
					continue
				if "files" not in project:
					self.log.warn("Discarding project missing 'files': {0}".format(path))
					continue

				project["id"] = normalize_id(project["id"])
				project_id = project["id"]
				if "name" in project:
					project_name = ": " + project["name"]
				else:
					project_name = ""

				if match_id(project_id, includes) and not match_id(project_id, excludes):
					if project_id in project_ids:
						self.log.error("Duplicated project id at {0}".format(path))
						exit(-1)

					self.log.info("  {0}{1} (included)".format(project_id, project_name))

					project = init_project_files(project, os.path.dirname(path), file_object)
					self.projects += [project]
				else:
					self.log.info("  {0}{1} (excluded)".format(project_id, project_name))

		# Create the wok engine and the workflow instance
		
		self.case_conf_builder.add_value("projects", self.projects)

		self._wok_run(MUTATIONS_FLOW_NAME, container="{}-{}".format(self.user_id, self.workspace))
Example #4
0
def multiple_projects(projects_port):
	log = task.logger
	conf = task.conf

	log.info("Scanning multiple projects ...")

	scan_paths = conf.get("scan_paths")
	if scan_paths is None or not Data.is_list(scan_paths):
		raise InternalError("Unexpected scan_paths type: {0}".format(type(scan_paths)))

	# Prepare includes and excludes
	
	includes = conf.get("includes")
	if includes is not None and not Data.is_list(includes):
		raise InternalError("Unexpected includes type: {0}".format(type(includes)))
	if includes is None or len(includes) == 0:
		includes = ["^.*$"]

	excludes = conf.get("excludes")
	if excludes is not None and not Data.is_list(excludes):
		raise InternalError("Unexpected excludes type: {0}".format(type(excludes)))
	if excludes is None:
		excludes = []

	log.debug("scan_paths:\n{0}".format("\n  ".join(conf["scan_paths"])))
	log.debug("Includes:\n{0}".format("\n  ".join(includes)))
	log.debug("Excludes:\n{0}".format("\n  ".join(excludes)))
	
	# compile regular expressions

	includes = [re.compile(inc) for inc in includes]
	excludes = [re.compile(exc) for exc in excludes]

	# scan paths

	project_ids = set()

	for scan_path in scan_paths:
		for path, project in list_projects(log, scan_path):
			if "id" not in project:
				log.warn("Discarding project that doesn't have 'id': {0}".format(path))
				continue
			if "files" not in project:
				log.warn("Discarding project that doesn't have 'files': {0}".format(path))
				continue

			project_id = project["id"]
			if "name" in project:
				project_name = ": " + project["name"]
			else:
				project_name = ""

			if match_id(project_id, includes) and not match_id(project_id, excludes):
				if project_id in project_ids:
					raise Exception("Duplicated project id at {0}".format(path))

				log.info("Included {0}{1}".format(project_id, project_name))

				make_project(log, conf, project, base_path=os.path.dirname(path))

				projects_port.send(project)
			else:
				log.info("Excluded {0}{1}".format(project_id, project_name))