def fetch(self, category=CATEGORY_CODEP_PYREVERSE, paths=None, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME, branches=None, latest_items=False): """Fetch commits and code (package and class) dependencies information.""" if not self.entrypoint and category == CATEGORY_CODEP_PYREVERSE: raise GraalError(cause="Entrypoint cannot be null") if not self.exec_path and category == CATEGORY_CODEP_JADOLINT: raise GraalError(cause="Exec path cannot be null") if category == CATEGORY_CODEP_PYREVERSE: self.analyzer_kind = PYREVERSE self.analyzer = PyreverseAnalyzer() elif category == CATEGORY_CODEP_JADOLINT: self.analyzer_kind = JADOLINT self.analyzer = JadolintAnalyzer(self.exec_path, analysis=DEPENDENCIES) else: raise GraalError(cause="Unknown category %s" % category) items = super().fetch(category, from_date=from_date, to_date=to_date, branches=branches, latest_items=latest_items) return items
def analyze(self, **kwargs): """Add information using CLOC :param file_path: file path :param repository_level: set to True if analysis has to be performed on a repository :returns result: dict of the results of the analysis """ file_path = kwargs['file_path'] repository_level = kwargs.get('repository_level', False) try: message = subprocess.check_output(['cloc', file_path]).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Cloc failed at %s, %s" % (file_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() if repository_level: results = self.__analyze_repository(message) else: results = self.__analyze_file(message) results['ext'] = GraalRepository.extension(file_path) return results
def fetch(self, category=CATEGORY_COQUA_PYLINT, paths=None, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME, branches=None, latest_items=False): """Fetch commits and add code quality information.""" if category == CATEGORY_COQUA_PYLINT: self.analyzer_kind = PYLINT elif category == CATEGORY_COQUA_FLAKE8: self.analyzer_kind = FLAKE8 else: raise GraalError(cause="Unknown category %s" % category) self.module_analyzer = ModuleAnalyzer(self.details, self.analyzer_kind) items = super().fetch(category, from_date=from_date, to_date=to_date, branches=branches, latest_items=latest_items) return items
def __init__(self, uri, git_path, worktreepath=DEFAULT_WORKTREE_PATH, entrypoint=None, in_paths=None, out_paths=None, details=False, tag=None, archive=None): super().__init__(uri, git_path, worktreepath, entrypoint=entrypoint, in_paths=in_paths, out_paths=out_paths, details=details, tag=tag, archive=archive) if not self.entrypoint: raise GraalError(cause="Entrypoint cannot be null") self.analyzer_kind = None self.analyzer = None
def fetch(self, category=CATEGORY_COLIC_NOMOS, paths=None, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME, branches=None, latest_items=False): """Fetch commits and add license information.""" if category == CATEGORY_COLIC_SCANCODE: self.analyzer_kind = SCANCODE elif category == CATEGORY_COLIC_NOMOS: self.analyzer_kind = NOMOS else: raise GraalError(cause="Unknown category %s" % category) self.analyzer = LicenseAnalyzer(self.exec_path, self.analyzer_kind) items = super().fetch(category, from_date=from_date, to_date=to_date, branches=branches, latest_items=latest_items) return items
def analyze(self, **kwargs): """Add information about license :param file_path: file path :returns result: dict of the results of the analysis """ result = {'licenses': []} file_path = kwargs['file_path'] try: msg = subprocess.check_output( [self.exec_path, '--json-pp', '-', '--license', file_path]).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Scancode failed at %s, %s" % (file_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() licenses_raw = json.loads(msg) if 'files' not in licenses_raw: return result result['licenses'] = licenses_raw['files'][0]['licenses'] return result
def __analyze_scancode(self, file_path): """Add information about license and copyright using scancode :param file_path: file path (in case of scancode) """ result = { 'licenses': [], 'copyrights': [], } try: msg = subprocess.check_output([ self.exec_path, '--json-pp', '-', '--license', '--copyright', file_path ]).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Scancode failed at %s, %s" % (file_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() scancode_raw = json.loads(msg) if 'files' in scancode_raw: result['licenses'] = scancode_raw['files'][0]['licenses'] result['copyrights'] = scancode_raw['files'][0]['copyrights'] return result
def analyze(self, **kwargs): """Add information about license :param file_path: file path :returns result: dict of the results of the analysis """ result = {'licenses': []} file_path = kwargs['file_path'] try: msg = subprocess.check_output([self.exec_path, file_path]).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Nomos failed at %s, %s" % (file_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() licenses_raw = re.findall(self.search_pattern, msg) licenses = [] for license_raw in licenses_raw: license_digested = license_raw.split("license(s)")[1].strip() licenses.append(license_digested) if licenses: result['licenses'] = licenses return result
def fetch(self, category=CATEGORY_COCOM_LIZARD_FILE, paths=None, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME, branches=None, latest_items=False): """Fetch commits and add code complexity information.""" items = super().fetch(category, from_date=from_date, to_date=to_date, branches=branches, latest_items=latest_items) if category == CATEGORY_COCOM_LIZARD_FILE: self.analyzer_kind = LIZARD_FILE elif category == CATEGORY_COCOM_LIZARD_REPOSITORY: self.analyzer_kind = LIZARD_REPOSITORY elif category == CATEGORY_COCOM_SCC_FILE: self.analyzer_kind = SCC_FILE elif category == CATEGORY_COCOM_SCC_REPOSITORY: self.analyzer_kind = SCC_REPOSITORY else: raise GraalError(cause="Unknown category %s" % category) if "_file" in category: self.analyzer = FileAnalyzer(self.details, self.analyzer_kind) else: self.analyzer = RepositoryAnalyzer(self.details, self.analyzer_kind) return items
def analyze(self, **kwargs): """Get Jadolint results for a Dockerfile. :param file_path: file path :param result: dict of the results of the analysis """ results = [] result = {self.analysis: results} file_path = kwargs['file_path'] if self.analysis == DEPENDENCIES: cmd = ['java', '-jar', self.exec_path, file_path, '--deps'] else: cmd = ['java', '-jar', self.exec_path, file_path, '--smells'] try: msg = subprocess.check_output(cmd).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Jadolint failed at %s, %s" % (file_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() results_raw = msg.split('\n') for res_raw in results_raw: res = res_raw.strip() if res: results.append(res) return result
def analyze(self, **kwargs): """Get a UML class diagrams from a Python project. :param module_path: module path :param result: dict of the results of the analysis """ result = {} module_path = kwargs['module_path'] try: subprocess.check_output(['pyreverse', module_path]).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Pyreverse failed at %s, %s" % (module_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() class_diagram = os.path.join(self.tmp_path, CLASSES_FILE_NAME) if os.path.exists(class_diagram): graph_classes = self.__dotfile2json(class_diagram) result['classes'] = graph_classes package_diagram = os.path.join(self.tmp_path, PACKAGES_FILE_NAME) if os.path.exists(package_diagram): graph_packages = self.__dotfile2json(package_diagram) result['packages'] = graph_packages return result
def __init__(self, uri, git_path, exec_path, worktreepath=DEFAULT_WORKTREE_PATH, entrypoint=None, in_paths=None, out_paths=None, tag=None, archive=None): super().__init__(uri, git_path, worktreepath, entrypoint=entrypoint, in_paths=in_paths, out_paths=out_paths, tag=tag, archive=archive) if not GraalRepository.exists(exec_path): raise GraalError(cause="executable path %s not valid" % exec_path) self.exec_path = exec_path self.analyzer_kind = None self.analyzer = None
def analyze(self, **kwargs): """Add quality checks data using Pylint. :param module_path: module path :param details: if True, it returns information about single modules :returns result: dict of the results of the analysis """ module_path = kwargs['module_path'] details = kwargs['details'] try: msg = subprocess.check_output( ['pylint', '-rn', '--output-format=text', module_path]).decode("utf-8") except subprocess.CalledProcessError as e: msg = e.output.decode("utf-8") if not msg.startswith("***"): raise GraalError(cause="Pylint failed at %s, %s" % (module_path, msg)) finally: subprocess._cleanup() end = False code_quality = None mod_details = [] module_name = "" lines = msg.split('\n') modules = {} for line in lines: if line.startswith("***"): if mod_details: modules.update({module_name: mod_details}) module_name = line.strip("*").strip().replace("Module ", "") mod_details = [] elif line.strip() == "": continue elif line.startswith("----"): modules.update({module_name: mod_details}) end = True else: if end: code_quality = line.split("/")[0].split(" ")[-1] break else: mod_details.append(line) result = { 'quality': code_quality, 'num_modules': len(modules), 'warnings': sum([len(mod) for mod in modules]) } if details: result['modules'] = modules return result
def __init__(self, exec_path, cli=False): if not GraalRepository.exists(exec_path): raise GraalError(cause="executable path %s not valid" % exec_path) self.exec_path = exec_path self.cli = cli if self.cli: exec_path = self.exec_path.replace(SCANCODE_CLI_EXEC, CONFIGURE_EXEC) _ = subprocess.check_output([exec_path]).decode("utf-8")
def metadata_category(item): """Extracts the category from a Code item. This backend generates two types of item which can be: 'code_quality_pylint' or 'code_quality_flake8' """ if item['analyzer'] == PYLINT: return CATEGORY_COQUA_PYLINT elif item['analyzer'] == FLAKE8: return CATEGORY_COQUA_FLAKE8 else: raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item): """Extracts the category from a Code item. This backend generates two types of item which can be: 'code_license_nomos' or 'code_license_scancode'. """ if item['analyzer'] == NOMOS: return CATEGORY_COLIC_NOMOS elif item['analyzer'] == SCANCODE: return CATEGORY_COLIC_SCANCODE else: raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item): """Extracts the category from a Code item. This backend generates two types of item which can be: 'code_language_linguist' or 'code_language_cloc' """ if item['analyzer'] == LINGUIST: return CATEGORY_COLANG_LINGUIST elif item['analyzer'] == CLOC: return CATEGORY_COLANG_CLOC else: raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item): """Extracts the category from a Code item. This backend generates the following types of item: - 'code_dependencies_pyreverse' - 'code_dependencies_jadolint' """ if item['analyzer'] == PYREVERSE: return CATEGORY_CODEP_PYREVERSE elif item['analyzer'] == JADOLINT: return CATEGORY_CODEP_JADOLINT else: raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item): """Extracts the category from a Code item. This backend generates the following types of item: - 'code_complexity_lizard_file'. - 'code_complexity_lizard_repository' """ if item['analyzer'] == LIZARD_FILE: return CATEGORY_COCOM_LIZARD_FILE elif item['analyzer'] == LIZARD_REPOSITORY: return CATEGORY_COCOM_LIZARD_REPOSITORY else: raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def fetch(self, category=CATEGORY_COQUA_PYLINT, paths=None, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME, branches=None, latest_items=False): """Fetch commits and add code quality information.""" if not self.entrypoint and category in [ CATEGORY_COQUA_FLAKE8, CATEGORY_COQUA_PYLINT ]: raise GraalError(cause="Entrypoint cannot be null") if not self.exec_path and category == CATEGORY_COQUA_JADOLINT: raise GraalError(cause="Exec path cannot be null") if category == CATEGORY_COQUA_PYLINT: self.analyzer_kind = PYLINT self.analyzer = ModuleAnalyzer(self.details, self.analyzer_kind) elif category == CATEGORY_COQUA_FLAKE8: self.analyzer_kind = FLAKE8 self.analyzer = ModuleAnalyzer(self.details, self.analyzer_kind) elif category == CATEGORY_COQUA_JADOLINT: self.analyzer_kind = JADOLINT self.analyzer = JadolintAnalyzer(self.exec_path, analysis=SMELLS) else: raise GraalError(cause="Unknown category %s" % category) items = super().fetch(category, from_date=from_date, to_date=to_date, branches=branches, latest_items=latest_items) return items
def metadata_category(item): """Extracts the category from a Code item. This backend generates the following types of item: - 'code_license_nomos' - 'code_license_scancode' - 'code_license_scancode_cli' """ if item['analyzer'] == NOMOS: return CATEGORY_COLIC_NOMOS elif item['analyzer'] == SCANCODE: return CATEGORY_COLIC_SCANCODE elif item['analyzer'] == SCANCODE_CLI: return CATEGORY_COLIC_SCANCODE_CLI else: raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def fetch(self, category=CATEGORY_COLANG_LINGUIST, paths=None, from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME, branches=None, latest_items=False): """Fetch commits and add code language information.""" if category == CATEGORY_COLANG_LINGUIST: self.analyzer_kind = LINGUIST elif category == CATEGORY_COLANG_CLOC: self.analyzer_kind = CLOC else: raise GraalError(cause="Unknown category %s" % category) self.repository_analyzer = RepositoryAnalyzer(self.details, self.analyzer_kind) items = super().fetch(category, branches=branches, latest_items=latest_items) return items
def analyze(self, **kwargs): """Add information about LOC, blank and commented lines using CLOC :param file_path: file path :returns result: dict of the results of the analysis """ result = {'blanks': 0, 'comments': 0, 'loc': 0 } file_path = kwargs['file_path'] flag = False try: msg = subprocess.check_output(['cloc', file_path]).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Cloc failed at %s, %s" % (file_path, e.output.decode("utf-8"))) finally: subprocess._cleanup() for line in msg.split("\n"): if flag: if not line.startswith("-----"): digested = " ".join(line.split()) info_file = digested.split(" ") blank_lines = int(info_file[2]) commented_lines = int(info_file[3]) loc = int(info_file[4]) result['blanks'] = blank_lines result['comments'] = commented_lines result['loc'] = loc break if line.lower().startswith("language"): flag = True result['ext'] = file_path.split(".")[-1] return result
def __analyze_scancode_cli(self, file_paths): """Add information about license using scancode-cli :param file_paths: file paths (in case of scancode_cli for concurrent execution on files) :returns result: list of the results of the analysis """ result = [] try: cmd_scancli = ['python3', self.exec_path] cmd_scancli.extend(file_paths) msg = subprocess.check_output(cmd_scancli).decode("utf-8") except subprocess.CalledProcessError as e: raise GraalError(cause="Scancode failed at %s, %s" % (file_paths, e.output.decode("utf-8"))) finally: subprocess._cleanup() output_content = '' outputs_json = [] for line in msg.split('\n'): if line == '': if output_content: output_json = json.loads(output_content)[1:] outputs_json.append(output_json) output_content = '' else: output_content += line if output_content: output_json = json.loads(output_content)[1:] outputs_json.append(output_json) for output_json in outputs_json: file_info = output_json[0]['files'][0] result.append(file_info) return result
def __init__(self, exec_path, analysis): if not GraalRepository.exists(exec_path): raise GraalError(cause="executable path %s not valid" % exec_path) self.exec_path = exec_path self.analysis = analysis
def __init__(self, exec_path): if not GraalRepository.exists(exec_path): raise GraalError(cause="executable path %s not valid" % exec_path) self.exec_path = exec_path self.search_pattern = re.compile(r'license\(s\) .*$')
def analyze(self, **kwargs): """Add security issue data using Bandit. :param folder_path: folder path :param details: if True, it returns information about single vulnerabilities :returns result: dict of the results of the analysis """ folder_path = kwargs['folder_path'] details = kwargs['details'] try: msg = subprocess.check_output(['bandit', '-r', folder_path]).decode("utf-8") except subprocess.CalledProcessError as e: msg = e.output.decode("utf-8") if not msg.startswith("Run started:"): raise GraalError(cause="Bandit failed at %s, %s" % (folder_path, msg)) finally: subprocess._cleanup() vulns = [] severities = [] confidences = [] loc = None descr = None severity = None confidence = None inIssue = False inOverview = False lines = msg.lower().split('\n') for line in lines: if line.startswith(">> issue: "): descr = line.replace(">> issue: ", "") inIssue = True elif line.startswith("code scanned:"): inOverview = True else: if inIssue: line = line.strip() if line.startswith("severity:"): tokens = [t.strip(":") for t in line.split(" ")] severity = tokens[1] confidence = tokens[-1] severities.append(severity) confidences.append(confidence) elif line.startswith("location:"): location = line.replace("location: ", "").replace(folder_path, "") line = location.split(":")[-1] file = location.replace(":" + line, "") vuln = { "file": file, "line": int(line), "severity": severity, "confidence": confidence, "descr": descr } vulns.append(vuln) severity = None confidence = None descr = None inIssue = False elif inOverview: if line.startswith("\ttotal lines of code:"): loc = line.split(":")[1].strip() loc = int(loc) break else: continue result = { 'loc_analyzed': loc, 'num_vulns': len(vulns), 'by_severity': self.__create_ranked_dict(severities), 'by_confidence': self.__create_ranked_dict(confidences) } if details: result['vulns'] = vulns return result