def fetch(self, category=CATEGORY_CODEP_PYREVERSE, paths=None,
from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
branches=None, latest_items=False):
"""Fetch commits and code (package and class) dependencies information."""
if not self.entrypoint and category == CATEGORY_CODEP_PYREVERSE:
raise GraalError(cause="Entrypoint cannot be null")
if not self.exec_path and category == CATEGORY_CODEP_JADOLINT:
raise GraalError(cause="Exec path cannot be null")
if category == CATEGORY_CODEP_PYREVERSE:
self.analyzer_kind = PYREVERSE
self.analyzer = PyreverseAnalyzer()
elif category == CATEGORY_CODEP_JADOLINT:
self.analyzer_kind = JADOLINT
self.analyzer = JadolintAnalyzer(self.exec_path, analysis=DEPENDENCIES)
else:
raise GraalError(cause="Unknown category %s" % category)
items = super().fetch(category,
from_date=from_date, to_date=to_date,
branches=branches, latest_items=latest_items)
return items
def analyze(self, **kwargs):
"""Add information using CLOC
:param file_path: file path
:param repository_level: set to True if analysis has to be performed on a repository
:returns result: dict of the results of the analysis
"""
file_path = kwargs['file_path']
repository_level = kwargs.get('repository_level', False)
try:
message = subprocess.check_output(['cloc',
file_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Cloc failed at %s, %s" %
(file_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
if repository_level:
results = self.__analyze_repository(message)
else:
results = self.__analyze_file(message)
results['ext'] = GraalRepository.extension(file_path)
return results
def fetch(self,
category=CATEGORY_COQUA_PYLINT,
paths=None,
from_date=DEFAULT_DATETIME,
to_date=DEFAULT_LAST_DATETIME,
branches=None,
latest_items=False):
"""Fetch commits and add code quality information."""
if category == CATEGORY_COQUA_PYLINT:
self.analyzer_kind = PYLINT
elif category == CATEGORY_COQUA_FLAKE8:
self.analyzer_kind = FLAKE8
else:
raise GraalError(cause="Unknown category %s" % category)
self.module_analyzer = ModuleAnalyzer(self.details, self.analyzer_kind)
items = super().fetch(category,
from_date=from_date,
to_date=to_date,
branches=branches,
latest_items=latest_items)
return items
def __init__(self,
uri,
git_path,
worktreepath=DEFAULT_WORKTREE_PATH,
entrypoint=None,
in_paths=None,
out_paths=None,
details=False,
tag=None,
archive=None):
super().__init__(uri,
git_path,
worktreepath,
entrypoint=entrypoint,
in_paths=in_paths,
out_paths=out_paths,
details=details,
tag=tag,
archive=archive)
if not self.entrypoint:
raise GraalError(cause="Entrypoint cannot be null")
self.analyzer_kind = None
self.analyzer = None
def fetch(self,
category=CATEGORY_COLIC_NOMOS,
paths=None,
from_date=DEFAULT_DATETIME,
to_date=DEFAULT_LAST_DATETIME,
branches=None,
latest_items=False):
"""Fetch commits and add license information."""
if category == CATEGORY_COLIC_SCANCODE:
self.analyzer_kind = SCANCODE
elif category == CATEGORY_COLIC_NOMOS:
self.analyzer_kind = NOMOS
else:
raise GraalError(cause="Unknown category %s" % category)
self.analyzer = LicenseAnalyzer(self.exec_path, self.analyzer_kind)
items = super().fetch(category,
from_date=from_date,
to_date=to_date,
branches=branches,
latest_items=latest_items)
return items
def analyze(self, **kwargs):
"""Add information about license
:param file_path: file path
:returns result: dict of the results of the analysis
"""
result = {'licenses': []}
file_path = kwargs['file_path']
try:
msg = subprocess.check_output(
[self.exec_path, '--json-pp', '-', '--license',
file_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Scancode failed at %s, %s" %
(file_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
licenses_raw = json.loads(msg)
if 'files' not in licenses_raw:
return result
result['licenses'] = licenses_raw['files'][0]['licenses']
return result
def __analyze_scancode(self, file_path):
"""Add information about license and copyright using scancode
:param file_path: file path (in case of scancode)
"""
result = {
'licenses': [],
'copyrights': [],
}
try:
msg = subprocess.check_output([
self.exec_path, '--json-pp', '-', '--license', '--copyright',
file_path
]).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Scancode failed at %s, %s" %
(file_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
scancode_raw = json.loads(msg)
if 'files' in scancode_raw:
result['licenses'] = scancode_raw['files'][0]['licenses']
result['copyrights'] = scancode_raw['files'][0]['copyrights']
return result
def analyze(self, **kwargs):
"""Add information about license
:param file_path: file path
:returns result: dict of the results of the analysis
"""
result = {'licenses': []}
file_path = kwargs['file_path']
try:
msg = subprocess.check_output([self.exec_path,
file_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Nomos failed at %s, %s" %
(file_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
licenses_raw = re.findall(self.search_pattern, msg)
licenses = []
for license_raw in licenses_raw:
license_digested = license_raw.split("license(s)")[1].strip()
licenses.append(license_digested)
if licenses:
result['licenses'] = licenses
return result
def fetch(self, category=CATEGORY_COCOM_LIZARD_FILE, paths=None,
from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
branches=None, latest_items=False):
"""Fetch commits and add code complexity information."""
items = super().fetch(category,
from_date=from_date, to_date=to_date,
branches=branches, latest_items=latest_items)
if category == CATEGORY_COCOM_LIZARD_FILE:
self.analyzer_kind = LIZARD_FILE
elif category == CATEGORY_COCOM_LIZARD_REPOSITORY:
self.analyzer_kind = LIZARD_REPOSITORY
elif category == CATEGORY_COCOM_SCC_FILE:
self.analyzer_kind = SCC_FILE
elif category == CATEGORY_COCOM_SCC_REPOSITORY:
self.analyzer_kind = SCC_REPOSITORY
else:
raise GraalError(cause="Unknown category %s" % category)
if "_file" in category:
self.analyzer = FileAnalyzer(self.details, self.analyzer_kind)
else:
self.analyzer = RepositoryAnalyzer(self.details, self.analyzer_kind)
return items
def analyze(self, **kwargs):
"""Get Jadolint results for a Dockerfile.
:param file_path: file path
:param result: dict of the results of the analysis
"""
results = []
result = {self.analysis: results}
file_path = kwargs['file_path']
if self.analysis == DEPENDENCIES:
cmd = ['java', '-jar', self.exec_path, file_path, '--deps']
else:
cmd = ['java', '-jar', self.exec_path, file_path, '--smells']
try:
msg = subprocess.check_output(cmd).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Jadolint failed at %s, %s" %
(file_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
results_raw = msg.split('\n')
for res_raw in results_raw:
res = res_raw.strip()
if res:
results.append(res)
return result
def analyze(self, **kwargs):
"""Get a UML class diagrams from a Python project.
:param module_path: module path
:param result: dict of the results of the analysis
"""
result = {}
module_path = kwargs['module_path']
try:
subprocess.check_output(['pyreverse', module_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Pyreverse failed at %s, %s" % (module_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
class_diagram = os.path.join(self.tmp_path, CLASSES_FILE_NAME)
if os.path.exists(class_diagram):
graph_classes = self.__dotfile2json(class_diagram)
result['classes'] = graph_classes
package_diagram = os.path.join(self.tmp_path, PACKAGES_FILE_NAME)
if os.path.exists(package_diagram):
graph_packages = self.__dotfile2json(package_diagram)
result['packages'] = graph_packages
return result
def __init__(self,
uri,
git_path,
exec_path,
worktreepath=DEFAULT_WORKTREE_PATH,
entrypoint=None,
in_paths=None,
out_paths=None,
tag=None,
archive=None):
super().__init__(uri,
git_path,
worktreepath,
entrypoint=entrypoint,
in_paths=in_paths,
out_paths=out_paths,
tag=tag,
archive=archive)
if not GraalRepository.exists(exec_path):
raise GraalError(cause="executable path %s not valid" % exec_path)
self.exec_path = exec_path
self.analyzer_kind = None
self.analyzer = None
def analyze(self, **kwargs):
"""Add quality checks data using Pylint.
:param module_path: module path
:param details: if True, it returns information about single modules
:returns result: dict of the results of the analysis
"""
module_path = kwargs['module_path']
details = kwargs['details']
try:
msg = subprocess.check_output(
['pylint', '-rn', '--output-format=text',
module_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
msg = e.output.decode("utf-8")
if not msg.startswith("***"):
raise GraalError(cause="Pylint failed at %s, %s" %
(module_path, msg))
finally:
subprocess._cleanup()
end = False
code_quality = None
mod_details = []
module_name = ""
lines = msg.split('\n')
modules = {}
for line in lines:
if line.startswith("***"):
if mod_details:
modules.update({module_name: mod_details})
module_name = line.strip("*").strip().replace("Module ", "")
mod_details = []
elif line.strip() == "":
continue
elif line.startswith("----"):
modules.update({module_name: mod_details})
end = True
else:
if end:
code_quality = line.split("/")[0].split(" ")[-1]
break
else:
mod_details.append(line)
result = {
'quality': code_quality,
'num_modules': len(modules),
'warnings': sum([len(mod) for mod in modules])
}
if details:
result['modules'] = modules
return result
def __init__(self, exec_path, cli=False):
if not GraalRepository.exists(exec_path):
raise GraalError(cause="executable path %s not valid" % exec_path)
self.exec_path = exec_path
self.cli = cli
if self.cli:
exec_path = self.exec_path.replace(SCANCODE_CLI_EXEC,
CONFIGURE_EXEC)
_ = subprocess.check_output([exec_path]).decode("utf-8")
def metadata_category(item):
"""Extracts the category from a Code item.
This backend generates two types of item which can be:
'code_quality_pylint' or 'code_quality_flake8'
"""
if item['analyzer'] == PYLINT:
return CATEGORY_COQUA_PYLINT
elif item['analyzer'] == FLAKE8:
return CATEGORY_COQUA_FLAKE8
else:
raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item):
"""Extracts the category from a Code item.
This backend generates two types of item which can be:
'code_license_nomos' or 'code_license_scancode'.
"""
if item['analyzer'] == NOMOS:
return CATEGORY_COLIC_NOMOS
elif item['analyzer'] == SCANCODE:
return CATEGORY_COLIC_SCANCODE
else:
raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item):
"""Extracts the category from a Code item.
This backend generates two types of item which can be:
'code_language_linguist' or 'code_language_cloc'
"""
if item['analyzer'] == LINGUIST:
return CATEGORY_COLANG_LINGUIST
elif item['analyzer'] == CLOC:
return CATEGORY_COLANG_CLOC
else:
raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item):
"""Extracts the category from a Code item.
This backend generates the following types of item:
- 'code_dependencies_pyreverse'
- 'code_dependencies_jadolint'
"""
if item['analyzer'] == PYREVERSE:
return CATEGORY_CODEP_PYREVERSE
elif item['analyzer'] == JADOLINT:
return CATEGORY_CODEP_JADOLINT
else:
raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def metadata_category(item):
"""Extracts the category from a Code item.
This backend generates the following types of item:
- 'code_complexity_lizard_file'.
- 'code_complexity_lizard_repository'
"""
if item['analyzer'] == LIZARD_FILE:
return CATEGORY_COCOM_LIZARD_FILE
elif item['analyzer'] == LIZARD_REPOSITORY:
return CATEGORY_COCOM_LIZARD_REPOSITORY
else:
raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def fetch(self,
category=CATEGORY_COQUA_PYLINT,
paths=None,
from_date=DEFAULT_DATETIME,
to_date=DEFAULT_LAST_DATETIME,
branches=None,
latest_items=False):
"""Fetch commits and add code quality information."""
if not self.entrypoint and category in [
CATEGORY_COQUA_FLAKE8, CATEGORY_COQUA_PYLINT
]:
raise GraalError(cause="Entrypoint cannot be null")
if not self.exec_path and category == CATEGORY_COQUA_JADOLINT:
raise GraalError(cause="Exec path cannot be null")
if category == CATEGORY_COQUA_PYLINT:
self.analyzer_kind = PYLINT
self.analyzer = ModuleAnalyzer(self.details, self.analyzer_kind)
elif category == CATEGORY_COQUA_FLAKE8:
self.analyzer_kind = FLAKE8
self.analyzer = ModuleAnalyzer(self.details, self.analyzer_kind)
elif category == CATEGORY_COQUA_JADOLINT:
self.analyzer_kind = JADOLINT
self.analyzer = JadolintAnalyzer(self.exec_path, analysis=SMELLS)
else:
raise GraalError(cause="Unknown category %s" % category)
items = super().fetch(category,
from_date=from_date,
to_date=to_date,
branches=branches,
latest_items=latest_items)
return items
def metadata_category(item):
"""Extracts the category from a Code item.
This backend generates the following types of item:
- 'code_license_nomos'
- 'code_license_scancode'
- 'code_license_scancode_cli'
"""
if item['analyzer'] == NOMOS:
return CATEGORY_COLIC_NOMOS
elif item['analyzer'] == SCANCODE:
return CATEGORY_COLIC_SCANCODE
elif item['analyzer'] == SCANCODE_CLI:
return CATEGORY_COLIC_SCANCODE_CLI
else:
raise GraalError(cause="Unknown analyzer %s" % item['analyzer'])
def fetch(self, category=CATEGORY_COLANG_LINGUIST, paths=None,
from_date=DEFAULT_DATETIME, to_date=DEFAULT_LAST_DATETIME,
branches=None, latest_items=False):
"""Fetch commits and add code language information."""
if category == CATEGORY_COLANG_LINGUIST:
self.analyzer_kind = LINGUIST
elif category == CATEGORY_COLANG_CLOC:
self.analyzer_kind = CLOC
else:
raise GraalError(cause="Unknown category %s" % category)
self.repository_analyzer = RepositoryAnalyzer(self.details, self.analyzer_kind)
items = super().fetch(category, branches=branches, latest_items=latest_items)
return items
def analyze(self, **kwargs):
"""Add information about LOC, blank and commented lines using CLOC
:param file_path: file path
:returns result: dict of the results of the analysis
"""
result = {'blanks': 0,
'comments': 0,
'loc': 0
}
file_path = kwargs['file_path']
flag = False
try:
msg = subprocess.check_output(['cloc', file_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Cloc failed at %s, %s" % (file_path, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
for line in msg.split("\n"):
if flag:
if not line.startswith("-----"):
digested = " ".join(line.split())
info_file = digested.split(" ")
blank_lines = int(info_file[2])
commented_lines = int(info_file[3])
loc = int(info_file[4])
result['blanks'] = blank_lines
result['comments'] = commented_lines
result['loc'] = loc
break
if line.lower().startswith("language"):
flag = True
result['ext'] = file_path.split(".")[-1]
return result
def __analyze_scancode_cli(self, file_paths):
"""Add information about license using scancode-cli
:param file_paths: file paths (in case of scancode_cli for concurrent execution on files)
:returns result: list of the results of the analysis
"""
result = []
try:
cmd_scancli = ['python3', self.exec_path]
cmd_scancli.extend(file_paths)
msg = subprocess.check_output(cmd_scancli).decode("utf-8")
except subprocess.CalledProcessError as e:
raise GraalError(cause="Scancode failed at %s, %s" %
(file_paths, e.output.decode("utf-8")))
finally:
subprocess._cleanup()
output_content = ''
outputs_json = []
for line in msg.split('\n'):
if line == '':
if output_content:
output_json = json.loads(output_content)[1:]
outputs_json.append(output_json)
output_content = ''
else:
output_content += line
if output_content:
output_json = json.loads(output_content)[1:]
outputs_json.append(output_json)
for output_json in outputs_json:
file_info = output_json[0]['files'][0]
result.append(file_info)
return result
def __init__(self, exec_path, analysis):
if not GraalRepository.exists(exec_path):
raise GraalError(cause="executable path %s not valid" % exec_path)
self.exec_path = exec_path
self.analysis = analysis
def __init__(self, exec_path):
if not GraalRepository.exists(exec_path):
raise GraalError(cause="executable path %s not valid" % exec_path)
self.exec_path = exec_path
self.search_pattern = re.compile(r'license\(s\) .*$')
def analyze(self, **kwargs):
"""Add security issue data using Bandit.
:param folder_path: folder path
:param details: if True, it returns information about single vulnerabilities
:returns result: dict of the results of the analysis
"""
folder_path = kwargs['folder_path']
details = kwargs['details']
try:
msg = subprocess.check_output(['bandit', '-r',
folder_path]).decode("utf-8")
except subprocess.CalledProcessError as e:
msg = e.output.decode("utf-8")
if not msg.startswith("Run started:"):
raise GraalError(cause="Bandit failed at %s, %s" %
(folder_path, msg))
finally:
subprocess._cleanup()
vulns = []
severities = []
confidences = []
loc = None
descr = None
severity = None
confidence = None
inIssue = False
inOverview = False
lines = msg.lower().split('\n')
for line in lines:
if line.startswith(">> issue: "):
descr = line.replace(">> issue: ", "")
inIssue = True
elif line.startswith("code scanned:"):
inOverview = True
else:
if inIssue:
line = line.strip()
if line.startswith("severity:"):
tokens = [t.strip(":") for t in line.split(" ")]
severity = tokens[1]
confidence = tokens[-1]
severities.append(severity)
confidences.append(confidence)
elif line.startswith("location:"):
location = line.replace("location: ",
"").replace(folder_path, "")
line = location.split(":")[-1]
file = location.replace(":" + line, "")
vuln = {
"file": file,
"line": int(line),
"severity": severity,
"confidence": confidence,
"descr": descr
}
vulns.append(vuln)
severity = None
confidence = None
descr = None
inIssue = False
elif inOverview:
if line.startswith("\ttotal lines of code:"):
loc = line.split(":")[1].strip()
loc = int(loc)
break
else:
continue
result = {
'loc_analyzed': loc,
'num_vulns': len(vulns),
'by_severity': self.__create_ranked_dict(severities),
'by_confidence': self.__create_ranked_dict(confidences)
}
if details:
result['vulns'] = vulns
return result
