def store_index(self, target_dir):
""" Zip files in target_dir/central-index dir and store to S3 """
with tempdir() as temp_dir:
central_index_dir = os.path.join(target_dir, self._INDEX_DIRNAME)
archive_path = os.path.join(temp_dir, self._INDEX_ARCHIVE)
Archive.zip_file(central_index_dir, archive_path, junk_paths=True)
self.store_file(archive_path, self._INDEX_ARCHIVE)
def store_depcheck_db(self, data_dir):
""" Zip CVE DB file and store to S3 """
with tempdir() as archive_dir:
archive_path = os.path.join(archive_dir, self._DB_ARCHIVE)
db_file_path = os.path.join(data_dir, self._DB_FILENAME)
Archive.zip_file(db_file_path, archive_path, junk_paths=True)
self.store_file(archive_path, self._DB_ARCHIVE)
def retrieve_depcheck_db_if_exists(self, data_dir):
""" Retrieve zipped CVE DB file as stored on S3 and extract"""
if self.object_exists(self._DB_ARCHIVE):
with tempdir() as archive_dir:
archive_path = os.path.join(archive_dir, self._DB_ARCHIVE)
self.retrieve_file(self._DB_ARCHIVE, archive_path)
Archive.extract_zip(archive_path, data_dir)
return True
return False
def _resolve_versions(to_solve):
"""
Resolve version ranges in to_solve
:param to_solve: {"groupId:artifactId": "version-range"}
:return: {"groupId:artifactId": "version"}
"""
if not to_solve:
return {}
with tempdir() as tmpdir:
with cwd(tmpdir):
MavenSolver._generate_pom_xml(to_solve)
return MavenSolver._dependencies_from_pom_xml()
def retrieve_index_if_exists(self, target_dir):
""" Retrieve central-index.zip from S3 and extract into target_dir/central-index"""
if self.object_exists(self._INDEX_ARCHIVE):
with tempdir() as temp_dir:
archive_path = os.path.join(temp_dir, self._INDEX_ARCHIVE)
central_index_dir = os.path.join(target_dir,
self._INDEX_DIRNAME)
self.retrieve_file(self._INDEX_ARCHIVE, archive_path)
Archive.extract_zip(archive_path,
central_index_dir,
mkdest=True)
return True
return False
def execute(self, arguments):
"""
:param arguments: optional argument 'only_already_scanned' to run only on already analysed packages
:return: EPV dict describing which packages should be analysed
"""
only_already_scanned = arguments.pop('only_already_scanned',
True) if arguments else True
ignore_modification_time = arguments.pop('ignore_modification_time',
False) if arguments else False
self._strict_assert(not arguments)
s3 = StoragePool.get_connected_storage('S3OWASPDepCheck')
with tempdir() as temp_data_dir:
s3.retrieve_depcheck_db_if_exists(temp_data_dir)
self._update_dep_check_db(temp_data_dir)
s3.store_depcheck_db(temp_data_dir)
cve_db = self._get_snyk_vulndb()
s3 = StoragePool.get_connected_storage('S3Snyk')
s3.store_vulndb(cve_db)
last_sync_datetime = s3.update_sync_date()
to_update = []
for package_name, cve_records in cve_db.get('npm', {}).items():
for record in cve_records:
modification_time = datetime_parser.parse(
record['modificationTime'])
if ignore_modification_time or modification_time >= last_sync_datetime:
affected_versions = self._get_versions_to_scan(
package_name, record['semver']['vulnerable'],
only_already_scanned)
for version in affected_versions:
to_update.append({
'ecosystem': 'npm',
'name': package_name,
'version': version
})
return {'modified': to_update}
def _get_snyk_vulndb(self):
"""
:return: retrieve Snyk CVE db
"""
with tempdir() as vulndb_dir:
# clone vulndb git repo
self.log.debug("Cloning snyk/vulndb repo")
Git.clone(self._VULNDB_GIT_REPO, vulndb_dir)
with cwd(vulndb_dir):
# install dependencies
self.log.debug("Installing snyk/vulndb dependencies")
TimedCommand.get_command_output(['npm', 'install'])
# generate database (json in file)
self.log.debug("Generating snyk/vulndb")
TimedCommand.get_command_output([
os.path.join('cli', 'shrink.js'), 'data',
self._VULNDB_FILENAME
])
# parse the JSON so we are sure that we have a valid JSON
with open(self._VULNDB_FILENAME) as f:
return json.load(f)
def _run_owasp_dep_check(self, scan_path, experimental=False):
def _clean_dep_check_tmp():
for dcdir in glob.glob(os.path.join(gettempdir(), 'dctemp*')):
rmtree(dcdir)
s3 = StoragePool.get_connected_storage('S3OWASPDepCheck')
depcheck = os.path.join(os.environ['OWASP_DEP_CHECK_PATH'], 'bin', 'dependency-check.sh')
with tempdir() as temp_data_dir:
retrieved = s3.retrieve_depcheck_db_if_exists(temp_data_dir)
if not retrieved:
self.log.debug('No cached OWASP Dependency-Check DB, generating fresh now ...')
command = [depcheck, '--updateonly', '--data', temp_data_dir]
# give DependencyCheck 30 minutes to download the DB
TimedCommand.get_command_output(command, graceful=False, timeout=1800)
report_path = os.path.join(temp_data_dir, 'report.xml')
command = [depcheck,
'--noupdate',
'--format', 'XML',
'--project', 'test',
'--data', temp_data_dir,
'--scan', scan_path,
'--out', report_path]
if experimental:
command.extend(['--enableExperimental'])
output = []
try:
self.log.debug('Running OWASP Dependency-Check to scan %s for vulnerabilities' %
scan_path)
output = TimedCommand.get_command_output(command,
graceful=False,
timeout=600) # 10 minutes
with open(report_path) as r:
report_dict = anymarkup.parse(r.read())
except (TaskError, FileNotFoundError) as e:
_clean_dep_check_tmp()
for line in output:
self.log.warning(line)
self.log.exception(str(e))
return {'summary': ['OWASP Dependency-Check scan failed'],
'status': 'error',
'details': []}
# If the CVEDBSyncTask has never been run before, we just had to create the DB ourselves
# Make the life easier for other workers and store it to S3
s3.store_depcheck_db_if_not_exists(temp_data_dir)
_clean_dep_check_tmp()
results = []
dependencies = report_dict.get('analysis', {}).get('dependencies', {}).get('dependency', [])
if not isinstance(dependencies, list):
dependencies = [dependencies]
for dependency in dependencies:
vulnerabilities = dependency.get('vulnerabilities', {}).get('vulnerability', [])
if not isinstance(vulnerabilities, list):
vulnerabilities = [vulnerabilities]
for vulnerability in vulnerabilities:
av = vulnerability.get('cvssAccessVector')
av = av[0] if av else '?'
ac = vulnerability.get('cvssAccessComplexity')
ac = ac[0] if ac else '?'
au = vulnerability.get('cvssAuthenticationr')
au = au[0] if au else '?'
c = vulnerability.get('cvssConfidentialImpact')
c = c[0] if c else '?'
i = vulnerability.get('cvssIntegrityImpact')
i = i[0] if i else '?'
a = vulnerability.get('cvssAvailabilityImpact')
a = a[0] if a else '?'
vector = "AV:{AV}/AC:{AC}/Au:{Au}/C:{C}/I:{I}/A:{A}".\
format(AV=av, AC=ac, Au=au, C=c, I=i, A=a)
result = {
'cvss': {
'score': vulnerability.get('cvssScore'),
'vector': vector
}
}
references = vulnerability.get('references', {}).get('reference', [])
if not isinstance(references, list):
references = [references]
result['references'] = [r.get('url') for r in references]
for field in ['severity', 'description']:
result[field] = vulnerability.get(field)
result['id'] = vulnerability.get('name')
results.append(result)
return {'summary': [r['id'] for r in results],
'status': 'success',
'details': results}