class DependencyCheck():
"""
"""
def __init__(self):
self.java = Java()
self.const = Constants()
self.utils = Utils()
self.config = Config()
def dependency_check(self, repo: str):
parent_dir = dirname(
dirname(os.path.abspath(os.path.dirname(__file__))))
os.chdir(parent_dir + "/tools/dependency-check/bin/")
try:
self.utils.execute_cmd(
"./dependency-check.sh --scan %s%s -f JSON -o %s%s" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo,
self.config.PATRONUS_DOWNLOAD_LOCATION, repo), repo)
logging.info("Successfully ran dependency-check on repo %s" %
(repo))
except:
logging.debug("Error running dependency-check on repo %s" % (repo))
return
def dependency_check_maven(self, repo: str):
try:
os.chdir("%s/%s" % (self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
self.utils.execute_cmd("mvn compile", repo)
self.utils.execute_cmd("mvn dependency-check:check", repo)
logging.info("Successfully ran dependency-check on repo %s" %
(repo))
except:
logging.debug("Error running dependency-check on repo %s" (repo))
return
def dependency_check_gradle(self, repo: str):
try:
os.chdir("%s/%s" % (self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
self.utils.execute_cmd("./gradlew dependencyCheckAnalyze", repo)
logging.info("Successfully ran dependency-check on repo %s" %
(repo))
except:
logging.debug("Error running dependency-check on repo %s" (repo),
repo)
return
class Java():
"""
"""
def __init__(self):
self.config = Config()
self.NEXUS_URL_OLD = self.config.NEXUS_URL_OLD
self.NEXUS_URL_NEW = self.config.NEXUS_URL_NEW
self.const = Constants()
self.utils = Utils()
self.maven = []
self.gradle = []
def check_build(self, repo: str):
"""
Check if build is maven or gradle
"""
build = ""
mvn = Path("%s%s/pom.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
gradle = Path("%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
if mvn.is_file():
build = "maven"
self.maven.append(repo)
return build
elif gradle.is_file():
build = "gradle"
self.maven.append(repo)
return build
else:
pass
def build_maven(self, repo: str):
"""
"""
os.chdir("%s%s" % (self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
try:
rc = self.utils.execute_cmd("mvn compile", repo)
rc2 = self.utils.execute_cmd("mvn spotbugs:spotbugs", repo)
logging.info("successfully build maven project %s " % (repo))
except:
logging.debug('Error building maven project %s' % (repo))
return
def build_gradle(self, repo: str):
"""
"""
os.chdir("%s%s" % (self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
try:
self.utils.execute_cmd("./gradlew clean check -x test", repo)
logging.info("successfully build gradle project %s " % (repo))
except:
logging.debug("Error building gradle project %s" % (repo))
return
def register_all_namespaces(self, filename):
"""
https://stackoverflow.com/questions/54439309/how-to-preserve-namespaces-when-parsing-xml-via-elementtree-in-python
"""
namespaces = dict(
[node for _, node in ET.iterparse(filename, events=['start-ns'])])
for ns in namespaces:
ET.register_namespace(ns, namespaces[ns])
return
def insert_build_tag(self, repo):
return
def add_build_tag(self, repo: str):
try:
tree = ET.parse("%s%s/pom.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
root = tree.getroot()
filename = "%s%s/pom.xml" % (
self.config.PATRONUS_DOWNLOAD_LOCATION, repo)
namespaces = dict([
node for _, node in ET.iterparse(filename, events=['start-ns'])
])
for ns in namespaces:
ET.register_namespace(ns, namespaces[ns])
root.append(ET.fromstring(self.const.POM_BUILD_TAG))
tree.write("%s%s/pom.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
xml_declaration=True)
except:
logging.debug("Failed modified pom.xml for maven project %s" %
(repo))
return
def modify_pom_for_findsecbugs(self, repo: str):
"""
"""
try:
tree = ET.parse("%s%s/pom.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo))
root = tree.getroot()
filename = "%s%s/pom.xml" % (
self.config.PATRONUS_DOWNLOAD_LOCATION, repo)
namespaces = dict([
node for _, node in ET.iterparse(filename, events=['start-ns'])
])
for ns in namespaces:
ET.register_namespace(ns, namespaces[ns])
build = root.find('{http://maven.apache.org/POM/4.0.0}build')
if build is None:
self.add_build_tag(repo)
build = root.find('{http://maven.apache.org/POM/4.0.0}build')
plugins = build.find('{http://maven.apache.org/POM/4.0.0}plugins')
plugins.append(ET.fromstring(self.const.FINDSECBUGS_XML))
plugins.append(ET.fromstring(self.const.DEPENDENCY_CHECK_XML))
tree.write("%s%s/pom.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
xml_declaration=True)
self.update_nexus_url(repo)
logging.info("successfully modified pom.xml for maven project %s" %
(repo))
except:
logging.debug("Failed modified pom.xml for maven project %s" %
(repo))
return
def parse_xml(self, repo: str):
"""
"""
with open("%s%s/pom.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as xml_file:
soup = BeautifulSoup(xml_file, "lxml")
artifactid = soup.artifactid.string
version = soup.version.string
if soup.packaging is not None:
packaging = soup.packaging.string
else:
packaging = "jar"
return "%s_%s.%s" % (artifactid, version, packaging)
def convert_xml_to_json(self, repo: str):
"""
"""
if self.check_build(repo) is "maven":
if os.path.exists("%s%s/target/spotbugsXml.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)):
with open(
"%s%s/target/spotbugsXml.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as file:
xmlString = file.read()
jsonString = json.dumps(xmltodict.parse(xmlString),
indent=4)
with open(
"%s%s/target/spotbugsXml.json" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
'w') as f:
f.write(jsonString)
if os.path.exists("%s%s/spotbugsXml.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)):
with open(
"%s%s/spotbugsXml.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as file:
res = xmltodict.parse(file.read())
result = json.dumps(res, indent=4)
with open(
"%s%s/spotbugsXml.json" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
'w') as f:
f.write(jsonString)
elif self.check_build(repo) is "gradle":
if os.path.exists("%s%s/build/reports/findbugs/main.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)):
with open(
"%s%s/build/reports/findbugs/main.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as file:
xmlString = file.read()
jsonString = json.dumps(xmltodict.parse(xmlString),
indent=4)
with open(
"%s%s/build/reports/findbugs/main.json" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
'w') as f:
f.write(jsonString)
if os.path.exists("%s%s/main.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)):
with open(
"%s%s/main.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as file:
xmlString = file.read()
jsonString = json.dumps(xmltodict.parse(xmlString),
indent=4)
with open(
"%s%s/main.json" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo),
'w') as f:
f.write(jsonString)
else:
pass
return
def modify_gradle_for_findsecbugs(self, repo: str):
"""
"""
lookup_lineno = []
lookup_2_lineno = []
try:
with open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as myFile:
for num, line in enumerate(myFile, 1):
if self.const.FINDSECBUGS_PATTERN_1 in line:
lookup_lineno.append(num)
# reading gradle file
f = open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo), "r")
contents = f.readlines()
f.close()
#apply plugin
contents.insert(lookup_lineno[0],
self.const.FINDSECBUGS_PATTERN_VALUE_1)
contents.insert(lookup_lineno[0],
self.const.DEPENDENCY_CHECK_PATTERN_VALUE_2)
# writing contents
f = open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo), "w")
contents = "".join(contents)
f.write(contents)
f.close()
with open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)) as myFile:
for num, line in enumerate(myFile, 1):
if self.const.FINDSECBUGS_PATTERN_2 in line:
lookup_2_lineno.append(num)
# reading gradle file
f = open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo), "r")
contents = f.readlines()
f.close()
#dependencies {
contents.insert(lookup_2_lineno[-1] + 2,
self.const.FINDSECBUGS_PATTERN_VALUE_2)
contents.insert(lookup_2_lineno[0] + 1,
self.const.DEPENDENCY_CHECK_PATTERN_VALUE_1)
# writing contents
f = open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo), "w")
contents = "".join(contents)
f.write(contents)
f.close()
f = open(
"%s%s/build.gradle" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo), "a")
f.write(self.const.FINDSECBUGS_PATTERN_VALUE_3)
f.write(self.const.DEPENDENCY_CHECK_PATTERN_VALUE_3)
f.close()
self.update_nexus_url(repo)
logging.info(
"successfully modified build.gradle for gradle project %s" %
(repo))
except:
logging.debug(
"Error modifying build.gradle for gradle project %s" % (repo))
return
def project_build(self, repo: str):
"""
"""
maven = []
gradle = []
if self.check_build(repo) == "maven":
try:
self.modify_pom_for_findsecbugs(repo)
self.build_maven(repo)
self.retry_for_failed_attempts(repo, "maven")
self.convert_xml_to_json(repo)
logging.info("Successfully build gradle project %s" % (repo))
except:
logging.debug("Failed building gradle project %s" % (repo))
elif self.check_build(repo) == "gradle":
try:
self.modify_gradle_for_findsecbugs(repo)
self.build_gradle(repo)
self.retry_for_failed_attempts(repo, "gradle")
self.convert_xml_to_json(repo)
logging.info("Successfully build gradle project %s" % (repo))
except:
logging.debug("Failed building gradle project %s" % (repo))
else:
pass
return
def retry_for_failed_attempts(self, repo: str, build: str):
if build is "maven":
if not os.path.exists(
"%s%s/target/spotbugsXml.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)):
self.fsb(repo, "maven")
if build is "gradle":
if not os.path.exists(
"%s%s/build/reports/findbugs/main.xml" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo)):
self.fsb(repo, "/gradle")
return
def fsb(self, repo: str, build: str):
"""
"""
parent_dir = dirname(
dirname(os.path.abspath(os.path.dirname(__file__))))
os.chdir(parent_dir + "/tools/findsecbugs")
try:
if build is "maven":
self.utils.execute_cmd(
"./findsecbugs.sh -xml -output %s%s/spotbugsXml.xml %s%s" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo,
self.config.PATRONUS_DOWNLOAD_LOCATION, repo), repo)
self.convert_xml_to_json(repo)
if build is "gradle":
self.utils.execute_cmd(
"./findsecbugs.sh -xml -output %s%s/main.xml %s%s" %
(self.config.PATRONUS_DOWNLOAD_LOCATION, repo,
self.config.PATRONUS_DOWNLOAD_LOCATION, repo), repo)
self.convert_xml_to_json(repo)
except Exception as e:
logging.debug("Error running find-sec-bugs on %s. Error: %s" %
(repo, e))
return