def extract_tag_info(tag, repo_name):
is_milestone = False
tag_tokens = tag.split(".")
if len(tag_tokens[1]) == 2:
# Format matches YYYY.mm.dd-M20500622-01
if tag[11] == "M":
is_milestone = True
tag_dt = datetime.datetime.strptime(tag[0:10], '%Y.%m.%d')
tag_label = tag_dt.strftime("%Y.%m.%d")
return (tag_dt, is_milestone, tag_label)
elif len(tag_tokens[1]) == 4:
# Format matches YYYY.mm00.0-MYYYYmmdd-01
if tag[12] == "M":
is_milestone = True
tag_dt = datetime.datetime.strptime(tag[13:21], '%Y%m%d')
else:
is_milestone = False
tag_dt = datetime.datetime.strptime(tag[12:20], '%Y%m%d')
tag_label = tag_dt.strftime("%Y.%m.%d")
return (tag_dt, is_milestone, tag_label)
else:
# Unknown!
print(" Date Style Unknown!: {}".format(tag))
msg = "Encountered unknown SVN tag datetime format [{}]".format(tag)
cx_logs.create_log_entry(status = "error", msg = msg, name = repo_name)
tag_dt = datetime.datetime.now()
tag_label = tag_dt.strftime("%Y.%m.%d")
return (tag_dt, is_milestone, tag_label)
def count_lines_of_code(repo_name, tag_date):
if cx_opts.count_loc:
print(" Counting lines of code")
#mvn = subprocess.Popen([shutil.which(cx_opts.cloc_prg_name), "."],
mvn = subprocess.Popen([cx_opts.cloc_prg_name, "."],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=0,
cwd=cx_opts.working_dir,
shell=True)
for line in mvn.stdout:
if "SUM" in line:
sums = line.split()
msg = "Counted projects LOC.".format(repo_name, tag_date)
cx_logs.create_log_entry(
msg=msg,
name=repo_name,
release_date=tag_date,
files_count=int(sums[1]),
comment_lines=int(sums[3]),
blank_lines=int(sums[2]),
code_lines=int(sums[4]),
total_lines_count=(int(sums[2]) + int(sums[3]) +
int(sums[4])))
print(line.strip())
def is_project_active(project):
global target_list
if cx_opts.use_targets == False:
return True
else:
for target in target_list:
if project == target:
return True
msg = "Found project {} in SVN but skipping it because its not in (targets.txt).".format(project)
cx_logs.create_log_entry(msg = msg)
print(" " + msg)
return False
def read_targets_file():
global target_list
if cx_opts.use_targets:
msg = "Target list activated, reading (targets.txt)."
print(msg)
cx_logs.create_log_entry(msg = msg)
try:
with open('targets.txt', 'r') as f:
target_list = f.read().splitlines()
except FileNotFoundError:
print("Could not read (targets.txt)!")
sys.exit()
print(target_list)
print("Target list read.")
else:
msg = "Target list is NOT active, all projects will be processed."
print(msg)
cx_logs.create_log_entry(msg = msg)
def find_last_commit_url(repo_name):
"""
Within a given repo there should be a 'tags' folder. This scans the tags
folder to find the entry with the most recent datestamp. Each tag is
assumed to have a format like this:
/<repo_name>/tags/2019.10.05-M20500622-01
2019.10.05-M20500622-01
# -> New sample 2019.1100.0-M20191123-01 <-- This version has the date after the M
NOTE: The 'M' after the 2019.10.05 date indicates 'Milestone'. We skip tags without that.
"""
newest_dt = None
tag_label = ""
prev_project_url = project_url = None
try:
tags_url = (cx_opts.svn_server + "/" + repo_name + "/tags")
repo = svn.remote.RemoteClient(tags_url, username = cx_opts.svn_username, password = cx_opts.svn_password)
tag_list = repo.list(extended=True)
for tag_entry in tag_list:
tag_dt, is_milestone, tag_label = extract_tag_info(tag_entry["name"], repo_name)
if is_milestone:
if newest_dt == None or newest_dt <= tag_dt:
print(" Found New Milestone")
prev_project_url = project_url
newest_dt = tag_dt
project_url = tags_url + "/" + tag_entry["name"]
if cx_opts.milestone_ver == 1 and prev_project_url is not None:
# In the case we should use the previous mileston, if it exists
print(" reverting to previous milestone!")
project_url = prev_project_url
if project_url != None:
msg = "Found release version for SVN project [{}] with release date [{}].".format(repo_name, tag_label)
print(" " + msg)
cx_logs.create_log_entry(msg = msg, name = repo_name, release_date = tag_label)
except svn.exception.SvnException:
msg = "A current release version could not be determined for SVN project '{}' because a SVN 'tags' folder could not be located!".format(repo_name)
print(" " + msg)
cx_logs.create_log_entry(status = "error", msg = msg, name = repo_name)
return (project_url, tag_label)
def get_maven_deps(repo_name, tag_date):
if cx_opts.maven_dependencies:
print(" Downloading Maven dependencies.")
#mvn = subprocess.Popen([shutil.which("mvn"), "dependency:unpack-dependencies", "-DoutputDirectory=maven_src"],
mvn = subprocess.Popen([
"mvn", "dependency:unpack-dependencies",
"-DoutputDirectory=maven_src"
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=0,
cwd=cx_opts.working_dir,
shell=True)
mvn.stdin.close()
status = True
for line in mvn.stdout:
if "[ERROR]" in line:
status = False
print(line.strip())
if status:
msg = "Maven succesfully downloaded source dependencies for the project [{}] with release date [{}].".format(
repo_name, tag_date)
cx_logs.create_log_entry(msg=msg,
name=repo_name,
release_date=tag_date)
print(" " + msg)
else:
msg = "Maven encountered error downloading source dependencies for the project [{}] with release date [{}].".format(
repo_name, tag_date)
cx_logs.create_log_entry(status="error",
msg=msg,
name=repo_name,
release_date=tag_date)
print(" " + msg)
else:
print(" Skipping Maven dependency download.")
def run_scan(repo_name, tag_date):
if cx_opts.checkmarx_scan:
print(" Dispatching scan to Checkmarx.")
location_path = os.getcwd() + "\\" + cx_opts.working_dir_raw
sast = subprocess.Popen([
cx_opts.cx_cli_path, "Scan", "-v", "-CxServer", cx_opts.cx_server,
"-projectName", "\"CxServer\\{}\"".format(repo_name), "-CxUser",
cx_opts.cx_username, "-CxPassword", cx_opts.cx_password,
"-Locationtype", "folder", "-locationpath", location_path,
"-Preset", cx_opts.cx_preset
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
bufsize=0)
sast.stdin.close()
status = True
for line in sast.stdout:
if "Scan progress request failure" in line:
status = False
msg = "Checkmarx encountered scan request error scanning the project [{}] with release date [{}].".format(
repo_name, tag_date)
cx_logs.create_log_entry(status="error",
msg=msg,
name=repo_name,
release_date=tag_date)
print(" " + msg)
elif "Unsuccessful login" in line:
status = False
msg = "Checkmarx encountered and unsuccessful login scanning the project [{}] with release date [{}].".format(
repo_name, tag_date)
cx_logs.create_log_entry(status="error",
msg=msg,
name=repo_name,
release_date=tag_date)
print(" " + msg)
print(line.strip())
if status:
msg = "Checkmarx succesfully scaned the project [{}] with release date [{}].".format(
repo_name, tag_date)
cx_logs.create_log_entry(msg=msg,
name=repo_name,
release_date=tag_date)
print(" " + msg)
else:
print(" Skipping Checkmarx scan.")
from datetime import datetime
cx_logs.start_log()
cx_utils.read_cli_opts()
cx_utils.clean_working_dir("")
if cx_opts.svn_password == None:
print("Please enter the SVN server password.")
cx_opts.svn_password = getpass()
if cx_opts.cx_password == None:
print("Please enter the Checkmarx server password.")
cx_opts.cx_password = getpass()
print("Reading projects from SVN Server")
now = datetime.now()
nowStr = now.strftime("%Y-%m-%d %H:%M:%S")
cx_logs.create_log_entry(
msg=
"Started sending scans at [{}] from SVN server [{}] to Checkmarx Server [{}]"
.format(nowStr, cx_opts.svn_server, cx_opts.cx_server))
client = svn.remote.RemoteClient(cx_opts.svn_server,
username=cx_opts.svn_username,
password=cx_opts.svn_password)
repo_list = client.list(extended=True)
cx_svn.read_targets_file()
for repo_entry in repo_list:
if repo_entry["is_directory"] == True and cx_svn.is_project_active(
repo_entry['name']):
print("\nFound Repo: {}".format(repo_entry['name']))
proj_url, tag_date = cx_svn.find_last_commit_url(repo_entry['name'])
if proj_url:
print(" Repo URL: {}".format(proj_url))
proj = svn.remote.RemoteClient(proj_url,
username=cx_opts.svn_username,