def repo_info_upd():
"""Updated repo_info page, to handle filtering."""
if request.form['btn'] == "Submit":
num_res = request.form['num_results']
r_user = request.form['repo_user']
r_name = request.form['repo_name']
r_cloned = request.form['is_cloned']
r_desc = request.form['repo_description']
# r_checked = request.form['repo_checked']
r_checked = ''
post_dict = {
'num_res': num_res,
'r_user': r_user,
'r_name': r_name,
'r_cloned': r_cloned,
'r_desc': r_desc,
'r_checked': r_checked
}
if request.form['btn'] == "Clear Filter":
post_dict = {
'num_res': 100,
'r_user': '',
'r_name': '',
'r_cloned': 'Any',
'r_desc': '',
'r_checked': ''
}
x = db.DbOps(db_u, db_p, db_h, db_db)
c_list = ['cloned', 'not_cloned']
res, num = x.display_repos(post_dict)
return render_template('repo_info.html',
results=res,
num_results=num,
select_opt=c_list,
p_dict=post_dict)
def api_repo():
"""Get most recent 100 results of Repos"""
x = db.DbOps(db_u, db_p, db_h, db_db)
res, pg, nq = x.display_repos_api()
results = process_repos(res, pg, nq)
resp = Response(results, status=200, mimetype='application/json')
return resp
def api_results():
"""Get most recent 100 results of Search Results"""
x = db.DbOps(db_u, db_p, db_h, db_db)
# results, number_results_in_table, page, num_results_from_query
res, pg, qnum = x.display_results_api()
results = process_results(res, pg, qnum)
resp = Response(results, status=200, mimetype='application/json')
return resp
def repo_info():
"""Initial repo_info page, showing various metadata for each repository."""
post_dict = {'num_res': 100, 'r_user': '', 'r_name': '',
'r_cloned': '', 'r_desc': '', 'r_checked': ''}
x = db.DbOps(db_u, db_p, db_h, db_db)
res = x.display_repos(post_dict)
c_list = ['cloned', 'not_cloned']
return render_template('repo_info.html', results=res, select_opt=c_list)
def mon():
"""Initial monitor page, showing GitMAD search results."""
post_dict = {'num_res':100, 'r_user':'', 'r_name':'',
'm_string': '', 'm_line':'', 'm_location':'', 'm_type':''}
x = db.DbOps(db_u, db_p, db_h, db_db)
res, num, stmt = x.display_match_results(post_dict['num_res'], post_dict)
s_val = regex_matches.to_match
s_list = []
for item in s_val:
s_list.append(item['match_type'])
return render_template('monitor.html', results=res, num_results=100,
select_opt=s_list, query=stmt, p_dict=post_dict)
def repo_info_upd():
"""Updated repo_info page, to handle filtering."""
num_res = request.form['num_results']
r_user = request.form['repo_user']
r_cloned = request.form['is_cloned']
r_desc = request.form['repo_description']
r_checked = request.form['repo_checked']
post_dict = {'num_res': num_res, 'r_user': r_user, 'r_cloned': r_cloned,
'r_desc': r_desc, 'r_checked': r_checked}
x = db.DbOps(db_u, db_p, db_h, db_db)
c_list = ['cloned', 'not_cloned']
res = x.display_repos(post_dict)
return render_template('repo_info.html', results=res, select_opt=c_list, p_dict=post_dict)
def mon_upd():
"""Updated monitor page, to handle filtering."""
num_res = request.form['num_results']
r_user = request.form['repo_user']
r_name = request.form['repo_name']
m_string = request.form['match_string']
m_line = request.form['match_line']
m_location = request.form['location']
m_type = request.form['match_type']
post_dict = {'num_res':num_res, 'r_user':r_user, 'r_name':r_name,
'm_string': m_string, 'm_line':m_line, 'm_location':m_location, 'm_type':m_type}
x = db.DbOps(db_u, db_p, db_h, db_db)
s_val = regex_matches.to_match
s_list = []
for item in s_val:
s_list.append(item['match_type'])
res, num, stmt = x.display_match_results(post_dict['num_res'], post_dict)
return render_template('monitor.html', results=res, num_results=num,
select_opt=s_list, query=stmt, p_dict=post_dict)
def iterate_thru_repos(self):
"""Driver for this class, to put other functions together, search, and generate results.
In addition, this will email results (if enabled) as well as insert them into the database.
"""
dbc = database.DbOps(self.db_user, self.db_pw, self.db_host,
self.db_database)
total_added = 0
total_list = []
for key in self.in_dict:
if 'cloned' in self.in_dict[key]:
repo_id = self.in_dict[key][7]
db_repo_id = dbc.get_recent_hash(repo_id)
repo = self.in_dict[key][0]
gen_file_list = self.generate_file_list(repo)
gen_item_info_list, f_recent_commit = self.search_file_list(
repo, gen_file_list, db_repo_id)
dbc.update_repo_info(self.in_dict[key], f_recent_commit)
num_added, added_list = dbc.update_repo_search_results(
self.in_dict[key], gen_item_info_list, self.search_type)
total_added += num_added
total_list.extend(added_list)
elif 'not_cloned' in self.in_dict[key]:
dbc.update_repo_info(self.in_dict[key], 'n/a')
print("Length of added list - a list of dicts")
print(len(total_list))
if total_list != []:
if self.do_email == 1:
num_results = len(total_list)
email_body = []
for item in range(0, len(total_list)):
if total_list[item]["Type"] == "Query Match":
email_body.append(total_list[item])
zip_path = self.write_files(total_list)
c = conf.Configure()
e_conf_file, e_file_check = c.check_for_file(c.e_filename)
e_from, e_to, e_domain, e_port, e_pw = c.populate_email_credentials(
e_conf_file, e_file_check)
e_alert = ea.EmailAction(e_from, e_to, e_domain, e_port, e_pw)
e_alert.send_alert(zip_path, num_results, email_body)
print("Items added to repo_search_results:")
print(total_added)
def api_repo_filter():
"""Accepts filters to return more specific results for Repository table."""
post_dict = request.get_json()
# {"num_res": 100, "page": 1, "repo_user": "******", "repo_name": "ccc", "repo_cloned": "yes", "repo_desc": "config"}
# Sanitizing the post_dict to only accept specific keys for filtering.
if ("num_res" in post_dict) or \
("page" in post_dict) or \
("repo_user" in post_dict) or \
("repo_name" in post_dict) or \
("repo_cloned" in post_dict) or \
("repo_desc" in post_dict):
pd = {}
if ("num_res" in post_dict) and (isinstance(post_dict["num_res"], int)
is True):
upd = {"num_res": post_dict["num_res"]}
pd.update(upd)
else:
upd = {"num_res": 100}
pd.update(upd)
if ("page"
in post_dict) and (isinstance(post_dict["page"], int) is True):
upd = {"page": post_dict["page"]}
pd.update(upd)
else:
upd = {"page": 1}
pd.update(upd)
if ("repo_user" in post_dict) and (isinstance(post_dict["repo_user"],
str) is True):
upd = {"repo_user": post_dict["repo_user"]}
pd.update(upd)
else:
upd = {"repo_user": "******"}
pd.update(upd)
if ("repo_name" in post_dict) and (isinstance(post_dict["repo_name"],
str) is True):
upd = {"repo_name": post_dict["repo_name"]}
pd.update(upd)
else:
upd = {"repo_name": "%"}
pd.update(upd)
if ("repo_cloned" in post_dict) \
and (isinstance(post_dict["repo_cloned"], str) is True) \
and ((post_dict["repo_cloned"] == "cloned") or (post_dict["repo_cloned"] == "not_cloned")):
upd = {"repo_cloned": post_dict["repo_cloned"]}
pd.update(upd)
else:
upd = {"repo_cloned": "%"}
pd.update(upd)
if ("repo_desc" in post_dict) and (isinstance(post_dict["repo_desc"],
str) is True):
upd = {"repo_desc": post_dict["repo_desc"]}
pd.update(upd)
else:
upd = {"repo_desc": "%"}
pd.update(upd)
x = db.DbOps(db_u, db_p, db_h, db_db)
res, pg, nq = x.display_repos_api(pd)
results = process_repos(res, pg, nq)
resp = Response(results, status=200, mimetype='application/json')
return resp
else:
invalid_post = {
"error":
"Invalid POST parameters passed in request.",
"helpString":
'''Send a JSON object (Content-Type: application/json) with at least one of the following keys:
1) page - Results are given in pages of 100. {"page": 2} equals 101-200, {"page": 3} equals 201-300 and so on.
2) repo_user - Username of the individual posting the repository.
3) repo_name - Name of the repository.
4) repo_cloned - Whether the repo(s) being searched for were cloned locally or not due to size {"repo_cloned": "not_cloned"}
5) repo_desc - Keywords to search for in the repository description {"repo_desc": "config"}
POST data example (at least one of these keys must be used):
{
"page": 1,
"repo_user": "******",
"repo_name": "<Name of the repository>",
"repo_cloned": "<cloned|not_cloned>",
"repo_desc": "<text string to search for in the repository description>"
}'''
}
resp = Response(json.dumps(invalid_post),
status=400,
mimetype='application/json')
return resp
def api_results_filter():
"""Accepts filters to return more specific results for Results table."""
# Sanitizing the post_dict to only accept specific keys for filtering.
post_dict = request.get_json()
if ("num_res" in post_dict) or \
("page" in post_dict) or \
("match_type" in post_dict) or \
("match_string" in post_dict) or \
("match_location" in post_dict) or \
("match_line" in post_dict) or \
("match_update_type" in post_dict) or \
("match_author" in post_dict) or \
("match_message" in post_dict):
pd = {}
if ("num_res" in post_dict) and (isinstance(post_dict["num_res"], int)
is True):
upd = {"num_res": post_dict["num_res"]}
pd.update(upd)
else:
upd = {"num_res": 100}
pd.update(upd)
if ("page"
in post_dict) and (isinstance(post_dict["page"], int) is True):
upd = {"page": post_dict["page"]}
pd.update(upd)
else:
upd = {"page": 1}
pd.update(upd)
if ("match_type" in post_dict) and (isinstance(post_dict["match_type"],
str) is True):
upd = {"match_type": post_dict["match_type"]}
pd.update(upd)
else:
upd = {"match_type": "%"}
pd.update(upd)
if ("match_string" in post_dict) and (isinstance(
post_dict["match_string"], str) is True):
upd = {"match_string": post_dict["match_string"]}
pd.update(upd)
else:
upd = {"match_string": "%"}
pd.update(upd)
if ("match_location" in post_dict) and (isinstance(
post_dict["match_location"], str) is True):
upd = {"match_location": post_dict["match_location"]}
pd.update(upd)
else:
upd = {"match_location": "%"}
pd.update(upd)
if ("match_line" in post_dict) and (isinstance(post_dict["match_line"],
str) is True):
upd = {"match_line": post_dict["match_line"]}
pd.update(upd)
else:
upd = {"match_line": "%"}
pd.update(upd)
if ("match_author" in post_dict) and (isinstance(
post_dict["match_author"], str) is True):
upd = {"match_author": post_dict["match_author"]}
pd.update(upd)
else:
upd = {"match_author": "%"}
pd.update(upd)
if ("match_message" in post_dict) and (isinstance(
post_dict["match_message"], str) is True):
upd = {"match_message": post_dict["match_message"]}
pd.update(upd)
else:
upd = {"match_message": "%"}
pd.update(upd)
if ("match_update_type" in post_dict) and (isinstance(
post_dict["match_update_type"], str) is True):
upd = {"match_update_type": post_dict["match_update_type"]}
pd.update(upd)
else:
upd = {"match_update_type": "%"}
pd.update(upd)
x = db.DbOps(db_u, db_p, db_h, db_db)
res, pg, nq = x.display_results_api(pd)
results = process_results(res, pg, nq)
resp = Response(results, status=200, mimetype='application/json')
return resp
else:
invalid_post = {
"error":
"Invalid POST parameters passed in request.",
"helpString":
'''Send a JSON object (Content-Type: application/json) with at least one of the following keys:
1) page - Results are given in pages of 100. {"page": 2} equals 101-200, {"page": 3} equals 201-300 and so on.
2) match_type - String for type of match {"match_type": ["password", "username"]}
3) match_string - A string to match the matched item {"match_string": "password"}
4) match_location - A string to match the file path {"match_location": "conf"}
5) match_line - A string that will match on the line the item was found {"match_line": "jdbc:mysql://"}
6) match_update_type - '+' for items added to the repo, '-' for items deleted. {"match_update_type": "-"}
7) match_author - String for the author of the commit {"match_author": "johndoe"}
8) match_message - String to match the commit message {"match_message": "credentials"}
# {"num_res":33, "page": 2, "match_type": "Password", "match_string": "secret",
# "match_location": "config", "match_line": "jdbc", "match_update_type": ["+","-"],
# "match_author": "John Doe", "match_message": "config"}
POST data example (at least one of these keys must be used):
{
"page": 2,
"match_type": "<type of match>",
"match_string": "<item that was matched>",
"match_location": "<file location where item was matched>",
"match_line": "<match something on the line of the initial match>",
"match_update_type": ["+","-"] addition or deletion to/from repo,
"match_author": "<commit author>",
"match_message": "<commit message>"
}'''
}
resp = Response(json.dumps(invalid_post),
status=400,
mimetype='application/json')
return resp