def update_indicator(indicator_id, report_id):
print('{} {}'.format(indicator_id, report_id))
form_data = request.form
report_indicator = Links.query.filter_by(report=report_id).filter_by(
indicator=indicator_id).first()
if report_indicator:
print('link found')
if form_data.get('kill_chain'):
print('kill chain {}'.format(form_data.get('kill_chain')))
report_indicator.kill_chain = form_data.get('kill_chain')
if form_data.get('confidence'):
print('confidence {}'.format(form_data.get('confidence')))
report_indicator.confidence = form_data.get('confidence')
if form_data.get('diamond_model'):
print('diamond_model {}'.format(form_data.get('diamond_model')))
report_indicator.diamond_model = form_data.get('diamond_model')
db.session.commit()
db.session.flush()
else:
report_indicator = Links(report=report_id, indicator=indicator_id)
report_indicator.kill_chain = form_data.get('kill_chain', 'Unknown')
report_indicator.confidence = form_data.get('confidence', 'Low')
report_indicator.diamond_model = form_data.get('diamond_model',
'Unknown')
add_db_entry(report_indicator)
return 'success'
def user_profile():
form = LinksForm()
if form.validate_on_submit():
links = Links(form.youtube.data, form.twitter.data,
form.instagram.data, form.facebook.data)
current_user.links = links
current_user.save()
return render_template('user_profile.html', form=form)
def add_url_to_db(url=None,sentiment=None):
if url is None or sentiment is None:
return None
Session = sessionmaker(bind=engine)
session = Session()
url_id = get_salted_hash(url)
url_text = url
record = session.query(Links).filter_by(url_id=url_id).first()
if record:
record.sentiment = sentiment
else:
cc_url = Links(url_id=url_id,url_text=url_text,sentiment=sentiment)
session.add(cc_url)
session.commit()
session.close()
def home():
if request.method == 'GET':
return render_template('index.html')
elif request.method == 'POST':
if 'link' in request.form:
link = Links.query.filter_by(original=request.form['link']).first()
if link:
pass
else:
link = Links(request.form['link'])
db.session.add(link)
db.session.commit()
return render_template('index.html', link_name=str(link.id))
else:
return "WHAT"
else:
return "GET OUTTA HERE"
def get_pagination(self):
"""
function parser the pagination xml file
"""
for pagination in self.meta:
total = pagination.find('total').text
count = pagination.find('count').text
per_page = pagination.find('per_page').text
current_page = pagination.find('current_page').text
total_pages = pagination.find('total_pages').text
links = []
for child in pagination:
if child.tag == 'links':
next = child.find('next').text
links.append(Links(next))
return Pagination(total, count, per_page, current_page,
total_pages, links)
def dashboard(unique_id):
selected_link_info = None
name = get_user_name(Users.query.filter_by(unique_id=unique_id).first())
if UserDashboard.query.filter_by(unique_id=unique_id).first() is not None:
selected_link = ""
obj = UserDashboard.query.filter_by(unique_id=unique_id).all()
for o in obj:
selected_link = o.short_url
return redirect(url_for("dashboard_with_links",
unique_id=unique_id,
selected_link=selected_link,
selected_link_info=selected_link_info,
name=name))
check_create_button = False
if request.method == "POST":
check_create_button = True
user = UserDashboard()
user.unique_id = unique_id
user.original_url = request.form["long_url"]
user.title = request.form["long_url"]
short_url = generate_short_link()
user.short_url = short_url
redirect_link = Links()
redirect_link.original_url = user.original_url
redirect_link.short_url = user.short_url
db.session.add_all([redirect_link, user])
db.session.commit()
return render_template("dashboard.html",
flag=check_create_button,
short_url=short_url,
unique_id=unique_id,
selected_link_info=selected_link_info,
name=name)
return render_template("dashboard.html",
flag=check_create_button,
unique_id=unique_id,
selected_link_info=selected_link_info,
name=name)
def save_to_db(self,const):
# store in db, uses self.data Extract objects, iterate through and generate the appropriate injections for the db
if const is "search_term":
s_db = Search(date=timezone.now(),term=self.data[0].search_term)
print "Adding %s data into db."% s_db
s_db.save()
for q in self.data:
print q
# save data around Search term for each Extract object in self.data
# each Extract object has multiple links, get them all and associate to the created search term
try:
for url in q.job_urls:
l_db = Links(search=s_db, link=url)
l_db.save()
# each Extract object has a single location, get it and associate it to search term
if q.loc != "":
loc_db = Location(city=q.city,state=q.state)
loc_db.save()
# each Extract object has a summary attribute that has all the data, modify the data pool to fit the parameters specified by user
# and store the data in a Results table associated to its Search table
summary = q.pool_summary(pos=self.pos, with_filter=self.with_filter, lower=self.lower, with_bigrams=self.with_bigrams)
data = summary[('Word', 'Word_Count', 'POS_Tag')]
for tup in data:
w = str(tup[0])
c = tup[1]
try:
p = str(tup[2])
except IndexError:
p = ""
r_db = Results(search=s_db,location=loc_db,word=w,count=c,pos=p,is_bigram=self.with_bigrams)
r_db.save()
except:
if q.loc != "":
loc_db = Location(city=q.city,state=q.state)
loc_db.save()
r_db = Results(search=s_db,location=loc_db,word="N/A",count=0,pos="",is_bigram=False)
r_db.save()
def getConcepts(category_id):
"""
This function renders the Concepts Template
It lists concepts linked to the Category the user has clicked on
"""
categories = session.query(Category).filter_by(
id=category_id).one_or_none()
concepts = session.query(Concept).filter_by(category_id=category_id)
links = session.query(Links).filter_by(category_id=category_id)
if request.method == 'POST':
newLink = Links(name=request.form['name'],
link=request.form['link'],
category_id=category_id)
session.add(newLink)
flash('New Link "%s" Successfully Added' % newLink.name)
session.commit()
return redirect(url_for('getConcepts', category_id=category_id))
else:
return render_template('concepts.html',
categories=categories,
concepts=concepts,
links=links,
login_session=login_session)
def add_link():
# getting ip address of the user
try:
ip = request.environ['HTTP_X_FORWARDED_FOR'] # if behind a proxy
except KeyError:
ip = request.environ['REMOTE_ADDR']
ip_addr = IpAddresses.query.filter_by(ipAddress=ip).first()
# if IP is in database then increment the count else add the IP in the database
if ip_addr is not None:
if ip_addr.count < 6:
ip_addr.count += 1
else:
ip_addr = IpAddresses(ip)
ip_addr.count = 1
db.session.add(ip_addr)
db.session.commit()
# only if URL shortener requests are less than 4, run the shortener
if ip_addr.count < 6:
original_url = request.form['url'] # getting url from the form
# adding https:// to the url
if "https://" not in original_url and "http://" not in original_url:
original_url = "https://" + original_url
link = Links(original_url=original_url)
db.session.add(link)
db.session.commit()
# sending success response to javascript file
return jsonify({'result': 'success', 'new_link': link.short_url, 'long_link': link.original_url})
else:
return redirect(url_for('index'))
#ADD STOPS TO DATABASE
with open("stops.csv", "r") as file_to_read:
csv_to_read = csv.reader(file_to_read)
for row in csv_to_read:
current_stop = Stops.query.filter_by(id=row[1]).first()
if current_stop != None:
continue
new_stop = Stops(row[1], row[0], row[3], row[2])
db.session.add(new_stop)
db.session.commit()
#ADD LINKS TO DATABASE
with open("stops.csv", "r") as file_to_read:
csv_to_read = csv.reader(file_to_read)
for row in csv_to_read:
new_link = Links(generate_random_string(10, 20), row[1], row[5])
db.session.add(new_link)
db.session.commit()
#ADD EDGES TO DATABASE
with open("edges.csv", "r") as file_to_read:
csv_to_read = csv.reader(file_to_read)
for row in csv_to_read:
new_edge = Edges(generate_random_string(10, 20), row[0], row[1],
int(row[2]))
db.session.add(new_edge)
db.session.commit()
print("Finished")
def parse_indicators(summary, report_id, queue):
ips = re.findall(IP_REGEX,summary)
domains = re.findall(DOMAIN_REGEX,summary)
attacks = re.findall(ATTACK_REGEX,summary)
sha256 = re.findall(SHA_REGEX,summary)
emails = re.findall(EMAIL_REGEX, summary)
md5 = re.findall(MD5_REGEX, summary)
sha512 = re.findall(SHA512_REGEX, summary)
urls = re.findall(URL_REGEX, summary)
cve = re.findall(CVE_REGEX, summary)
org_id=User.query.filter_by(id=current_user.id).first().organization
#get a list of indicator ids to link to report
indicator_ids={}
for match in ips:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='IP')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in urls:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='URL')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in md5:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='MD5 Hash')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in domains:
#so don't do as a domain if an email address
if match not in emails + ips + urls:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='Domain')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in attacks:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='MITRE ATT&CK Technique')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in sha256:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='SHA256 Hash')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in sha512:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='SHA512 Hash')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in emails:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='Email')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
for match in cve:
match_check = Indicators.query.filter_by(indicator=match).first()
if match_check:
indicator_ids[match_check.id]=match
else:
# Indicator doesn't exist, create new indicator and link
new_indicator = Indicators(indicator=match,indicator_type='CVE')
add_db_entry(new_indicator)
indicator_ids[new_indicator.id]=match
#consolidated all this down here
for id in indicator_ids:
match=indicator_ids[id]
# Kickoff a task to enrich the new indicator
job = queue.enqueue('main.enrich_pipeline', json.dumps({'indicator': str(match),'organization':org_id}))
new_link=Links(indicator=id, report=report_id,kill_chain='Unknown',diamond_model='Unknown',confidence='Low')
if not link_exists(new_link):
add_db_entry(new_link)
def dashboard_with_links(unique_id, selected_link):
# Getting all the info from the database
name = get_user_name(Users.query.filter_by(unique_id=unique_id).first())
user_info = UserDashboard().query.filter_by(unique_id=unique_id).all()
total_visits = 0
max_visits = -1
max_country_name = ""
total_links = 0
labels = []
bar_chart_data = []
count_labels = []
# Getting info for creating chart
for user in user_info:
total_visits += user.visits
total_links += 1
date = user.date_created.strftime("%d")
month = user.date_created.strftime("%b")
out = month + " " + date
count_labels.append(out)
if out not in labels:
labels.append(out)
if user.max_country_visit > max_visits:
max_visits = user.max_country_visit
max_country_name = user.max_country_visit_name
count_labels_dict = Counter(count_labels)
for label in count_labels_dict:
bar_chart_data.append(count_labels_dict[label])
background_color = ['rgba(215, 146, 104, 1)'] * total_links # setting background color
selected_link_info = UserDashboard().query.filter_by(short_url=selected_link).first()
flag = request.args.get("flag")
# if flag is true the it is request for editing the link
if request.method == "POST" and flag == "True":
try:
user = UserDashboard.query.filter_by(short_url=selected_link).first()
update_link = Links.query.filter_by(short_url=selected_link).first()
customized_link = (request.form["short_url_customized"])[25:]
if request.form["short_url_title"] != "":
user.title = request.form["short_url_title"]
user.short_url = customized_link
update_link.short_url = customized_link
db.session.commit()
return redirect(url_for("dashboard",
unique_id=unique_id,
flag=False,
selected_link_info=selected_link_info,
bar_chart_data=bar_chart_data,
labels=json.dumps(labels),
background_color=background_color,
name=name,
total_visits=total_visits,
max_visits=max_visits,
max_country_name=max_country_name))
except BadRequestKeyError:
return redirect(url_for("dashboard",
unique_id=unique_id,
flag=False,
selected_link_info=selected_link_info,
bar_chart_data=bar_chart_data,
labels=json.dumps(labels),
background_color=background_color,
name=name,
total_visits=total_visits,
max_visits=max_visits,
max_country_name=max_country_name))
check_create_button = False
# Creating a new link
if request.method == "POST" and flag != "True":
check_create_button = True
user = UserDashboard()
user.unique_id = unique_id
user.original_url = request.form["long_url"]
user.title = request.form["long_url"]
short_url = generate_short_link()
user.short_url = short_url
redirect_link = Links()
redirect_link.original_url = user.original_url
redirect_link.short_url = user.short_url
db.session.add_all([redirect_link, user])
db.session.commit()
return render_template("dashboard.html",
flag=check_create_button,
short_url=short_url,
selected_link_info=selected_link_info,
bar_chart_data=bar_chart_data,
labels=json.dumps(labels),
background_color=background_color,
unique_id=unique_id,
name=name,
total_visits=total_visits,
max_visits=max_visits,
max_country_name=max_country_name)
return render_template("dashboard.html",
flag=check_create_button,
unique_id=unique_id,
user_info=user_info,
selected_link_info=selected_link_info,
bar_chart_data=bar_chart_data,
labels=json.dumps(labels),
background_color=background_color,
name=name,
total_visits=total_visits,
max_visits=max_visits,
max_country_name=max_country_name)
