def profile():
try:
user = User.query.filter_by(user=current_user.user.lower()).first()
imd = ImmutableMultiDict(request.form)
records = helpers.convert(imd)
if 'currentpw' in records:
if hashlib.md5(records['currentpw'].encode(
'utf-8')).hexdigest() == user.password:
if records['newpw'] == records['newpwvalidation']:
user.password = hashlib.md5(
records['newpw'].encode('utf-8')).hexdigest()
db_session.commit()
errormessage = "Password updated successfully."
return render_template('profile.html',
errormessage=errormessage)
else:
errormessage = "New passwords don't match."
return render_template('profile.html',
errormessage=errormessage)
else:
errormessage = "Current password is incorrect."
return render_template('profile.html',
errormessage=errormessage)
return render_template('profile.html')
except Exception as e:
return render_template('error.html', error=e)
def register():
form = RegisterForm()
if form.validate_on_submit():
user = db_session.query(User).filter_by(
user=form.user.data.lower()).first()
if user:
flash('User exists.')
else:
user = User(form.user.data.lower(), form.key.data, form.email.data)
db_session.add(user)
# Set up the settings table when the first user is registered.
if not Setting.query.filter_by(_id=1).first():
settings = Setting('off', 'off', 'off', 'off', 'off', 'off',
'off', 'off', 'off', 'off', 'off', 'off',
'off', 'off', '', '', '', '', '', '', '',
'', '', '', '', '')
db_session.add(settings)
# Commit all database changes once they have been completed
db_session.commit()
login_user(user)
if current_user.is_authenticated:
return redirect(url_for('home'))
return render_template('register.html', form=form, title='Register')
def addrelationship():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
row = Indicator.query.filter_by(object=records["id"]).first()
row.relationships = records["indicator"]
db_session.commit()
if (
records["type"] == "IPv4"
or records["type"] == "IPv6"
or records["type"] == "Domain"
or records["type"] == "Network"
):
return redirect(url_for("objectsummary", uid=str(records["id"])))
elif records["type"] == "Hash":
return redirect(url_for("filesobject", uid=str(records["id"])))
elif records["type"] == "Entity":
return redirect(url_for("victimobject", uid=str(records["id"])))
elif records["type"] == "Threat Actor":
return redirect(url_for("threatactorobject", uid=str(records["id"])))
except Exception as e:
return render_template("error.html", error=e)
def deletefilesobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
files = Indicator.query.filter_by(type='Hash')
return render_template('victims.html', network=files)
except Exception as e:
return render_template('error.html', error=e)
def deletevictimobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
victims = Indicator.query.filter_by(diamondmodel='Victim')
return render_template('victims.html', network=victims)
except Exception as e:
return render_template('error.html', error=e)
def deletethreatactorobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
threatactors = Indicator.query.filter_by(type='Threat Actor')
return render_template('threatactors.html', network=threatactors)
except Exception as e:
return render_template('error.html', error=e)
def deletethreatactorobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
threatactors = Indicator.query.filter_by(type='Threat Actor')
return render_template('threatactors.html', network=threatactors)
except Exception as e:
return render_template('error.html', error=e)
def deletefilesobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
files = Indicator.query.filter_by(type='Hash')
return render_template('victims.html', network=files)
except Exception as e:
return render_template('error.html', error=e)
def deletenetworkobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(('IPv4', 'IPv6', 'Domain', 'Network'))).all()
return render_template('networks.html', network=network)
except Exception as e:
return render_template('error.html', error=e)
def deletenetworkobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(('IPv4', 'IPv6', 'Domain', 'Network'))).all()
return render_template('networks.html', network=network)
except Exception as e:
return render_template('error.html', error=e)
def deletevictimobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
victims = Indicator.query.filter_by(diamondmodel='Victim')
return render_template('victims.html', network=victims)
except Exception as e:
return render_template('error.html', error=e)
def deletenetworkobject(uid):
try:
Indicator.query.filter_by(object=uid).delete()
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(("IPv4", "IPv6", "Domain", "Network"))).all()
return render_template("networks.html", network=network)
except Exception as e:
return render_template("error.html", error=e)
def apiroll():
print "Rolling API Key"
try:
print "Time to roll the key!"
user = User.query.filter_by(user=current_user.user.lower()).first()
user.apikey = hashlib.md5("{}{}".format(user, str(random.random())).encode('utf-8')).hexdigest()
db_session.commit()
return redirect(url_for('profile'))
except Exception as e:
return render_template('error.html', error=e)
def update_state(self, remind_name):
"""
更新提醒状态
0：表示不在提醒中
1：表示在提醒中
:return:
"""
db_session.query(remindModels).filter(remindModels.remind_name == remind_name).update(
{remindModels.state: 1})
db_session.commit()
def apiroll():
print "Rolling API Key"
try:
print "Time to roll the key!"
user = User.query.filter_by(user=current_user.user.lower()).first()
user.apikey = hashlib.md5("{}{}".format(user, str(random.random())).encode('utf-8')).hexdigest()
db_session.commit()
return redirect(url_for('profile'))
except Exception as e:
return render_template('error.html', error=e)
def add_message():
message_sort = Message.id
if request.method == 'POST':
text = request.form['text']
data1 = request.form['data1']
db_session.add(Message(text, data1))
db_session.commit()
return render_template('add_message.html', sorts=sorts, messages=db_session.query(Message).order_by(message_sort))
# return render_template('index.html', sorts=sorts, cars=db_session.query(Cars).order_by(sort))#(desc(sort))
else:
return render_template('add_message.html', sorts=sorts, messages=db_session.query(Message).order_by(message_sort))
def record_to_db(file_name):
myfile = open(file_name, 'w', encoding='cp1251')
for n, l, f, p in zip(names_list, links, photos, list_price):
# db_session.add(1,2,3,4)
#
# libs.car.add_cars(n, l, f, p)
db_session.add(Cars(n, l, f, p))
db_session.commit()
try:
line = '{}{}{}{}{}'.format(n, '\t', l, '\t', f, '\t', str(p))
myfile.write(line + '\n')
except:
pass
def register():
form = RegisterForm()
if form.validate_on_submit():
user = db_session.query(User).filter_by(user=form.user.data.lower()).first()
if user:
flash("User exists.")
else:
user = User(form.user.data.lower(), form.key.data, form.email.data)
db_session.add(user)
# Set up the settings table when the first user is registered.
if not Setting.query.filter_by(_id=1).first():
settings = Setting(
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
)
db_session.add(settings)
# Commit all database changes once they have been completed
db_session.commit()
login_user(user)
if current_user.is_authenticated:
return redirect(url_for("home"))
return render_template("register.html", form=form, title="Register")
def updateobject():
try:
# Updates entry information
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# taglist = records['tags'].split(",") - Unused
# indicator = Indicator.query.filter_by(object=records['object']).first() - Unused
try:
Indicator.query.filter_by(object=records["object"]).update(records)
except Exception as e:
# SQLAlchemy does not outright support altering tables.
for k, v in records.iteritems():
if Indicator.query.group_by(k).first() is None:
print "ALTER Table"
# db_session.engine.execute("ALTER TABLE indicators ADD COLUMN " + k + " TEXT DEFAULT ''")
db_session.commit()
# db_session.execute('ALTER TABLE indicators ADD COLUMN')
# con = helpers.db_connection()
# with con:
# cur = con.cursor()
# cur.execute(
# "ALTER TABLE indicators ADD COLUMN " + t + " TEXT DEFAULT ''")
# cur.execute("UPDATE indicators SET " + t + "= '" + records[
# t] + "' WHERE id = '" + records['id'] + "'")
if (
records["type"] == "IPv4"
or records["type"] == "IPv6"
or records["type"] == "Domain"
or records["type"] == "Network"
):
return redirect(url_for("objectsummary", uid=str(records["object"])))
elif records["type"] == "Hash":
return redirect(url_for("filesobject", uid=str(records["object"])))
elif records["type"] == "Entity":
return redirect(url_for("victimobject", uid=str(records["object"])))
elif records["type"] == "Threat Actor":
return redirect(url_for("threatactorobject", uid=str(records["object"])))
except Exception as e:
return render_template("error.html", error=e)
def updateobject():
try:
# Updates entry information
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# taglist = records['tags'].split(",") - Unused
# indicator = Indicator.query.filter_by(object=records['object']).first() - Unused
try:
Indicator.query.filter_by(object=records['object']).update(records)
except Exception as e:
# SQLAlchemy does not outright support altering tables.
for k, v in records.iteritems():
if Indicator.query.group_by(k).first() is None:
print 'ALTER Table'
# db_session.engine.execute("ALTER TABLE indicators ADD COLUMN " + k + " TEXT DEFAULT ''")
db_session.commit()
# db_session.execute('ALTER TABLE indicators ADD COLUMN')
# con = helpers.db_connection()
# with con:
# cur = con.cursor()
# cur.execute(
# "ALTER TABLE indicators ADD COLUMN " + t + " TEXT DEFAULT ''")
# cur.execute("UPDATE indicators SET " + t + "= '" + records[
# t] + "' WHERE id = '" + records['id'] + "'")
if records['type'] == "IPv4" or records['type'] == "IPv6" or records['type'] == "Domain" or \
records['type'] == "Network":
return redirect(
url_for('objectsummary', uid=str(records['object'])))
elif records['type'] == "Hash":
return redirect(url_for('filesobject', uid=str(records['object'])))
elif records['type'] == "Entity":
return redirect(url_for('victimobject',
uid=str(records['object'])))
elif records['type'] == "Threat Actor":
return redirect(
url_for('threatactorobject', uid=str(records['object'])))
except Exception as e:
return render_template('error.html', error=e)
def sign_up():
if request.method == "POST":
login = request.form["login"]
password = request.form["password"]
error = None
if not login:
error = "Username is required."
elif not password:
error = "Password is required."
elif (db_session.query(Users).filter_by(login=login).first()is not None):
error = f"User {login} is already registered."
if error is None:
db_session.add(Users(login, generate_password_hash(password)))
db_session.commit()
return redirect(url_for("index"))
flash(error)
return render_template("sign_up.html")
def addrelationship():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = libs.helpers.convert(imd)
row = Indicator.query.filter_by(object=records['id']).first()
row.relationships = records['indicator']
db_session.commit()
if records['type'] == "IPv4" or records['type'] == "IPv6" or records['type'] == "Domain" or \
records['type'] == "Network":
return redirect(url_for('objectsummary', uid=str(records['id'])))
elif records['type'] == "Hash":
return redirect(url_for('filesobject', uid=str(records['id'])))
elif records['type'] == "Entity":
return redirect(url_for('victimobject', uid=str(records['id'])))
elif records['type'] == "Threat Actor":
return redirect(url_for('threatactorobject', uid=str(records['id'])))
except Exception as e:
return render_template('error.html', error=e)
def profile():
try:
user = User.query.filter_by(user=current_user.user.lower()).first()
imd = ImmutableMultiDict(request.form)
records = helpers.convert(imd)
if 'currentpw' in records:
if hashlib.md5(records['currentpw'].encode('utf-8')).hexdigest() == user.password:
if records['newpw'] == records['newpwvalidation']:
user.password = hashlib.md5(records['newpw'].encode('utf-8')).hexdigest()
db_session.commit()
errormessage = "Password updated successfully."
return render_template('profile.html', errormessage=errormessage)
else:
errormessage = "New passwords don't match."
return render_template('profile.html', errormessage=errormessage)
else:
errormessage = "Current password is incorrect."
return render_template('profile.html', errormessage=errormessage)
return render_template('profile.html')
except Exception as e:
return render_template('error.html', error=e)
def work(self, wnum):
self.log.debug(f'{wnum} worker started')
rab_connection = RabbitQueue(CRAWLER_EXCHANGE_NAME, CRAWLER_QUEUE_NAME)
db_connection = DbPg(logger=None)
# driver, prox = self.init_browser()
for raw_msg in rab_connection.get_generator(self.exit_event):
if not raw_msg:
if self.exit_event.wait(2):
break
continue
msg = raw_msg.json()
print(msg)
if 'url' not in msg:
self.log.warning(f'{wnum}: bad task: {msg}')
raw_msg.ack()
continue
print()
if msg['num'] == 0:
msg['url'] = PAGE_URL0
# msg['url'] = msg['url'].split('?')[0]
print("0",msg)
try:
# driver.get(msg['url'])
request = requests.get(msg['url'], headers=HEADERS).content
soup = BeautifulSoup(request, 'html.parser')
# container = soup.select("li.search-page__result")
self.log.debug(msg['url'])
# self.log.debug(driver.current_url)
time.sleep(1)
names_list = []
container_names = soup.select('div.information-container h2 a')
for name in container_names:
str_name = name.text
#name = str_name.strip()
print(str_name)
names_list.append(str_name)
links = []
container_links = soup.select('div.information-container h2 a')
for i in container_links:
ii = i['href'].split("&")[0]
# ii = i['href']
full_link = ("https://www.autotrader.co.uk" + ii)
link = full_link.split('?')[0]
links.append(link)
#print(link)
photos = []
container_photo = soup.select('figure.listing-main-image a img')
for link_photo in container_photo:
photos.append(link_photo['src'])
#print(link_photo['src'])
list_price = []
container_text = soup.find_all("a", attrs={ "class" : "js-click-handler listing-fpa-link listings-price-link tracking-standard-link"})
for i in container_text:
pr = i.find_all("div", attrs={ "class" : "vehicle-price"})
str_price = "".join((re.findall(r'[0-9]{,3},[0-9]{,3}', str(pr))))
price =27*int(str_price.replace(',', ''))
list_price.append(price)
for n, l, f, p in zip(names_list, links, photos, list_price):
db_session.add(Cars(n, l, f, p))
db_session.commit()
data = '{}{}{}{}{}'.format(n, '\t', l, '\t', f, '\t',str(p))
# parse with selenium
# rows = driver.find_elements_by_css_selector("tr")
# if not rows:
# self.log.debug(f'{wnum}: not rows in table')
# raw_msg.nack(requeue=True)
# break
#
# for row in rows:
# cells = row.find_elements_by_css_selector("td")
# if not cells:
# continue
#
# data = {
# 'img_url': cells[0].find_element_by_css_selector(
# 'img').get_attribute('src'),
# 'country': cells[1].find_element_by_css_selector(
# 'span').get_attribute('title'),
# 'vessel_name': cells[1].text.split('\n')[0],
# 'vessel_type': cells[1].text.split('\n')[1],
# 'year': cells[2].text,
# 'gt': cells[3].text,
# 'dwt': cells[4].text,
# 'sz': cells[5].text
# }
# vlength, vwidth = [int(v.strip()) for v in data['sz'].split('/')]
self.log.debug(data)
# db_connection.insert_ship(car)
# db_connection.exec_query(f'''
# INSERT INTO pages (page_num)
# VALUES({msg['num']})
# ''')
db_session.add(Pages(msg['num']))
db_session.commit()
raw_msg.ack()
except Exception as e0:
self.log.exception()(f'{wnum}: get page error: {e0}')##self.log.error
raw_msg.nack(requeue=True)
prox = None
if USE_PROXY:
self.proxy_gen.back_proxy(prox, str(e0))
# driver.close()
# driver, prox = self.init_browser()
time.sleep(random.randrange(1, 5))
rab_connection.close()
# db_connection.close()
self.log.info(f'{wnum}: worker exit')
def work(self, wnum):
self.log.debug(f'{wnum} worker started')
rab_connection = RabbitQueue(CRAWLER_EXCHANGE_NAME, CRAWLER_QUEUE_NAME)
db_connection = DbPg(logger=None)
for raw_msg in rab_connection.get_generator(self.exit_event):
if not raw_msg:
if self.exit_event.wait(2):
break
continue
msg = raw_msg.json()
print(msg)
if 'url' not in msg:
self.log.warning(f'{wnum}: bad task: {msg}')
raw_msg.ack()
continue
print()
if msg['num'] == 0:
msg['url'] = PAGE_URL0
print("0", msg)
try:
request = requests.get(msg['url'], headers=HEADERS).content
soup = BeautifulSoup(request, 'html.parser')
self.log.debug(msg['url'])
time.sleep(1)
names_list = []
container_names = soup.select('div.information-container h2 a')
for name in container_names:
str_name = name.text
print(str_name)
names_list.append(str_name)
links = []
container_links = soup.select('div.information-container h2 a')
for i in container_links:
ii = i['href'].split("&")[0]
full_link = ("https://www.autotrader.co.uk" + ii)
link = full_link.split('?')[0]
links.append(link)
photos = []
container_photo = soup.select(
'figure.listing-main-image a img')
for link_photo in container_photo:
photos.append(link_photo['src'])
list_price = []
container_text = soup.find_all(
"a",
attrs={
"class":
"js-click-handler listing-fpa-link listings-price-link tracking-standard-link"
})
for i in container_text:
pr = i.find_all("div", attrs={"class": "vehicle-price"})
str_price = "".join((re.findall(r'[0-9]{,3},[0-9]{,3}',
str(pr))))
price = 27 * int(str_price.replace(',', ''))
list_price.append(price)
for n, l, f, p in zip(names_list, links, photos, list_price):
db_session.add(Cars(n, l, f, p))
db_session.commit()
data = '{}{}{}{}{}'.format(n, '\t', l, '\t', f, '\t',
str(p))
self.log.debug(data)
db_session.add(Pages(msg['num']))
db_session.commit()
raw_msg.ack()
except Exception as e0:
self.log.exception()(
f'{wnum}: get page error: {e0}') ##self.log.error
raw_msg.nack(requeue=True)
prox = None
if USE_PROXY:
self.proxy_gen.back_proxy(prox, str(e0))
time.sleep(random.randrange(1, 5))
rab_connection.close()
self.log.info(f'{wnum}: worker exit')
def delete_Message():
try:
num_rows_deleted = db_session.query(Message).delete()
db_session.commit()
except:
db_session.rollback()
def updatesettings():
try:
something = request.form
imd = ImmutableMultiDict(something)
newdict = helpers.convert(imd)
# Query the first set of settings, could query custom settings for individual users
settings = Setting.query.filter_by(_id=1).first()
# Make sure we're updating the settings instead of overwriting them
if 'threatcrowd' in newdict.keys():
settings.threatcrowd = 'on'
else:
settings.threatcrowd = 'off'
for pt_type in ['pt_pdns', 'pt_whois', 'pt_pssl', 'pt_host_attr']:
auth = [newdict['pt_username'], newdict['pt_api_key']]
if pt_type in newdict.keys() and ('' not in auth):
setattr(settings, pt_type, 'on')
else:
setattr(settings, pt_type, 'off')
if 'cuckoo' in newdict.keys():
settings.cuckoo = 'on'
else:
settings.cuckoo = 'off'
if 'vtinfo' in newdict.keys() and newdict['apikey'] is not '':
settings.vtinfo = 'on'
else:
settings.vtinfo = 'off'
if 'vtfile' in newdict.keys() and newdict['apikey'] is not '':
settings.vtfile = 'on'
else:
settings.vtfile = 'off'
if 'circlinfo' in newdict.keys() and newdict['circlusername'] is not '':
settings.circlinfo = 'on'
else:
settings.circlinfo = 'off'
if 'circlssl' in newdict.keys() and newdict['circlusername'] is not '':
settings.circlssl = 'on'
else:
settings.circlssl = 'off'
if 'whoisinfo' in newdict.keys():
settings.whoisinfo = 'on'
else:
settings.whoisinfo = 'off'
if 'farsightinfo' in newdict.keys() and newdict['farsightkey'] is not '':
settings.farsightinfo = 'on'
else:
settings.farsightinfo = 'off'
if 'shodaninfo' in newdict.keys() and newdict['shodankey'] is not '':
settings.shodaninfo = 'on'
else:
settings.shodaninfo = 'off'
if 'odnsinfo' in newdict.keys() and newdict['odnskey'] is not '':
settings.odnsinfo = 'on'
else:
settings.odnsinfo = 'off'
settings.farsightkey = newdict['farsightkey']
settings.apikey = newdict['apikey']
settings.odnskey = newdict['odnskey']
settings.httpproxy = newdict['httpproxy']
settings.httpsproxy = newdict['httpsproxy']
settings.cuckoohost = newdict['cuckoohost']
settings.cuckooapiport = newdict['cuckooapiport']
settings.circlusername = newdict['circlusername']
settings.circlpassword = newdict['circlpassword']
settings.pt_username = newdict['pt_username']
settings.pt_api_key = newdict['pt_api_key']
settings.shodankey = newdict['shodankey']
db_session.commit()
settings = Setting.query.first()
return render_template('settings.html', records=settings)
except Exception as e:
return render_template('error.html', error=e)
def newobject():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# Import indicators from Cuckoo for the selected analysis task
if "type" in records and "cuckoo" in records["type"]:
host_data, dns_data, sha1, firstseen = cuckoo.report_data(records["cuckoo_task_id"])
if host_data and dns_data and sha1 and firstseen:
# Import IP Indicators from Cuckoo Task
for ip in host_data:
ind = Indicator.query.filter_by(object=ip).first()
if ind is None:
indicator = Indicator(
ip.strip(),
"IPv4",
firstseen,
"",
"Infrastructure",
records["campaign"],
"Low",
"",
records["tags"],
"",
)
db_session.add(indicator)
db_session.commit()
# Import Domain Indicators from Cuckoo Task
for dns in dns_data:
ind = Indicator.query.filter_by(object=dns["request"]).first()
if ind is None:
indicator = Indicator(
dns["request"],
"Domain",
firstseen,
"",
"Infrastructure",
records["campaign"],
"Low",
"",
records["tags"],
"",
)
db_session.add(indicator)
db_session.commit()
# Import File/Hash Indicators from Cuckoo Task
ind = Indicator.query.filter_by(object=sha1).first()
if ind is None:
indicator = Indicator(
sha1,
"Hash",
firstseen,
"",
"Capability",
records["campaign"],
"Low",
"",
records["tags"],
"",
)
db_session.add(indicator)
db_session.commit()
# Redirect to Dashboard after successful import
return redirect(url_for("home"))
else:
errormessage = "Task is not a file analysis"
return redirect(url_for("import_indicators"))
if "inputtype" in records:
# Makes sure if you submit an IPv4 indicator, it's an actual IP
# address.
ipregex = re.match(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", records["inputobject"])
# Convert the inputobject of IP or Domain to a list for Bulk Add functionality.
records["inputobject"] = records["inputobject"].split(",")
for newobject in records["inputobject"]:
if records["inputtype"] == "IPv4":
if ipregex:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
ipv4_indicator = Indicator(
newobject.strip(),
records["inputtype"],
records["inputfirstseen"],
records["inputlastseen"],
records["diamondmodel"],
records["inputcampaign"],
records["confidence"],
records["comments"],
records["tags"],
None,
)
db_session.add(ipv4_indicator)
db_session.commit()
network = Indicator.query.filter(
Indicator.type.in_(("IPv4", "IPv6", "Domain", "Network"))
).all()
else:
errormessage = "Entry already exists in database."
return render_template(
"newobject.html",
errormessage=errormessage,
inputtype=records["inputtype"],
inputobject=newobject,
inputfirstseen=records["inputfirstseen"],
inputlastseen=records["inputlastseen"],
inputcampaign=records["inputcampaign"],
comments=records["comments"],
diamondmodel=records["diamondmodel"],
tags=records["tags"],
)
else:
errormessage = "Not a valid IP Address."
return render_template(
"newobject.html",
errormessage=errormessage,
inputtype=records["inputtype"],
inputobject=newobject,
inputfirstseen=records["inputfirstseen"],
inputlastseen=records["inputlastseen"],
confidence=records["confidence"],
inputcampaign=records["inputcampaign"],
comments=records["comments"],
diamondmodel=records["diamondmodel"],
tags=records["tags"],
)
else:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
indicator = Indicator(
newobject.strip(),
records["inputtype"],
records["inputfirstseen"],
records["inputlastseen"],
records["diamondmodel"],
records["inputcampaign"],
records["confidence"],
records["comments"],
records["tags"],
None,
)
db_session.add(indicator)
db_session.commit()
else:
errormessage = "Entry already exists in database."
return render_template(
"newobject.html",
errormessage=errormessage,
inputtype=records["inputtype"],
inputobject=newobject,
inputfirstseen=records["inputfirstseen"],
inputlastseen=records["inputlastseen"],
inputcampaign=records["inputcampaign"],
comments=records["comments"],
diamondmodel=records["diamondmodel"],
tags=records["tags"],
)
# TODO: Change 'network' to 'object' in HTML templates to standardize on verbiage
if (
records["inputtype"] == "IPv4"
or records["inputtype"] == "Domain"
or records["inputtype"] == "Network"
or records["inputtype"] == "IPv6"
):
network = Indicator.query.filter(Indicator.type.in_(("IPv4", "IPv6", "Domain", "Network"))).all()
return render_template("networks.html", network=network)
elif records["diamondmodel"] == "Victim":
victims = Indicator.query.filter(Indicator.diamondmodel == ("Victim")).all()
return render_template("victims.html", network=victims)
elif records["inputtype"] == "Hash":
files = Indicator.query.filter(Indicator.type == ("Hash")).all()
return render_template("files.html", network=files)
else:
threatactors = Indicator.query.filter(Indicator.type == ("Threat Actors")).all()
return render_template("threatactors.html", network=threatactors)
except Exception as e:
return render_template("error.html", error=e)
def delete_Pages():
try:
num_rows_deleted = db_session.query(Pages).delete()
db_session.commit()
except:
db_session.rollback()
def updatesettings():
try:
something = request.form
imd = ImmutableMultiDict(something)
newdict = helpers.convert(imd)
# Query the first set of settings, could query custom settings for individual users
settings = Setting.query.filter_by(_id=1).first()
# Make sure we're updating the settings instead of overwriting them
if "threatcrowd" in newdict.keys():
settings.threatcrowd = "on"
else:
settings.threatcrowd = "off"
for pt_type in ["pt_pdns", "pt_whois", "pt_pssl", "pt_host_attr"]:
auth = [newdict["pt_username"], newdict["pt_api_key"]]
if pt_type in newdict.keys() and ("" not in auth):
setattr(settings, pt_type, "on")
else:
setattr(settings, pt_type, "off")
if "cuckoo" in newdict.keys():
settings.cuckoo = "on"
else:
settings.cuckoo = "off"
if "vtinfo" in newdict.keys() and newdict["apikey"] is not "":
settings.vtinfo = "on"
else:
settings.vtinfo = "off"
if "vtfile" in newdict.keys() and newdict["apikey"] is not "":
settings.vtfile = "on"
else:
settings.vtfile = "off"
if "circlinfo" in newdict.keys() and newdict["circlusername"] is not "":
settings.circlinfo = "on"
else:
settings.circlinfo = "off"
if "circlssl" in newdict.keys() and newdict["circlusername"] is not "":
settings.circlssl = "on"
else:
settings.circlssl = "off"
if "whoisinfo" in newdict.keys():
settings.whoisinfo = "on"
else:
settings.whoisinfo = "off"
if "farsightinfo" in newdict.keys() and newdict["farsightkey"] is not "":
settings.farsightinfo = "on"
else:
settings.farsightinfo = "off"
if "shodaninfo" in newdict.keys() and newdict["shodankey"] is not "":
settings.shodaninfo = "on"
else:
settings.shodaninfo = "off"
if "odnsinfo" in newdict.keys() and newdict["odnskey"] is not "":
settings.odnsinfo = "on"
else:
settings.odnsinfo = "off"
settings.farsightkey = newdict["farsightkey"]
settings.apikey = newdict["apikey"]
settings.odnskey = newdict["odnskey"]
settings.httpproxy = newdict["httpproxy"]
settings.httpsproxy = newdict["httpsproxy"]
settings.cuckoohost = newdict["cuckoohost"]
settings.cuckooapiport = newdict["cuckooapiport"]
settings.circlusername = newdict["circlusername"]
settings.circlpassword = newdict["circlpassword"]
settings.pt_username = newdict["pt_username"]
settings.pt_api_key = newdict["pt_api_key"]
settings.shodankey = newdict["shodankey"]
db_session.commit()
settings = Setting.query.first()
return render_template("settings.html", records=settings)
except Exception as e:
return render_template("error.html", error=e)
def newobject():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# Import indicators from Cuckoo for the selected analysis task
if 'type' in records and 'cuckoo' in records['type']:
host_data, dns_data, sha1, firstseen = cuckoo.report_data(records['cuckoo_task_id'])
if host_data and dns_data and sha1 and firstseen:
# Import IP Indicators from Cuckoo Task
for ip in host_data:
ip = ip['ip']
ind = Indicator.query.filter_by(object=ip).first()
if ind is None:
indicator = Indicator(ip.strip(), 'IPv4', firstseen, '', 'Infrastructure', records['campaign'],
'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import Domain Indicators from Cuckoo Task
for dns in dns_data:
ind = Indicator.query.filter_by(object=dns['request']).first()
if ind is None:
indicator = Indicator(dns['request'], 'Domain', firstseen, '', 'Infrastructure',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import File/Hash Indicators from Cuckoo Task
ind = Indicator.query.filter_by(object=sha1).first()
if ind is None:
indicator = Indicator(sha1, 'Hash', firstseen, '', 'Capability',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Redirect to Dashboard after successful import
return redirect(url_for('home'))
else:
errormessage = 'Task is not a file analysis'
return redirect(url_for('import_indicators'))
if 'inputtype' in records:
# Makes sure if you submit an IPv4 indicator, it's an actual IP
# address.
ipregex = re.match(
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', records['inputobject'])
# Convert the inputobject of IP or Domain to a list for Bulk Add functionality.
records['inputobject'] = records['inputobject'].split(',')
for newobject in records['inputobject']:
if records['inputtype'] == "IPv4":
if ipregex:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
ipv4_indicator = Indicator(newobject.strip(), records['inputtype'],
records['inputfirstseen'], records['inputlastseen'],
records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(ipv4_indicator)
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(
('IPv4', 'IPv6', 'Domain', 'Network'))).all()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
errormessage = "Not a valid IP Address."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'],
inputobject=newobject, inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
confidence=records['confidence'], inputcampaign=records['inputcampaign'],
comments=records['comments'], diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
indicator = Indicator(newobject.strip(), records['inputtype'], records['inputfirstseen'],
records['inputlastseen'], records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(indicator)
db_session.commit()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
# TODO: Change 'network' to 'object' in HTML templates to standardize on verbiage
if records['inputtype'] == "IPv4" or records['inputtype'] == "Domain" or records['inputtype'] == "Network"\
or records['inputtype'] == "IPv6":
network = Indicator.query.filter(Indicator.type.in_(('IPv4', 'IPv6', 'Domain', 'Network'))).all()
return render_template('networks.html', network=network)
elif records['diamondmodel'] == "Victim":
victims = Indicator.query.filter(Indicator.diamondmodel == ('Victim')).all()
return render_template('victims.html', network=victims)
elif records['inputtype'] == "Hash":
files = Indicator.query.filter(Indicator.type == ('Hash')).all()
return render_template('files.html', network=files)
else:
threatactors = Indicator.query.filter(Indicator.type == ('Threat Actors')).all()
return render_template('threatactors.html', network=threatactors)
except Exception as e:
return render_template('error.html', error=e)
def delete_Users():
try:
num_rows_deleted = db_session.query(Cars).delete()
db_session.commit()
except:
db_session.rollback()
def updatesettings():
try:
something = request.form
imd = ImmutableMultiDict(something)
newdict = helpers.convert(imd)
# Query the first set of settings, could query custom settings for individual users
settings = Setting.query.filter_by(_id=1).first()
# Make sure we're updating the settings instead of overwriting them
if 'threatcrowd' in newdict.keys():
settings.threatcrowd = 'on'
else:
settings.threatcrowd = 'off'
for pt_type in ['pt_pdns', 'pt_whois', 'pt_pssl', 'pt_host_attr']:
auth = [newdict['pt_username'], newdict['pt_api_key']]
if pt_type in newdict.keys() and ('' not in auth):
setattr(settings, pt_type, 'on')
else:
setattr(settings, pt_type, 'off')
if 'cuckoo' in newdict.keys():
settings.cuckoo = 'on'
else:
settings.cuckoo = 'off'
if 'vtinfo' in newdict.keys() and newdict['apikey'] is not '':
settings.vtinfo = 'on'
else:
settings.vtinfo = 'off'
if 'vtfile' in newdict.keys() and newdict['apikey'] is not '':
settings.vtfile = 'on'
else:
settings.vtfile = 'off'
if 'circlinfo' in newdict.keys() and newdict['circlusername'] is not '':
settings.circlinfo = 'on'
else:
settings.circlinfo = 'off'
if 'circlssl' in newdict.keys() and newdict['circlusername'] is not '':
settings.circlssl = 'on'
else:
settings.circlssl = 'off'
if 'whoisinfo' in newdict.keys():
settings.whoisinfo = 'on'
else:
settings.whoisinfo = 'off'
if 'farsightinfo' in newdict.keys() and newdict['farsightkey'] is not '':
settings.farsightinfo = 'on'
else:
settings.farsightinfo = 'off'
if 'shodaninfo' in newdict.keys() and newdict['shodankey'] is not '':
settings.shodaninfo = 'on'
else:
settings.shodaninfo = 'off'
if 'odnsinfo' in newdict.keys() and newdict['odnskey'] is not '':
settings.odnsinfo = 'on'
else:
settings.odnsinfo = 'off'
settings.farsightkey = newdict['farsightkey']
settings.apikey = newdict['apikey']
settings.odnskey = newdict['odnskey']
settings.httpproxy = newdict['httpproxy']
settings.httpsproxy = newdict['httpsproxy']
settings.cuckoohost = newdict['cuckoohost']
settings.cuckooapiport = newdict['cuckooapiport']
settings.circlusername = newdict['circlusername']
settings.circlpassword = newdict['circlpassword']
settings.pt_username = newdict['pt_username']
settings.pt_api_key = newdict['pt_api_key']
settings.shodankey = newdict['shodankey']
db_session.commit()
settings = Setting.query.first()
return render_template('settings.html', records=settings)
except Exception as e:
return render_template('error.html', error=e)
def newobject():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# Import indicators from Cuckoo for the selected analysis task
if 'type' in records and 'cuckoo' in records['type']:
host_data, dns_data, sha1, firstseen = cuckoo.report_data(records['cuckoo_task_id'])
if host_data and dns_data and sha1 and firstseen:
# Import IP Indicators from Cuckoo Task
for ip in host_data:
ind = Indicator.query.filter_by(object=ip).first()
if ind is None:
indicator = Indicator(ip.strip(), 'IPv4', firstseen, '', 'Infrastructure', records['campaign'],
'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import Domain Indicators from Cuckoo Task
for dns in dns_data:
ind = Indicator.query.filter_by(object=dns['request']).first()
if ind is None:
indicator = Indicator(dns['request'], 'Domain', firstseen, '', 'Infrastructure',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import File/Hash Indicators from Cuckoo Task
ind = Indicator.query.filter_by(object=sha1).first()
if ind is None:
indicator = Indicator(sha1, 'Hash', firstseen, '', 'Capability',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Redirect to Dashboard after successful import
return redirect(url_for('home'))
else:
errormessage = 'Task is not a file analysis'
return redirect(url_for('import_indicators'))
if 'inputtype' in records:
# Makes sure if you submit an IPv4 indicator, it's an actual IP
# address.
ipregex = re.match(
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', records['inputobject'])
# Convert the inputobject of IP or Domain to a list for Bulk Add functionality.
records['inputobject'] = records['inputobject'].split(',')
for newobject in records['inputobject']:
if records['inputtype'] == "IPv4":
if ipregex:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
ipv4_indicator = Indicator(newobject.strip(), records['inputtype'],
records['inputfirstseen'], records['inputlastseen'],
records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(ipv4_indicator)
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(
('IPv4', 'IPv6', 'Domain', 'Network'))).all()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
errormessage = "Not a valid IP Address."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'],
inputobject=newobject, inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
confidence=records['confidence'], inputcampaign=records['inputcampaign'],
comments=records['comments'], diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
indicator = Indicator(newobject.strip(), records['inputtype'], records['inputfirstseen'],
records['inputlastseen'], records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(indicator)
db_session.commit()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
# TODO: Change 'network' to 'object' in HTML templates to standardize on verbiage
if records['inputtype'] == "IPv4" or records['inputtype'] == "Domain" or records['inputtype'] == "Network"\
or records['inputtype'] == "IPv6":
network = Indicator.query.filter(Indicator.type.in_(('IPv4', 'IPv6', 'Domain', 'Network'))).all()
return render_template('networks.html', network=network)
elif records['diamondmodel'] == "Victim":
victims = Indicator.query.filter(Indicator.diamondmodel == ('Victim')).all()
return render_template('victims.html', network=victims)
elif records['inputtype'] == "Hash":
files = Indicator.query.filter(Indicator.type == ('Hash')).all()
return render_template('files.html', network=files)
else:
threatactors = Indicator.query.filter(Indicator.type == ('Threat Actors')).all()
return render_template('threatactors.html', network=threatactors)
except Exception as e:
return render_template('error.html', error=e)
