def register():
form = RegisterForm()
if form.validate_on_submit():
user = db_session.query(User).filter_by(
user=form.user.data.lower()).first()
if user:
flash('User exists.')
else:
user = User(form.user.data.lower(), form.key.data, form.email.data)
db_session.add(user)
# Set up the settings table when the first user is registered.
if not Setting.query.filter_by(_id=1).first():
settings = Setting('off', 'off', 'off', 'off', 'off', 'off',
'off', 'off', 'off', 'off', 'off', 'off',
'off', 'off', '', '', '', '', '', '', '',
'', '', '', '', '')
db_session.add(settings)
# Commit all database changes once they have been completed
db_session.commit()
login_user(user)
if current_user.is_authenticated:
return redirect(url_for('home'))
return render_template('register.html', form=form, title='Register')
def add_message():
message_sort = Message.id
if request.method == 'POST':
text = request.form['text']
data1 = request.form['data1']
db_session.add(Message(text, data1))
db_session.commit()
return render_template('add_message.html', sorts=sorts, messages=db_session.query(Message).order_by(message_sort))
# return render_template('index.html', sorts=sorts, cars=db_session.query(Cars).order_by(sort))#(desc(sort))
else:
return render_template('add_message.html', sorts=sorts, messages=db_session.query(Message).order_by(message_sort))
def record_to_db(file_name):
myfile = open(file_name, 'w', encoding='cp1251')
for n, l, f, p in zip(names_list, links, photos, list_price):
# db_session.add(1,2,3,4)
#
# libs.car.add_cars(n, l, f, p)
db_session.add(Cars(n, l, f, p))
db_session.commit()
try:
line = '{}{}{}{}{}'.format(n, '\t', l, '\t', f, '\t', str(p))
myfile.write(line + '\n')
except:
pass
def register():
form = RegisterForm()
if form.validate_on_submit():
user = db_session.query(User).filter_by(user=form.user.data.lower()).first()
if user:
flash("User exists.")
else:
user = User(form.user.data.lower(), form.key.data, form.email.data)
db_session.add(user)
# Set up the settings table when the first user is registered.
if not Setting.query.filter_by(_id=1).first():
settings = Setting(
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"off",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
)
db_session.add(settings)
# Commit all database changes once they have been completed
db_session.commit()
login_user(user)
if current_user.is_authenticated:
return redirect(url_for("home"))
return render_template("register.html", form=form, title="Register")
def sign_up():
if request.method == "POST":
login = request.form["login"]
password = request.form["password"]
error = None
if not login:
error = "Username is required."
elif not password:
error = "Password is required."
elif (db_session.query(Users).filter_by(login=login).first()is not None):
error = f"User {login} is already registered."
if error is None:
db_session.add(Users(login, generate_password_hash(password)))
db_session.commit()
return redirect(url_for("index"))
flash(error)
return render_template("sign_up.html")
def newobject():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# Import indicators from Cuckoo for the selected analysis task
if 'type' in records and 'cuckoo' in records['type']:
host_data, dns_data, sha1, firstseen = cuckoo.report_data(records['cuckoo_task_id'])
if host_data and dns_data and sha1 and firstseen:
# Import IP Indicators from Cuckoo Task
for ip in host_data:
ip = ip['ip']
ind = Indicator.query.filter_by(object=ip).first()
if ind is None:
indicator = Indicator(ip.strip(), 'IPv4', firstseen, '', 'Infrastructure', records['campaign'],
'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import Domain Indicators from Cuckoo Task
for dns in dns_data:
ind = Indicator.query.filter_by(object=dns['request']).first()
if ind is None:
indicator = Indicator(dns['request'], 'Domain', firstseen, '', 'Infrastructure',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import File/Hash Indicators from Cuckoo Task
ind = Indicator.query.filter_by(object=sha1).first()
if ind is None:
indicator = Indicator(sha1, 'Hash', firstseen, '', 'Capability',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Redirect to Dashboard after successful import
return redirect(url_for('home'))
else:
errormessage = 'Task is not a file analysis'
return redirect(url_for('import_indicators'))
if 'inputtype' in records:
# Makes sure if you submit an IPv4 indicator, it's an actual IP
# address.
ipregex = re.match(
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', records['inputobject'])
# Convert the inputobject of IP or Domain to a list for Bulk Add functionality.
records['inputobject'] = records['inputobject'].split(',')
for newobject in records['inputobject']:
if records['inputtype'] == "IPv4":
if ipregex:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
ipv4_indicator = Indicator(newobject.strip(), records['inputtype'],
records['inputfirstseen'], records['inputlastseen'],
records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(ipv4_indicator)
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(
('IPv4', 'IPv6', 'Domain', 'Network'))).all()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
errormessage = "Not a valid IP Address."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'],
inputobject=newobject, inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
confidence=records['confidence'], inputcampaign=records['inputcampaign'],
comments=records['comments'], diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
indicator = Indicator(newobject.strip(), records['inputtype'], records['inputfirstseen'],
records['inputlastseen'], records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(indicator)
db_session.commit()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
# TODO: Change 'network' to 'object' in HTML templates to standardize on verbiage
if records['inputtype'] == "IPv4" or records['inputtype'] == "Domain" or records['inputtype'] == "Network"\
or records['inputtype'] == "IPv6":
network = Indicator.query.filter(Indicator.type.in_(('IPv4', 'IPv6', 'Domain', 'Network'))).all()
return render_template('networks.html', network=network)
elif records['diamondmodel'] == "Victim":
victims = Indicator.query.filter(Indicator.diamondmodel == ('Victim')).all()
return render_template('victims.html', network=victims)
elif records['inputtype'] == "Hash":
files = Indicator.query.filter(Indicator.type == ('Hash')).all()
return render_template('files.html', network=files)
else:
threatactors = Indicator.query.filter(Indicator.type == ('Threat Actors')).all()
return render_template('threatactors.html', network=threatactors)
except Exception as e:
return render_template('error.html', error=e)
def newobject():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# Import indicators from Cuckoo for the selected analysis task
if 'type' in records and 'cuckoo' in records['type']:
host_data, dns_data, sha1, firstseen = cuckoo.report_data(records['cuckoo_task_id'])
if host_data and dns_data and sha1 and firstseen:
# Import IP Indicators from Cuckoo Task
for ip in host_data:
ind = Indicator.query.filter_by(object=ip).first()
if ind is None:
indicator = Indicator(ip.strip(), 'IPv4', firstseen, '', 'Infrastructure', records['campaign'],
'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import Domain Indicators from Cuckoo Task
for dns in dns_data:
ind = Indicator.query.filter_by(object=dns['request']).first()
if ind is None:
indicator = Indicator(dns['request'], 'Domain', firstseen, '', 'Infrastructure',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Import File/Hash Indicators from Cuckoo Task
ind = Indicator.query.filter_by(object=sha1).first()
if ind is None:
indicator = Indicator(sha1, 'Hash', firstseen, '', 'Capability',
records['campaign'], 'Low', '', records['tags'], '')
db_session.add(indicator)
db_session.commit()
# Redirect to Dashboard after successful import
return redirect(url_for('home'))
else:
errormessage = 'Task is not a file analysis'
return redirect(url_for('import_indicators'))
if 'inputtype' in records:
# Makes sure if you submit an IPv4 indicator, it's an actual IP
# address.
ipregex = re.match(
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', records['inputobject'])
# Convert the inputobject of IP or Domain to a list for Bulk Add functionality.
records['inputobject'] = records['inputobject'].split(',')
for newobject in records['inputobject']:
if records['inputtype'] == "IPv4":
if ipregex:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
ipv4_indicator = Indicator(newobject.strip(), records['inputtype'],
records['inputfirstseen'], records['inputlastseen'],
records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(ipv4_indicator)
db_session.commit()
network = Indicator.query.filter(Indicator.type.in_(
('IPv4', 'IPv6', 'Domain', 'Network'))).all()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
errormessage = "Not a valid IP Address."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'],
inputobject=newobject, inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
confidence=records['confidence'], inputcampaign=records['inputcampaign'],
comments=records['comments'], diamondmodel=records['diamondmodel'],
tags=records['tags'])
else:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
indicator = Indicator(newobject.strip(), records['inputtype'], records['inputfirstseen'],
records['inputlastseen'], records['diamondmodel'], records['inputcampaign'],
records['confidence'], records['comments'], records['tags'], None)
db_session.add(indicator)
db_session.commit()
else:
errormessage = "Entry already exists in database."
return render_template('newobject.html', errormessage=errormessage,
inputtype=records['inputtype'], inputobject=newobject,
inputfirstseen=records['inputfirstseen'],
inputlastseen=records['inputlastseen'],
inputcampaign=records['inputcampaign'],
comments=records['comments'],
diamondmodel=records['diamondmodel'],
tags=records['tags'])
# TODO: Change 'network' to 'object' in HTML templates to standardize on verbiage
if records['inputtype'] == "IPv4" or records['inputtype'] == "Domain" or records['inputtype'] == "Network"\
or records['inputtype'] == "IPv6":
network = Indicator.query.filter(Indicator.type.in_(('IPv4', 'IPv6', 'Domain', 'Network'))).all()
return render_template('networks.html', network=network)
elif records['diamondmodel'] == "Victim":
victims = Indicator.query.filter(Indicator.diamondmodel == ('Victim')).all()
return render_template('victims.html', network=victims)
elif records['inputtype'] == "Hash":
files = Indicator.query.filter(Indicator.type == ('Hash')).all()
return render_template('files.html', network=files)
else:
threatactors = Indicator.query.filter(Indicator.type == ('Threat Actors')).all()
return render_template('threatactors.html', network=threatactors)
except Exception as e:
return render_template('error.html', error=e)
def work(self, wnum):
self.log.debug(f'{wnum} worker started')
rab_connection = RabbitQueue(CRAWLER_EXCHANGE_NAME, CRAWLER_QUEUE_NAME)
db_connection = DbPg(logger=None)
for raw_msg in rab_connection.get_generator(self.exit_event):
if not raw_msg:
if self.exit_event.wait(2):
break
continue
msg = raw_msg.json()
print(msg)
if 'url' not in msg:
self.log.warning(f'{wnum}: bad task: {msg}')
raw_msg.ack()
continue
print()
if msg['num'] == 0:
msg['url'] = PAGE_URL0
print("0", msg)
try:
request = requests.get(msg['url'], headers=HEADERS).content
soup = BeautifulSoup(request, 'html.parser')
self.log.debug(msg['url'])
time.sleep(1)
names_list = []
container_names = soup.select('div.information-container h2 a')
for name in container_names:
str_name = name.text
print(str_name)
names_list.append(str_name)
links = []
container_links = soup.select('div.information-container h2 a')
for i in container_links:
ii = i['href'].split("&")[0]
full_link = ("https://www.autotrader.co.uk" + ii)
link = full_link.split('?')[0]
links.append(link)
photos = []
container_photo = soup.select(
'figure.listing-main-image a img')
for link_photo in container_photo:
photos.append(link_photo['src'])
list_price = []
container_text = soup.find_all(
"a",
attrs={
"class":
"js-click-handler listing-fpa-link listings-price-link tracking-standard-link"
})
for i in container_text:
pr = i.find_all("div", attrs={"class": "vehicle-price"})
str_price = "".join((re.findall(r'[0-9]{,3},[0-9]{,3}',
str(pr))))
price = 27 * int(str_price.replace(',', ''))
list_price.append(price)
for n, l, f, p in zip(names_list, links, photos, list_price):
db_session.add(Cars(n, l, f, p))
db_session.commit()
data = '{}{}{}{}{}'.format(n, '\t', l, '\t', f, '\t',
str(p))
self.log.debug(data)
db_session.add(Pages(msg['num']))
db_session.commit()
raw_msg.ack()
except Exception as e0:
self.log.exception()(
f'{wnum}: get page error: {e0}') ##self.log.error
raw_msg.nack(requeue=True)
prox = None
if USE_PROXY:
self.proxy_gen.back_proxy(prox, str(e0))
time.sleep(random.randrange(1, 5))
rab_connection.close()
self.log.info(f'{wnum}: worker exit')
def newobject():
try:
something = request.form
imd = ImmutableMultiDict(something)
records = helpers.convert(imd)
# Import indicators from Cuckoo for the selected analysis task
if "type" in records and "cuckoo" in records["type"]:
host_data, dns_data, sha1, firstseen = cuckoo.report_data(records["cuckoo_task_id"])
if host_data and dns_data and sha1 and firstseen:
# Import IP Indicators from Cuckoo Task
for ip in host_data:
ind = Indicator.query.filter_by(object=ip).first()
if ind is None:
indicator = Indicator(
ip.strip(),
"IPv4",
firstseen,
"",
"Infrastructure",
records["campaign"],
"Low",
"",
records["tags"],
"",
)
db_session.add(indicator)
db_session.commit()
# Import Domain Indicators from Cuckoo Task
for dns in dns_data:
ind = Indicator.query.filter_by(object=dns["request"]).first()
if ind is None:
indicator = Indicator(
dns["request"],
"Domain",
firstseen,
"",
"Infrastructure",
records["campaign"],
"Low",
"",
records["tags"],
"",
)
db_session.add(indicator)
db_session.commit()
# Import File/Hash Indicators from Cuckoo Task
ind = Indicator.query.filter_by(object=sha1).first()
if ind is None:
indicator = Indicator(
sha1,
"Hash",
firstseen,
"",
"Capability",
records["campaign"],
"Low",
"",
records["tags"],
"",
)
db_session.add(indicator)
db_session.commit()
# Redirect to Dashboard after successful import
return redirect(url_for("home"))
else:
errormessage = "Task is not a file analysis"
return redirect(url_for("import_indicators"))
if "inputtype" in records:
# Makes sure if you submit an IPv4 indicator, it's an actual IP
# address.
ipregex = re.match(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", records["inputobject"])
# Convert the inputobject of IP or Domain to a list for Bulk Add functionality.
records["inputobject"] = records["inputobject"].split(",")
for newobject in records["inputobject"]:
if records["inputtype"] == "IPv4":
if ipregex:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
ipv4_indicator = Indicator(
newobject.strip(),
records["inputtype"],
records["inputfirstseen"],
records["inputlastseen"],
records["diamondmodel"],
records["inputcampaign"],
records["confidence"],
records["comments"],
records["tags"],
None,
)
db_session.add(ipv4_indicator)
db_session.commit()
network = Indicator.query.filter(
Indicator.type.in_(("IPv4", "IPv6", "Domain", "Network"))
).all()
else:
errormessage = "Entry already exists in database."
return render_template(
"newobject.html",
errormessage=errormessage,
inputtype=records["inputtype"],
inputobject=newobject,
inputfirstseen=records["inputfirstseen"],
inputlastseen=records["inputlastseen"],
inputcampaign=records["inputcampaign"],
comments=records["comments"],
diamondmodel=records["diamondmodel"],
tags=records["tags"],
)
else:
errormessage = "Not a valid IP Address."
return render_template(
"newobject.html",
errormessage=errormessage,
inputtype=records["inputtype"],
inputobject=newobject,
inputfirstseen=records["inputfirstseen"],
inputlastseen=records["inputlastseen"],
confidence=records["confidence"],
inputcampaign=records["inputcampaign"],
comments=records["comments"],
diamondmodel=records["diamondmodel"],
tags=records["tags"],
)
else:
object = Indicator.query.filter_by(object=newobject).first()
if object is None:
indicator = Indicator(
newobject.strip(),
records["inputtype"],
records["inputfirstseen"],
records["inputlastseen"],
records["diamondmodel"],
records["inputcampaign"],
records["confidence"],
records["comments"],
records["tags"],
None,
)
db_session.add(indicator)
db_session.commit()
else:
errormessage = "Entry already exists in database."
return render_template(
"newobject.html",
errormessage=errormessage,
inputtype=records["inputtype"],
inputobject=newobject,
inputfirstseen=records["inputfirstseen"],
inputlastseen=records["inputlastseen"],
inputcampaign=records["inputcampaign"],
comments=records["comments"],
diamondmodel=records["diamondmodel"],
tags=records["tags"],
)
# TODO: Change 'network' to 'object' in HTML templates to standardize on verbiage
if (
records["inputtype"] == "IPv4"
or records["inputtype"] == "Domain"
or records["inputtype"] == "Network"
or records["inputtype"] == "IPv6"
):
network = Indicator.query.filter(Indicator.type.in_(("IPv4", "IPv6", "Domain", "Network"))).all()
return render_template("networks.html", network=network)
elif records["diamondmodel"] == "Victim":
victims = Indicator.query.filter(Indicator.diamondmodel == ("Victim")).all()
return render_template("victims.html", network=victims)
elif records["inputtype"] == "Hash":
files = Indicator.query.filter(Indicator.type == ("Hash")).all()
return render_template("files.html", network=files)
else:
threatactors = Indicator.query.filter(Indicator.type == ("Threat Actors")).all()
return render_template("threatactors.html", network=threatactors)
except Exception as e:
return render_template("error.html", error=e)
def work(self, wnum):
self.log.debug(f'{wnum} worker started')
rab_connection = RabbitQueue(CRAWLER_EXCHANGE_NAME, CRAWLER_QUEUE_NAME)
db_connection = DbPg(logger=None)
# driver, prox = self.init_browser()
for raw_msg in rab_connection.get_generator(self.exit_event):
if not raw_msg:
if self.exit_event.wait(2):
break
continue
msg = raw_msg.json()
print(msg)
if 'url' not in msg:
self.log.warning(f'{wnum}: bad task: {msg}')
raw_msg.ack()
continue
print()
if msg['num'] == 0:
msg['url'] = PAGE_URL0
# msg['url'] = msg['url'].split('?')[0]
print("0",msg)
try:
# driver.get(msg['url'])
request = requests.get(msg['url'], headers=HEADERS).content
soup = BeautifulSoup(request, 'html.parser')
# container = soup.select("li.search-page__result")
self.log.debug(msg['url'])
# self.log.debug(driver.current_url)
time.sleep(1)
names_list = []
container_names = soup.select('div.information-container h2 a')
for name in container_names:
str_name = name.text
#name = str_name.strip()
print(str_name)
names_list.append(str_name)
links = []
container_links = soup.select('div.information-container h2 a')
for i in container_links:
ii = i['href'].split("&")[0]
# ii = i['href']
full_link = ("https://www.autotrader.co.uk" + ii)
link = full_link.split('?')[0]
links.append(link)
#print(link)
photos = []
container_photo = soup.select('figure.listing-main-image a img')
for link_photo in container_photo:
photos.append(link_photo['src'])
#print(link_photo['src'])
list_price = []
container_text = soup.find_all("a", attrs={ "class" : "js-click-handler listing-fpa-link listings-price-link tracking-standard-link"})
for i in container_text:
pr = i.find_all("div", attrs={ "class" : "vehicle-price"})
str_price = "".join((re.findall(r'[0-9]{,3},[0-9]{,3}', str(pr))))
price =27*int(str_price.replace(',', ''))
list_price.append(price)
for n, l, f, p in zip(names_list, links, photos, list_price):
db_session.add(Cars(n, l, f, p))
db_session.commit()
data = '{}{}{}{}{}'.format(n, '\t', l, '\t', f, '\t',str(p))
# parse with selenium
# rows = driver.find_elements_by_css_selector("tr")
# if not rows:
# self.log.debug(f'{wnum}: not rows in table')
# raw_msg.nack(requeue=True)
# break
#
# for row in rows:
# cells = row.find_elements_by_css_selector("td")
# if not cells:
# continue
#
# data = {
# 'img_url': cells[0].find_element_by_css_selector(
# 'img').get_attribute('src'),
# 'country': cells[1].find_element_by_css_selector(
# 'span').get_attribute('title'),
# 'vessel_name': cells[1].text.split('\n')[0],
# 'vessel_type': cells[1].text.split('\n')[1],
# 'year': cells[2].text,
# 'gt': cells[3].text,
# 'dwt': cells[4].text,
# 'sz': cells[5].text
# }
# vlength, vwidth = [int(v.strip()) for v in data['sz'].split('/')]
self.log.debug(data)
# db_connection.insert_ship(car)
# db_connection.exec_query(f'''
# INSERT INTO pages (page_num)
# VALUES({msg['num']})
# ''')
db_session.add(Pages(msg['num']))
db_session.commit()
raw_msg.ack()
except Exception as e0:
self.log.exception()(f'{wnum}: get page error: {e0}')##self.log.error
raw_msg.nack(requeue=True)
prox = None
if USE_PROXY:
self.proxy_gen.back_proxy(prox, str(e0))
# driver.close()
# driver, prox = self.init_browser()
time.sleep(random.randrange(1, 5))
rab_connection.close()
# db_connection.close()
self.log.info(f'{wnum}: worker exit')