def get_intel_feed():
options = request.args.to_dict()
limit = int(options.get('limit', '1000'))
hours_ago = int(options.get('hours_ago', '4'))
extra = dict(options)
for name in ('hours_ago', 'limit', 'api_key',):
if name in extra:
del extra[name]
for name in options.keys():
if name not in ('hours_ago', 'limit',):
del options[name]
extra['ne__protocol'] = 'pcap'
results = Clio().session._tops(['source_ip', 'honeypot', 'protocol', 'destination_port'], top=limit, hours_ago=hours_ago, **extra)
results = [r for r in results if r['protocol'] != 'ftpdatalisten']
cache = {}
for r in results:
source_ip = r['source_ip']
if source_ip not in cache:
# TODO: may want to make one big query to mongo here...
cache[source_ip] = [m.to_dict() for m in Clio().metadata.get(ip=r['source_ip'], honeypot='p0f')]
r['meta'] = cache[source_ip]
return {
'data':results,
'meta':{
'size': len(results),
'query': 'intel_feed',
'options': options
}
}
def graph_combos():
clio=Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle,show_x_labels=True, config=PYGAL_CONFIG)
bar_chart.title = "Kippo/Cowrie Top User/Passwords"
clio=Clio()
top_combos =clio.hpfeed.count_combos(get_credentials_payloads(clio))
for combo in top_combos:
bar_chart.add(combo[0],[{'label':str(combo[0]),'xlink':'','value':combo[1]}])
return bar_chart.render_response()
def graph_top_attackers():
clio=Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle,show_x_labels=True, config=PYGAL_CONFIG)
bar_chart.title = "Kippo/Cowrie Top Attackers"
clio=Clio()
top_attackers = top_kippo_cowrie_attackers(clio)
print top_attackers
for attacker in top_attackers:
bar_chart.add(str(attacker['source_ip']), attacker['count'])
return bar_chart.render_response()
def graph_combos():
clio = Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle, show_x_labels=True, config=PYGAL_CONFIG)
bar_chart.title = "Kippo/Cowrie Top User/Passwords"
clio = Clio()
top_combos = clio.hpfeed.count_combos(get_credentials_payloads(clio))
for combo_list in top_combos:
user,password = combo_list
user = remove_control_characters(user)
bar_chart.add(user,[{'label':user,'xlink': '', 'value':password}])
return bar_chart.render_response()
def graph_passwords():
clio = Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle, show_x_labels=True, config=PYGAL_CONFIG)
bar_chart.title = "Kippo/Cowrie Top Passwords"
clio = Clio()
top_passwords = clio.hpfeed.count_passwords(get_credentials_payloads(clio))
for password_data in top_passwords:
password,count = password_data
password = remove_control_characters(password)
bar_chart.add(password, [{'label': password, 'xlink': '', 'value':count}])
return bar_chart.render_response()
def graph_top_attackers():
clio = Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle,
show_x_labels=True,
config=PYGAL_CONFIG)
bar_chart.title = "Kippo Top Attackers"
clio = Clio()
top_attackers = clio.session._tops('source_ip', 10, honeypot='kippo')
print top_attackers
for attacker in top_attackers:
bar_chart.add(str(attacker['source_ip']), int(attacker['count']))
return bar_chart.render_response()
def top_attackers():
options = request.args.to_dict()
limit = int(options.get('limit', '1000'))
hours_ago = int(options.get('hours_ago', '4'))
extra = dict(options)
for name in (
'hours_ago',
'limit',
'api_key',
):
if name in extra:
del extra[name]
for name in options.keys():
if name not in (
'hours_ago',
'limit',
):
del options[name]
results = Clio().session._tops(['source_ip', 'honeypot'],
top=limit,
hours_ago=hours_ago,
**extra)
return jsonify(data=results,
meta={
'size': len(results),
'query': 'top_attackers',
'options': options
})
def credential_list_csv():
fieldnames = ['username', 'password', 'count']
options = request.args.to_dict()
limit = int(options.get('limit', '1000'))
hours_ago = int(options.get('hours_ago', '4'))
clio = Clio()
credentials = clio.hpfeed.get_credentials(
get_credentials_payloads(clio, limit, hours_ago))
outf = StringIO()
wr = csv.DictWriter(outf,
fieldnames=fieldnames,
delimiter='\t',
lineterminator='\n')
wr.writeheader()
for cred in credentials:
wr.writerow({
'username': cred[0][0],
'password': cred[0][1],
'count': cred[1]
})
response_data = outf.getvalue()
outf.close()
response = make_response(response_data)
response.headers['Content-type'] = 'text/plain'
return response
def new_clio_connection():
from mhn.common.clio import Clio
import os
return Clio(os.getenv('MONGO_HOST'), int(os.getenv('MONGO_PORT')),
True if os.getenv('MONGO_AUTH') == 'true' else False,
os.getenv('MONGO_USER'), os.getenv('MONGO_PASSWORD'),
os.getenv('MONGO_AUTH_MECHANISM'))
def graph_users():
clio = Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle,
show_x_labels=True,
config=PYGAL_CONFIG)
bar_chart.title = "Cowrie username"
clio = Clio()
top_users = clio.hpfeed.count_users(get_credentials_payloads(clio))
for user in top_users:
bar_chart.add(user[0], [{
'label': str(user[0]),
'xlink': '',
'value': user[1]
}])
return bar_chart.render_response()
def get_feeds():
clio = Clio()
options = paginate_options(limit=10)
options['order_by'] = '-_id'
count,columns,feeds = clio.hpfeed.get_payloads(options, request.args.to_dict())
channel_list = clio.hpfeed.channel_map.keys()
feeds = mongo_pages(feeds, count, limit=10)
return render_template('ui/feeds.html', feeds=feeds, columns=columns, channel_list=channel_list, view='ui.get_feeds', **request.args.to_dict())
def graph_combos():
clio = Clio()
bar_chart = pygal.Bar(style=LightColorizedStyle,
show_x_labels=True,
config=PYGAL_CONFIG)
bar_chart.title = "Kippo Top User/Passwords"
clio = Clio()
top_combos = clio.hpfeed.count_combos(
clio.hpfeed.get_payloads({'limit': 10000},
{"channel": "kippo.sessions"})[2])
for combo in top_combos:
bar_chart.add(combo[0], [{
'label': str(combo[0]),
'xlink': '',
'value': combo[1]
}])
return bar_chart.render_response()
def get_attacks():
clio = Clio()
options = paginate_options(limit=10)
options['order_by'] = '-timestamp'
total = clio.session.count(**request.args.to_dict())
sessions = clio.session.get(
options=options, **request.args.to_dict())
sessions = mongo_pages(sessions, total, limit=10)
return render_template('ui/attacks.html', attacks=sessions,
sensors=Sensor.query, view='ui.get_attacks',
get_flag_ip=get_flag_ip, get_sensor_name=get_sensor_name, **request.args.to_dict())
def attacker_stats(ip):
options = request.args.to_dict()
hours_ago = int(options.get('hours_ago', '720')) # 30 days
for name in options.keys():
if name not in ('hours_ago', 'limit',):
del options[name]
results = Clio().session.attacker_stats(ip, hours_ago=hours_ago)
return jsonify(
data=results,
meta={
'query': 'attacker_stats',
'options': options
}
)
def dashboard():
clio = Clio()
# Number of attacks in the last 24 hours.
attackcount = clio.session.count(hours_ago=24)
# TOP 5 attacker ips.
top_attackers = clio.session.top_attackers(top=5, hours_ago=24)
# TOP 5 attacked ports
top_ports = clio.session.top_targeted_ports(top=5, hours_ago=24)
# TOP 5 sigs
freq_sigs = clio.hpfeed.top_sigs(top=5, hours_ago=24)
return render_template('ui/dashboard.html',
attackcount=attackcount,
top_attackers=top_attackers,
top_ports=top_ports,
freq_sigs=freq_sigs,
get_flag_ip=get_flag_ip)
def create_sensor():
missing = Sensor.check_required(request.json)
if missing:
return error_response(errors.API_FIELDS_MISSING.format(missing), 400)
else:
sensor = Sensor(**request.json)
sensor.uuid = str(uuid1())
sensor.ip = request.remote_addr
Clio().authkey.new(**sensor.new_auth_dict()).post()
try:
db.session.add(sensor)
db.session.commit()
except IntegrityError:
return error_response(
errors.API_SENSOR_EXISTS.format(request.json['name']), 400)
else:
return jsonify(sensor.to_dict())
def get_feed():
from mhn.common.clio import Clio
from mhn.auth import current_user
authfeed = mhn.config['FEED_AUTH_REQUIRED']
if authfeed and not current_user.is_authenticated():
abort(404)
feed = AtomFeed('MHN HpFeeds Report', feed_url=request.url,
url=request.url_root)
sessions = Clio().session.get(options={'limit': 1000})
for s in sessions:
feedtext = u'Sensor "{identifier}" '
feedtext += '{source_ip}:{source_port} on sensorip:{destination_port}.'
feedtext = feedtext.format(**s.to_dict())
feed.add('Feed', feedtext, content_type='text',
published=s.timestamp, updated=s.timestamp,
url=makeurl(url_for('api.get_session', session_id=str(s._id))))
return feed
def credential_list():
options = request.args.to_dict()
limit = int(options.get('limit', '1000'))
hours_ago = int(options.get('hours_ago', '4'))
clio = Clio()
credentials = clio.hpfeed.get_credentials(
get_credentials_payloads(clio, limit, hours_ago))
results = []
for cred in credentials:
results.append({
'username': cred[0][0],
'password': cred[0][1],
'count': cred[1]
})
return jsonify(data=results,
meta={
'query': 'attacker_stats',
'options': options
})
def dashboard():
clio = Clio()
# Number of attacks in the last 24 hours.
attackcount = clio.session.count(hours_ago=24)
# TOP 5 attacker ips.
top_attackers = clio.session.top_attackers(top=5, hours_ago=24)
# TOP 5 attacked ports
top_ports = clio.session.top_targeted_ports(top=5, hours_ago=24)
# Top 5 honey pots with counts
top_hp = clio.session.top_hp(top=5, hours_ago=24)
# Top Honeypot sensors
top_sensor = clio.session.top_sensor(top=5, hours_ago=24)
# TOP 5 sigs
freq_sigs = clio.hpfeed.top_sigs(top=5, hours_ago=24)
return render_template('ui/dashboard.html',
attackcount=attackcount,
top_attackers=top_attackers,
top_ports=top_ports,
top_hp=top_hp,
top_sensor=top_sensor,
freq_sigs=freq_sigs,
get_sensor_name=get_sensor_name,
get_flag_ip=get_flag_ip,
get_country_ip=get_country_ip)
def get_metadatum(metadata_id):
return _get_one_resource(Clio().metadata, metadata_id)
def delete_sensor(uuid):
sensor = Sensor.query.filter_by(uuid=uuid).first_or_404()
Clio().authkey.delete(identifier=uuid)
db.session.delete(sensor)
db.session.commit()
return jsonify({})
def get_metadata():
return _get_query_resource(Clio().metadata, request.args.to_dict())
def get_url(url_id):
return _get_one_resource(Clio().url, url_id)
def get_files():
return _get_query_resource(Clio().file, request.args.to_dict())
def get_dorks():
return _get_query_resource(Clio().dork, request.args.to_dict())
def get_urls():
return _get_query_resource(Clio().url, request.args.to_dict())
def get_sessions():
return _get_query_resource(Clio().session, request.args.to_dict())
def get_feeds():
return _get_query_resource(Clio().hpfeed, request.args.to_dict())
def get_file(file_id):
return _get_one_resource(Clio().file, file_id)
def get_dork(dork_id):
return _get_one_resource(Clio().dork, dork_id)