def get_host(self, ip_address):
try:
click.echo('[%s_Shodan_get_host] Complete with %d items' %
(get_datetime(), ip_address))
except Exception as e:
click.echo('[%s_Shodan_get_host] Complete with %d items' %
(get_datetime(), ip_address))
def send_message():
auth = odbserver.auth_user(request.headers['Authorization'])
if auth:
return jsonify(auth)
form = request.form.to_dict(flat=True)
response = CB.send_message(message=form['message'],
conversation_id=form['conversationId'])
odbserver.send_message({
"text": response['message'],
"title": response['skill'],
"sender": form['userName'],
"receiver": CB.uuid,
"sessionId": request.headers['SESSIONID']
})
odbserver.send_message({
"text": response['response'],
"title": response['skill'],
"sender": CB.uuid,
"receiver": form['userName'],
"sessionId": request.headers['SESSIONID']
})
return jsonify({
"status": 200,
"message": "Message sent at %s" % get_datetime(),
"data": response
})
def get_location_by_latlon(location, loc_string, db):
"""
:param location:
:param db:
:return:
"""
r = db.client.command('''
select key, class_name, Category, description, Latitude, Longitude, city,
icon, title from Location where Latitude = %f and Longitude = %f
''' % (location.latitude, location.longitude))
if len(r) == 0:
node = db.create_node(
**{
"class_name":
"Location",
"title":
loc_string,
"icon":
db.ICON_LOCATION,
"group":
"Locations",
"attributes": [
{
"label": "Created",
"value": get_datetime()
},
{
"label": "Latitude",
"value": location.latitude
},
{
"label": "Longitude",
"value": location.longitude
},
{
"label": "importance",
"value": location.raw["importance"]
},
{
"label": "Category",
"value": location.raw["type"]
},
{
"label": "description",
"value": "%s %s" % (loc_string, location.address)
},
]
})
return node["data"]
else:
new_description = "%s %s" % (r[0].oRecordData["description"],
loc_string)
db.update(class_name="Location",
var="description",
val=new_description,
key=r[0].oRecordData["key"])
# update the location
return r
def create_game(self, **kwargs):
"""
Create a node that represents the game so all resources and player nodes can be linked to it
:param kwargs:
:return:
"""
game = self.create_node(class_name=sGame,
name=kwargs["gameName"],
created=get_datetime(),
current_round=0)
return self.node_to_d3(**game['data'])
def create_resource(self, **kwargs):
"""
Create a random resource or based on specifics in the kwargs
Assign colors based on CRIMEFILLED and symbolTypes as ASCOPE
:param self:
:param kwargs:
:return:
"""
r = random.choice(self.content['resources'])
# color, fontColor, size (200), symbolType (circle, cross, diamond, square, star, triangle, wye)
xpos, ypos = self.get_nation_based_location(
nation=kwargs['homeNation'])
resource = self.create_node(
class_name=sResource,
name="%s %s %s" %
(kwargs['homeNation'], r['ascope'], r['crimefilled']),
ascope=r['ascope'],
crimefilled=r['crimefilled'],
type=r['type'],
category=r['category'],
created=get_datetime(),
description="%s %s %s" %
(r['type'], r['ascope'], r['crimefilled']),
icon="TBD",
offence=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])),
defence=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])),
hitpoints=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])),
speed=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])),
xpos=xpos,
ypos=ypos,
group=kwargs['homeNation'],
zpos=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])),
active=True,
player=kwargs['player'],
color=self.styling[r['crimefilled']],
symbolType=self.styling[r['ascope']],
value=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])))
return resource
def custom_resource(self, **kwargs):
"""
Create a resource based on custom entries. If the ascope, crimefilled, type, category combination is not in the
content, add it and save for future loading.
:param kwargs:
:return:
"""
self.check_content(**kwargs)
xpos, ypos = self.get_nation_based_location(
nation=kwargs['homeNation'])
resource = self.create_node(
class_name=sResource,
name="%s %s %s" %
(kwargs['homeNation'], kwargs['ascope'], kwargs['crimefilled']),
ascope=kwargs['ascope'],
crimefilled=kwargs['crimefilled'],
type=kwargs['type'],
category=kwargs['category'],
created=get_datetime(),
description="%s %s %s" %
(kwargs['type'], kwargs['ascope'], kwargs['crimefilled']),
icon="TBD",
offence=int(kwargs['offence']),
defence=int(kwargs['defence']),
hitpoints=int(kwargs['hitpoints']),
speed=int(kwargs['speed']),
xpos=xpos,
ypos=ypos,
group=kwargs['homeNation'],
zpos=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])),
active=True,
player=kwargs['player'],
color=self.styling[kwargs['crimefilled']],
symbolType=self.styling[kwargs['ascope']],
value=int(
np.random.normal(loc=self.norm['mean'],
scale=self.norm['stdev'])))
self.create_edge(fromNode=kwargs['playerId'],
fromClass=sPlayer,
toNode=resource['data']['key'],
toClass=sResource,
edgeType="Owns")
gameState = self.get_game(**kwargs)
gameState['message'] = resource['data']['key']
return gameState
def update_node(self, **kwargs):
"""
Update a game node with attributes
:param kwargs:
:return:
"""
sql = "update %s set " % (kwargs['class_name'])
for k in kwargs:
if k not in self.no_update:
if change_if_number(kwargs[k]):
sql = sql + "%s = %s, " % (k, kwargs[k])
elif type(kwargs[k]) == bool:
sql = sql + "%s = '%s', " % (k, kwargs[k])
else:
sql = sql + "%s = '%s', " % (k, clean(kwargs[k]))
sql = sql[:len(sql) - 2] + " where key = %s" % kwargs[
'id'] # remove last comma and put filter
click.echo("[%s_game_update_node]: Running sql\n%s" %
(get_datetime(), sql))
self.client.command(sql)
def create_move(self, **kwargs):
"""
Create a move that will be run through the effects application engine when the
Game current round = the move assigned round to give players availability to delay
:param kwargs:
:return:
"""
move = {
"created": get_datetime(),
"round": kwargs['round'],
"class_name": sMove,
"description": "string",
"player": kwargs['playerKey']
}
m = self.create_node(**move)
for e in kwargs['effectKeys']:
self.create_edge(fromClass=sMove,
toClass=sEffect,
fromNode=m['data']['key'],
toNode=e,
edgeType=sIncludes)
for t in kwargs['targetKeys']:
self.create_edge(fromClass=sMove,
toClass=sResource,
fromNode=m['data']['key'],
toNode=t,
edgeType=sIncludes)
for r in kwargs['resourceKeys']:
self.create_edge(fromClass=sMove,
toClass=sResource,
fromNode=m['data']['key'],
toNode=r,
edgeType=sIncludes)
newGameState = self.get_game(gameKey=kwargs['gameKey'])
newGameState['result'] = move['round']
return newGameState
def search(self, searchterm):
"""
The provided string is used to search the database of banners in Shodan, with the additional option to provide
filters inside the search query using a "filter:value" format. For example, the following search query would
find Apache webservers located in Germany: "apache country:DE".
With a search term, the Shodan Crawl is extracted and related to each DEVICE which is then related to it's
VULNERABILITIES, each of which are related to VUL REFERENCES. Lastly, the Device is related to the LOCATION
it was found.
:param searchterm:
:return:
"""
click.echo('[%s_Shodan_search] Starting search for %s' %
(get_datetime(), searchterm))
try:
results = api.search(searchterm)
message = '[%s_Shodan_search] Complete with %d items' % (
get_datetime(), results["total"])
click.echo(message)
except Exception as e:
message = '[%s_Shodan_search] %s' % (get_datetime(), str(e))
click.echo(message)
return [], message
for r in results["matches"]:
# Set up the Shodan Crawler node from the row
if '_shodan' in r.keys():
if 'id' in r["_shodan"].keys():
s_keys = ["crawler", "module", "id"]
s_node = {
"class_name":
"Object",
"Category":
"Shodan crawler",
"Ext_key":
r["_shodan"]["id"],
"description":
"Shodan crawler %s" %
str(r["_shodan"]).replace("'", "").replace(
'"', "").replace("{", "").replace("}", "")
}
for sk in s_keys:
if sk in r['_shodan'].keys():
s_node[sk] = r['_shodan'][sk]
# Create the Shodan Crawler Node
s_node = self.create_node(**s_node)
# Set up the device found
d_keys = [
"ip", "os", "timestamp", "hash", "isp", "port", "info",
"version", "product", "ip_str", "asn", "org"
]
if 'ip_str' in r.keys() and 'port' in r.keys():
d_node = {
"class_name": "Object",
"Category": "Device",
"Ext_key": "%s:%s" % (r["ip_str"], r["port"])
}
for dk in d_keys:
if dk in r.keys():
d_node[dk] = r[dk]
# Create the Device found by the crawler
d_node = self.create_node(**d_node)
# Create the relationship
self.create_edge_new(edgeType="Discovered",
fromNode=s_node["data"]["key"],
toNode=d_node["data"]["key"])
# Get the vulnerabilities
if 'vulns' in r.keys():
for v in r['vulns']:
v_ok = True
cve = self.client.command('''
select rid from index:Vulnerability_Ext_key where key = '%s'
''' % v)
try:
if len(cve) < 1:
v_node = self.create_node(
class_name="Vulnerability",
Ext_key=v,
description=r['vulns'][v]
["summary"],
source="Shodan",
)["data"]["key"]
else:
try:
v_node = cve[0].oRecordData[
"rid"].get_hash()
except Exception as e:
click.echo(
'[%s_Shodan_search] Error with vulnerability node %s: %s'
% (get_datetime(), str(e), v))
v_ok = False
except Exception as e:
click.echo(
'[%s_Shodan_search] Error with vulnerability node %s: %s'
% (get_datetime(), str(e), v))
v_ok = False
if v_ok:
self.create_edge_new(
edgeType="Has",
fromNode=d_node["data"]["key"],
toNode=v_node)
# Get the References for each vulnerability
if "references" in r['vulns'][v]:
try:
for ref in r['vulns'][v][
"references"]:
ref_node = self.create_node(
class_name="Object",
Category=
"Vulnerability Reference",
Ext_key=ref,
source="Shodan",
description=
"Reference to %s" %
v)["data"]["key"]
self.create_edge_new(
edgeType="References",
fromNode=ref_node,
toNode=v_node)
except Exception as e:
click.echo(
'[%s_Shodan_search] Error with reference node %s: %s'
%
(get_datetime(), str(e), ref))
if 'location' in r.keys():
l_keys = [
"country_name", "city", "longitude", "latitude"
]
l_node = {"class_name": "Location"}
create = True
for lk in l_keys:
if r['location'][lk] != None:
l_node[lk.capitalize()] = r['location'][lk]
else:
create = False
if create:
l_node = self.create_node(
**l_node)["data"]["key"]
self.create_edge_new(
edgeType="LocatedAt",
fromNode=d_node["data"]["key"],
toNode=l_node)
return results, message
def get_game(self, **kwargs):
"""
Get a game that was saved to the Database in the following structure:
Game -> Players -> Resources
and return the same format of a GameState as in the Game setup. When filling the moves, the gameState may change
based on that function's trigger to update the round and run moves with effects.
:return:
"""
self.gameState = {
"nodes": [],
"links": [],
"players": [],
"gameName": None,
"moves": [],
"availableMoves": [],
"stability": 0,
"current_round": 0
}
sql = ('''
match {class: Game, as: g, where: (key = '%s')}.out(HasPlayer){class: V, as: p}.out(){class: V, as: r}
return g.name, g.current_round, p.key, p.name, p.created, p.group, p.score, p.status, p.icon,
r.key, r.name, r.ascope, r.crimefilled, r.type, r.category, r.created, r.description, r.player,
r.icon, r.offence, r.defence, r.hitpoints, r.speed, r.xpos, r.ypos, r.zpos, r.group, r.active, r.deleted, r.value,
r.class_name, r.color, r.objective, r.phase, r.measure, r.indicator, r.strat, r.goal, r.fontColor, r.symbolType
''' % kwargs['gameKey'])
nodeKeys = [] # Quality check to ensure no duplicates sent
self.gameState['key'] = kwargs['gameKey']
click.echo("[%s_gameserver_get_game] SQL: %s" % (get_datetime(), sql))
for o in self.client.command(sql):
if not self.gameState['gameName']:
self.gameState['gameName'] = o.oRecordData['g_name']
self.gameState['current_round'] = o.oRecordData[
'g_current_round']
current_round = int(o.oRecordData['g_current_round'])
click.echo("[%s_gameserver_get_game] Game name: %s" %
(get_datetime(), o.oRecordData['g_name']))
Player = {
"id": o.oRecordData['p_key'],
"name": o.oRecordData['p_name'],
"icon": o.oRecordData['p_icon'],
"group": o.oRecordData['p_group'],
"score": o.oRecordData['p_score'],
"status": o.oRecordData['p_status'],
"class_name": "Player"
}
if Player not in self.gameState['players']:
self.gameState['players'].append(Player)
self.gameState['nodes'].append(Player)
click.echo("[%s_gameserver_get_game] Player: %s" %
(get_datetime(), Player))
if o.oRecordData['r_class_name'] == sResource:
Node = {
"id": o.oRecordData['r_key'],
"name": o.oRecordData['r_name'],
"ascope": o.oRecordData['r_ascope'],
"crimefilled": o.oRecordData['r_crimefilled'],
"type": o.oRecordData['r_type'],
"category": o.oRecordData['r_category'],
"created": o.oRecordData['r_created'],
"description": o.oRecordData['r_description'],
"icon": o.oRecordData['r_icon'],
"offence": o.oRecordData['r_offence'],
"defence": o.oRecordData['r_defence'],
"hitpoints": o.oRecordData['r_hitpoints'],
"speed": o.oRecordData['r_speed'],
"xpos": o.oRecordData['r_xpos'],
"ypos": o.oRecordData['r_ypos'],
"zpos": o.oRecordData['r_zpos'],
"group": o.oRecordData['r_group'],
"active": o.oRecordData['r_active'],
"deleted": o.oRecordData['r_deleted'],
"value": o.oRecordData['r_value'],
"player": o.oRecordData['r_player'],
"class_name": o.oRecordData['r_class_name'],
"color": o.oRecordData['r_color']
}
elif o.oRecordData['r_class_name'] == sEffect:
Node = {
"id": o.oRecordData['r_key'],
"name": o.oRecordData['r_name'],
"class_name": sEffect,
"indicator": o.oRecordData['r_indicator'],
"measure": o.oRecordData['r_measure'],
"objective": o.oRecordData['r_objective'],
"phase": o.oRecordData['r_phase'],
"player": o.oRecordData['r_player'],
"strat": o.oRecordData['r_strat'],
"value": o.oRecordData['r_value'],
"goal": o.oRecordData['r_goal'],
}
if Node['id'] not in nodeKeys:
self.gameState['nodes'].append(Node)
self.gameState['links'].append({
"source": Player['id'],
"target": Node['id'],
"value": random.randint(1, 3)
})
if len(self.gameState['players']) == 0:
return "No Game found with key %s" % kwargs['gameKey']
self.gameState['moves'] = self.get_moves()
self.gameState['availableMoves'] = self.check_available_moves()
# Running moves might change the gameState so check to see if there was a change and then update
if self.gameState['current_round'] != current_round:
self.get_game(gameKey=kwargs['gameKey'])
return self.gameState
def create_player(self, **kwargs):
"""
TODO create dungeon master type player: gets dashboard view with inject and arbitration monitoring
Using the player name and home country create the player and assign resources and priority MPICE effects
player and resource data returned as:
{'message', 'data' { 'key', 'title', 'status', 'attributes' [ {label, value}]}}
xpos and ypos dependent on the Nation area. Also gives the player access to randomized foreign resources
:param self:
:param kwargs:
:return:
"""
# Choose the home nation for the player if not selected. Ensure it is not already chosen
playerd3 = {"nodes": [], "links": []}
homeNation = "None"
if 'homeNation' not in kwargs.keys():
inCache = False
while not inCache:
homeNation = random.choice(list(
self.content['nations'].keys()))
if homeNation not in self.cache['nations']:
self.cache['nations'].append(homeNation)
inCache = True
else:
homeNation = kwargs['homeNation']
# Create the player
player = self.create_node(class_name=sPlayer,
created=get_datetime(),
name=kwargs['name'],
score=0,
group=homeNation,
icon="TBD",
title=kwargs['name'])
player = self.node_to_d3(**player['data'])
playerd3['player'] = player
playerd3['nodes'].append(player)
# Assign MPICE priorities
i = 0
while i < 10:
e = self.create_effect(player=kwargs['name'])
self.create_edge(fromNode=player['id'],
fromClass=sPlayer,
toNode=e['data']['key'],
toClass=sEffect,
edgeType="Owns")
e = self.node_to_d3(**e['data'])
playerd3['links'].append({
"source": player['id'],
"target": e['id'],
"value": random.randint(1, 3)
})
playerd3['nodes'].append(e)
i += 1
# Create the resources
i = 0
while i < 50:
r = self.create_resource(homeNation=homeNation,
player=kwargs['name'])
self.create_edge(fromNode=player['id'],
fromClass=sPlayer,
toNode=r['data']['key'],
toClass=sResource,
edgeType="Owns")
r = self.node_to_d3(**r['data'])
playerd3['links'].append({
"source": player['id'],
"target": r['id'],
"value": random.randint(1, 3)
})
playerd3['nodes'].append(r)
i += 1
i = 0
while i < 25:
r = self.create_resource(player=kwargs['name'],
homeNation=random.choice(
list(self.content['nations'].keys())))
self.create_edge(fromNode=player['id'],
fromClass=sPlayer,
toNode=r['data']['key'],
toClass=sResource,
edgeType="Owns")
r = self.node_to_d3(**r['data'])
playerd3['links'].append({
"source": player['id'],
"target": r['id'],
"value": random.randint(1, 3)
})
playerd3['nodes'].append(r)
i += 1
playerd3['stability'] = random.randint(-999, 999)
return playerd3
def geo_get_hospitals(c):
# array to hold the location objects representing hospitals
h = []
# index to hold the hashed keys to prevent redundancies
index = []
# iterate throught the list of countries and get each page of hospitals
click.echo('[%s_GEO_get_hospitals] %s' % (get_datetime(), c))
url = "https://www.hospitalsworldguide.com/hospitals-in-%s/" % c
page = requests.get(url)
soup = bs(page.content, 'html.parser')
cities = soup.find_all('div', class_='city')
for ci in cities:
cityurl = ci.find(href=True).get('href')
citypage = requests.get("https://www.hospitalsworldguide.com" +
cityurl)
citysoup = bs(citypage.content,
'html.parser').find_all('div',
class_='information_linea')
for i in citysoup:
try:
ExternalLink = i.find_all(href=True)[0].get('href')
title = i.find_all(href=True)[0].get_text()
address = i.find_all(
'div', class_='information_contenido')[0].get_text()
address = address[:address.find('\xa0')]
latlong = geo_string(address)
if latlong:
Latitude = latlong.latitude
Longitude = latlong.longitude
else:
Latitude = 0.0
Longitude = 0.0
except:
ExternalLink = "https://www.hospitalsworldguide.com"
title = "Hosptial in %s" % c
address = "Unknown address"
Latitude = 0.0
Longitude = 0.0
hash_str = clean_concat(str(ExternalLink + title)).replace(",", "")
ext_key = hashlib.md5(hash_str.encode()).hexdigest()
loc = {
"class_name":
"Location",
"ExternalLink":
ExternalLink,
"title":
title,
"Ext_key":
ext_key,
"icon":
"sap-icon://building",
"Source":
"HosptialsWorldGuide",
"Latitude":
Latitude,
"Longitude":
Longitude,
"description":
"Hospital named %s located at %f, %f" %
(title, Latitude, Longitude)
}
if ext_key not in index:
h.append(loc)
index.append(ext_key)
return h
chatbot = Blueprint('chatbot', __name__)
# Set up the Chatserver basing the DB storage on the User ODB
CB = ChatServer()
try:
CB_id = CB.uuid
except:
CB_id = "None"
try:
odbserver.create_user({
'userName': CB_id,
'email': '*****@*****.**',
'passWord': randomString(36)
})
except:
click.echo("[%s_Chatbot_init] Setup required" % get_datetime())
@chatbot.route('/chatbot/send_message', methods=['POST'])
def send_message():
auth = odbserver.auth_user(request.headers['Authorization'])
if auth:
return jsonify(auth)
form = request.form.to_dict(flat=True)
response = CB.send_message(message=form['message'],
conversation_id=form['conversationId'])
odbserver.send_message({
"text": response['message'],
"title": response['skill'],
import json
from flask import jsonify, Blueprint, send_file, request
from apiserver.blueprints.home.models import ODB
from apiserver.utils import get_request_payload, check_for_file, get_datetime
import click
from werkzeug.utils import secure_filename
# Application Route specific object instantiation
home = Blueprint('home', __name__)
# Case where no DB has been established from which the message returned should let the user know to run the setup API
odbserver = ODB()
init_required = odbserver.open_db()
if init_required:
click.echo("[%s_Home_init] %s" % (get_datetime(), init_required))
@home.route('/home/db_init', methods=['GET'])
def db_init():
"""
API endpoint used when the DB has not been created
:return:
"""
result = odbserver.create_db()
return jsonify({"status": 200, "message": result})
@home.route('/', methods=['GET'])
def index():
return jsonify({
from apiserver.blueprints.osint.models import OSINT
from apiserver.utils import get_request_payload, get_datetime, check_for_file
from flask_cors import CORS
from apiserver.blueprints.osint.shodan import Shodan
import click
import json
# Create the blueprint and ensure CORS enabled for the webapp calls
osint = Blueprint('osint', __name__)
CORS(osint)
shodanserver = Shodan()
osintserver = OSINT()
# Case where no DB has been established from which the message returned should let the user know to run the setup API
init_required = osintserver.open_db()
if init_required:
click.echo("[%s_OSINT_view_init] %s" % (get_datetime(), init_required))
else:
osintserver.refresh_indexes()
shodanserver.open_db()
#osintserver.check_base_book()
#osintserver.fill_db()
@osint.route('/osint/db_init', methods=['GET'])
def db_init():
"""
API endpoint used when the DB has not been created
:return:
"""
result = osintserver.create_db()
return jsonify({"status": 200, "message": result})