def __init__(self, id, name, description, status, weakness):
tk.assertType(str,
id=id,
name=name,
description=description,
status=status,
weakness=weakness)
self.id = id
self.name = name
self.description = description
self.status = status
self.weakness = weakness
self.capec = None # Populated with pointers at runtime
def __init__(self, complexity, authentication, vector):
tk.assertType(str,
complexity=complexity,
authentication=authentication,
vector=vector)
if not (complexity.upper() in ["HIGH", "MEDIUM", "LOW"]
and authentication.upper()
in ["NONE", "SINGLE_INSTANCE", "MULTIPLE_INSTANCES"] and
vector.upper() in ["NETWORK", "LOCAL", "ADJACENT_NETWORK"]):
raise ValueError("incorrect values given")
self.complexity = complexity.upper()
self.authentication = authentication.upper()
self.vector = vector.upper()
def generate_full_query(self, f):
query = self.generate_minimal_query(f)
if current_user.is_authenticated():
if f['blacklistSelect'] == "on":
regexes = db.getRules('blacklist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
'$or': [{
'vulnerable_configuration': re.compile(exp)
}, {
'vulnerable_configuration': {
'$exists': False
}
}, {
'vulnerable_configuration': []
}]
})
if f['whitelistSelect'] == "hide":
regexes = db.getRules('whitelist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
'$or': [{
'vulnerable_configuration': re.compile(exp)
}, {
'vulnerable_configuration': {
'$exists': False
}
}, {
'vulnerable_configuration': []
}]
})
if f['unlistedSelect'] == "hide":
wlregexes = tk.compile(db.getRules('whitelist'))
blregexes = tk.compile(db.getRules('blacklist'))
query.append({
'$or': [{
'vulnerable_configuration': {
'$in': wlregexes
}
}, {
'vulnerable_configuration': {
'$in': blregexes
}
}]
})
return query
def markCPEs(self, cve):
blacklist = tk.compile(db.getRules('blacklist'))
whitelist = tk.compile(db.getRules('whitelist'))
for conf in cve['vulnerable_configuration']:
conf['list'] = 'none'
conf['match'] = 'none'
for w in whitelist:
if w.match(conf['id']):
conf['list'] = 'white'
conf['match'] = w
for b in blacklist:
if b.match(conf['id']):
conf['list'] = 'black'
conf['match'] = b
return cve
def JSON2HTMLTableFilter(data, stack=None):
_return = ""
if type(stack) == str: stack = [stack]
if type(data) == list:
if len(data) == 1:
_return += JSON2HTMLTableFilter(data[0], stack)
else:
_return += '<ul class="via4">'
for item in data:
_return += ('<li>%s</li>' %
JSON2HTMLTableFilter(item, stack))
_return += '</ul>'
elif type(data) == dict:
_return += '<table class="invisiTable">'
for key, val in sorted(data.items()):
_return += '<tr><td><b>%s</b></td><td>%s</td></tr>' % (
key, JSON2HTMLTableFilter(val, stack + [key]))
_return += '</table>'
elif type(data) == str:
if stack:
_return += "<a href='/link/" + doublequote('.'.join(
stack)) + "/" + doublequote(data) + "'>" #link opening
_return += "<span class='glyphicon glyphicon-link' aria-hidden='true'></span> </a>"
_return += "<a target='_blank' href='%s'>%s</a>" % (
data, data) if tk.isURL(data) else data
_return += ""
return _return
def markCPEs(self, cve):
blacklist = tk.compile(db.getRules('blacklist'))
whitelist = tk.compile(db.getRules('whitelist'))
for conf in cve['vulnerable_configuration']:
conf['list'] = 'none'
conf['match'] = 'none'
for w in whitelist:
if w.match(conf['id']):
conf['list'] = 'white'
conf['match'] = w
for b in blacklist:
if b.match(conf['id']):
conf['list'] = 'black'
conf['match'] = b
return cve
def cvesForCPE(cpe):
cpe = tk.toStringFormattedCPE(cpe)
data = []
if cpe:
cvesp = cves.last(rankinglookup=False, namelookup=False, via4lookup=True, capeclookup=False)
for x in db.cvesForCPE(cpe):
data.append(cvesp.getcve(x['id']))
return data
def __init__(self, confidentiality, integrity, availability):
tk.assertType(str,
confidentiality=confidentiality,
integrity=integrity,
availability=availability)
confidentiality = confidentiality.upper()
integrity = integrity.upper()
availability = availability.upper()
ACCEPTED = ["COMPLETE", "PARTIAL", "NONE"]
if (set(ACCEPTED + [confidentiality, integrity, availability]) !=
set(ACCEPTED)):
raise ValueError("incorrect values given")
self.confidentiality = confidentiality
self.integrity = integrity
self.availability = availability
def cvesForCPE(cpe):
cpe = tk.toStringFormattedCPE(cpe)
data = []
if cpe:
cvesp = cves.last(rankinglookup=False, namelookup=False, via4lookup=True, capeclookup=False)
for x in db.cvesForCPE(cpe):
data.append(cvesp.getcve(x['id']))
return cves
def list_mark(self, listed, cveList):
if listed not in ['white', 'black']: return list(cves)
items = tk.compile(db.getRules(listed+'list'))
# check the cpes (full or partially) in the black/whitelist
for i, cve in enumerate(list(cveList)): # the list() is to ensure we don't have a pymongo cursor object
for c in cve['vulnerable_configuration']:
if any(regex.match(c) for regex in items):
cveList[i][listed+'listed'] = 'yes'
return cveList
def api_admin_get_cron(self):
print(Advanced_API.getAuth())
validator = Toolkit.requestValidation(request, Advanced_API.getAuth()[1])
if not validator.check_is_master(): return validator.error
values = validator.retrive_object_value('getcronforuser',validator.user, True)
if not values:
return validator.error
else:
return values
def list_mark(self, listed, cveList):
if listed not in ['white', 'black']: return list(cves)
items = tk.compile(db.getRules(listed+'list'))
# check the cpes (full or partially) in the black/whitelist
for i, cve in enumerate(list(cveList)): # the list() is to ensure we don't have a pymongo cursor object
for c in cve['vulnerable_configuration']:
if any(regex.match(c) for regex in items):
cveList[i][listed+'listed'] = 'yes'
return cveList
def api_admin_delete_user(self):
mandatory = ['user']
validator = Toolkit.requestValidation(request, Advanced_API.getAuth()[1] )
if not validator.check_is_master(): return validator.error
if validator.check_if_json():
if not validator.check_attributes(mandatory):
return validator.error
if validator.check_object_exists("userExists", validator.content['user']):
db.deleteUser(validator.content['user'])
return make_response(jsonify(message="User {} deleted".format(validator.content['user'])))
return validator.error
def summarycvesForCPE(cpe):
cpe = tk.toStringFormattedCPE(cpe)
data = []
if cpe:
cvesp = cves.last(rankinglookup=False,
namelookup=False,
via4lookup=True,
capeclookup=False)
r = db.summarycvesForCPE(cpe)
data = r['results']
return data
def cvesForCPE(cpe):
cpe = tk.toStringFormattedCPE(cpe)
data = []
if cpe:
cvesp = cves.last(rankinglookup=False,
namelookup=False,
via4lookup=True,
capeclookup=False)
r = db.cvesForCPE(cpe)
for x in r["results"]:
data.append(cvesp.getcve(x["id"]))
return data
def generate_full_query(self, f):
query = self.generate_minimal_query(f)
if current_user.is_authenticated():
if f['blacklistSelect'] == "on":
regexes = db.getRules('blacklist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({'$or': [{'vulnerable_configuration': re.compile(exp)},
{'vulnerable_configuration': {'$exists': False}},
{'vulnerable_configuration': []} ]})
if f['whitelistSelect'] == "hide":
regexes = db.getRules('whitelist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({'$or': [{'vulnerable_configuration': re.compile(exp)},
{'vulnerable_configuration': {'$exists': False}},
{'vulnerable_configuration': []} ]})
if f['unlistedSelect'] == "hide":
wlregexes = tk.compile(db.getRules('whitelist'))
blregexes = tk.compile(db.getRules('blacklist'))
query.append({'$or': [{'vulnerable_configuration': {'$in': wlregexes}},
{'vulnerable_configuration': {'$in': blregexes}}]})
return query
def api_admin_delete_update_cron(self):
mandatory = ['name']
validator = Toolkit.requestValidation(request, Advanced_API.getAuth()[1])
if not validator.check_is_master(): return validator.error
if validator.check_if_json():
if not validator.check_attributes(mandatory):
return validator.error
if not validator.check_object_exists("checkcronexists", validator.content['name'], validator.user):
return validator.error
else:
db.deleteCronEntry(validator.content['name'], validator.user)
return make_response(jsonify(message='Cron {} deleted'.format(validator.content['name'])), 200)
else:
return validator.error
def fetchAndStoreData(self):
settings = self.db.readSettings()
now = calendar.timegm(time.gmtime())
since = "%sm"%int(math.ceil((now - settings["lastrun"])/60))
lifespan = TK.lifeSpanToMinutes(settings["datalife"])
since = since if int(since[:-1])<int(lifespan[:-1]) else lifespan
data = self.MispEx.getMISPData(since)
matches = self.MispEx.extractData(data, settings["datatype"],
settings["analysis_level"],
settings["threat_level"])
self.db.storeData(matches)
s = self.db.readSettings()
self.db.writeSettings(s["lastrun"], now, s["datatype"],
s["datalife"], s["analysis_level"],
s["threat_level"])
self.cleanOldRecords()
def api_admin_create_user(self):
mandatory = ['user', 'password']
validator = Toolkit.requestValidation(request, Advanced_API.getAuth()[1])
if not validator.check_is_master(): return validator.error
if validator.check_if_json():
if not validator.check_attributes(mandatory):
return validator.error
if not validator.check_object_exists("userExists", validator.content['user']):
admin = True if 'admin' in validator.content.keys() else False
hashed = True if 'hashed' in validator.content.keys() else False
localonly = True if 'localonly' in validator.content.keys() else False
db.addUser(user=validator.content['user'], pwd=validator.content['password'], admin=admin, localOnly=localonly, hashed=hashed)
return make_response(jsonify(message="User {} created".format(validator.content['user'])))
else:
return validator.error
else:
return validator.error
def api_admin_create_update_cron(self):
mandatory = ['name']
validator = Toolkit.requestValidation(request, Advanced_API.getAuth()[1])
if not validator.check_is_master(): return validator.error
if validator.check_if_json():
if not validator.check_attributes(mandatory):
return validator.error
else:
if not validator.check_object_exists("checkcronexists", validator.content['name'], validator.user):
recure = "180" if 'repeat' not in validator.content.keys() else \
self.repeatevery[validator.content['repeat']]
when = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') if 'when' not in \
validator.content.keys() else validator.content['when']
description = "" if 'description' not in validator.content.keys() else validator.content['description']
db.createCronEntry(name=validator.content['name'], when=when, repeat=recure, owner= validator.user, \
isadmincron=True, description=description)
return make_response(jsonify(message="Cron {} created".format(validator.content['name'])), 200)
else:
return validator.error
else:
return validator.error
def __init__(self, id, name, summary, prerequisites, solutions,
weaknesses):
tk.assertType(str,
id=id,
name=name,
summary=summary,
prerequisites=prerequisites,
solutions=solutions)
tk.assertType((list, tuple), weaknesses=weaknesses)
tk.assertTypeForAllIn(str, weaknesses)
self.id = id
self.name = name
self.summary = summary
self.prerequisites = prerequisites
self.solutions = solutions
self.weaknesses = list(weaknesses)
def JSON2HTMLTableFilter(data, stack = None):
_return = ""
if type(stack) == str: stack = [stack]
if type(data) == list:
if len(data) == 1:
_return += JSON2HTMLTableFilter(data[0], stack)
else:
_return += '<ul class="via4">'
for item in data:
_return += ('<li>%s</li>'%JSON2HTMLTableFilter(item, stack))
_return += '</ul>'
elif type(data) == dict:
_return += '<table class="invisiTable">'
for key, val in sorted(data.items()):
_return += '<tr><td><b>%s</b></td><td>%s</td></tr>'%(key, JSON2HTMLTableFilter(val, stack+[key]))
_return += '</table>'
elif type(data) == str:
if stack:
_return += "<a href='/link/"+doublequote('.'.join(stack))+"/"+doublequote(data)+"'>" #link opening
_return += "<span class='glyphicon glyphicon-link' aria-hidden='true'></span> </a>"
_return += "<a target='_blank' href='%s'>%s</a>"%(data, data) if tk.isURL(data) else data
_return += ""
return _return
def __init__(self,
id,
summary,
vulnerable_configuration,
published,
modified=None,
impact=None,
access=None,
cvss=None,
cwe=None,
references=None,
cvss_time=None):
if not references: references = []
tk.assertType(str, id=id, summary=summary)
tk.assertType((float, str, None), cvss=cvss)
tk.assertType(datetime, published=published)
tk.assertType((datetime, None), modified=modified, cvss_time=cvss_time)
tk.assertType(list,
vulnerable_configuration=vulnerable_configuration,
references=references)
tk.assertType((Impact, None), impact=impact)
tk.assertType((Access, None), access=access)
tk.assertType((CWE, None), cwe=cwe)
tk.assertTypeForAllIn(CPE, vulnerable_configuration)
tk.assertTypeForAllIn(str, references)
self.id = id.upper()
self.cvss = cvss and float(cvss) or None
self.summary = summary
self.vulnerable_configuration = vulnerable_configuration
self.published = published
self.modified = modified
self.impact = impact
self.access = access
self.cwe = cwe
self.references = references
self.cvss_time = cvss_time
def api_cpe22(self, cpe): cpe = tk.toOldCPE(cpe) return cpe if cpe else "None"
def __init__(self, id, title=None, references=None):
if not references: references = []
tk.assertType(str, id=id)
tk.assertType((str, None), title=title)
tk.assertType((list, tuple, None), references=references)
if references:
tk.assertTypeForAllIn(str, references)
self.id = tk.toStringFormattedCPE(id)
self.id_2_2 = tk.toOldCPE(id)
self.title = title if title else tk.cpeTitle(self.id)
self.references = references and list(references) or []
def api_cpe22(self, cpe):
cpe = tk.toOldCPE(cpe)
return cpe if cpe else "None"
def api_cpe23(self, cpe):
cpe = tk.toStringFormattedCPE(cpe)
return cpe if cpe else "None"
def api_cpe23(self, cpe): cpe = tk.toStringFormattedCPE(cpe) return cpe if cpe else "None"
def cleanOldRecords(self):
settings = self.db.readSettings()
now = calendar.timegm(time.gmtime())
lifespan = TK.lifeSpanToMinutes(settings["datalife"])
oldest = now - (int(lifespan[:-1]) * 60)
self.db.removeData("age < %s"%oldest)
type=int,
help='Analysis level (0: Initial, 1: Ongoing, 2: Completed)')
argParser.add_argument(
'-T',
metavar="integer",
type=int,
help='Threat level (4: Undefined, 3: Low, 2: Medium, 1: High)')
args = argParser.parse_args()
if not (args.t or args.e or args.d):
sys.exit("Please choose an option")
MispEx = MispExtractor(key=args.k, url=args.u)
data = MispEx.getMISPData(args.s)
if args.t:
print(json.dumps(MispEx.getTypes(data), indent=2, sort_keys=True))
elif args.e:
MispEx.getExample(data)
elif args.d:
search = args.d.lower()
if search not in MispEx.searches.keys():
sys.exit("Please use a valid search term")
matches = MispEx.extractData(data,
search,
threat_level=args.T,
analysis_level=args.A)
output = TK.generateCSV(matches)
_output(output)
