def cpeItemUpdateCallBack(cpe: dict,
buffer: list,
bufferSize: int,
counter: dict,
flush: bool = False):
if cpe:
item = process_cpe_item(cpe)
buffer.append(item)
currentSize = len(buffer)
if flush or currentSize >= bufferSize:
db.bulkUpdate("cpe", buffer)
buffer.clear()
counter["count"] = counter["count"] + currentSize
print(
str(datetime.now()) + " --> " + str(counter["count"]) +
" CPEs updated")
# dict
cpedict = Configuration.getCPEDict()
# make parser
parser = make_parser()
ch = CPEHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(cpedict)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cpedict))
i = db.getLastModified('cpe')
if i is not None:
if f.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
cpeList=[]
for x in progressbar(ch.cpe):
x['id']= toStringFormattedCPE(x['name'])
x['title']=x['title'][0]
x['cpe_2_2'] = x.pop('name')
if not x['references']: x.pop('references')
cpeList.append(x)
db.bulkUpdate("cpe", cpeList)
#update database info after successful program-run
db.setColUpdate('cpe', f.headers['last-modified'])
self.vendor[-1]['statement'] = self.statement
# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
(f, r) = Configuration.getFeedData('vendor')
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(Configuration.getFeedURL('vendor')))
last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True)
i = db.getLastModified('vendor')
if i is not None:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
statements = []
for statement in progressbar(ch.vendor):
if args.v:
print(statement)
statements.append(statement)
db.bulkUpdate('vendor', statements)
#update database info after successful program-run
db.setColUpdate('vendor', last_modified)
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cwedict))
lastmodified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified('cwe')
if i is not None:
if lastmodified == i:
print("Not modified")
sys.exit(0)
# preparing xml by saving in a tempfile and unzipping
tmpdir = tempfile.gettempdir()
tmpfile = tempfile.NamedTemporaryFile()
cwezip = open(tmpfile.name, 'wb')
cwezip.write(f.read())
cwezip.close()
with zipfile.ZipFile(tmpfile.name) as z:
z.extractall(tmpdir)
z.close()
f = open(os.path.join(tmpdir, 'cwec_v2.8.xml'))
# parse xml and store in database
parser.parse(f)
cweList=[]
for cwe in progressbar(ch.cwe):
cwe['description_summary']=cwe['description_summary'].replace("\t\t\t\t\t", " ")
if args.v:
print (cwe)
cweList.append(cwe)
db.bulkUpdate('cwe', cweList)
#update database info after successful program-run
db.setColUpdate('cwe', lastmodified)
self.vendor[-1]['statement'] = self.statement
# dictionary
vendordict = Configuration.getVendorDict()
# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
(f, r) = Configuration.getFile(vendordict, compressed = True)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict))
i = db.getLastModified('vendor')
if i is not None:
if r.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
statements=[]
for statement in progressbar(ch.vendor):
if args.v:
print (statement)
statements.append(statement)
db.bulkUpdate('vendor', statements)
#update database info after successful program-run
db.setColUpdate('vendor', r.headers['last-modified'])
parser.setContentHandler(ch)
# check modification date
try:
(f, r) = Configuration.getFeedData('cwe')
except Exception as e:
print(e)
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(Configuration.getFeedURL("cwe")))
lastmodified = parse_datetime(r.headers['last-modified'], ignoretz=True)
i = db.getLastModified('cwe')
if i is not None and not args.f:
if lastmodified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
cweList = []
for cwe in progressbar(ch.cwe):
cwe['description_summary'] = cwe['description_summary'].replace(
"\t\t\t\t\t", " ")
if args.v:
print(cwe)
cweList.append(cwe)
db.bulkUpdate('cwe', cweList)
#update database info after successful program-run
db.setColUpdate('cwe', lastmodified)
try:
redis.info()
except:
sys.exit("Redis server not running on %s:%s"%(Configuration.getRedisHost(),Configuration.getRedisPort()))
except Exception as e:
print(e)
sys.exit(1)
try:
(f, r) = Configuration.getFeedData('via4')
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getFeedURL("via4")))
# check modification date
lastmodified = parse_datetime(r.headers['last-modified'], ignoretz=True)
i=db.getLastModified("via4")
if i is not None:
if lastmodified == i:
print("Not modified")
sys.exit(0)
data = json.loads(f.read().decode('utf-8'))
cves = data['cves']
bulk = [dict(val, id=key) for key, val in cves.items() if key]
db.bulkUpdate('via4', bulk)
db.setColInfo('via4', 'sources', data['metadata']['sources'])
db.setColInfo('via4', 'searchables', data['metadata']['searchables'])
#update database info after successful program-run
db.setColUpdate('via4', lastmodified)
d2securl = Configuration.getd2secDict()
# make parser
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(d2securl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(d2securl))
last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified("d2sec")
if i is not None:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
exploitList = []
for exploit in progressbar(ch.d2sec):
print(exploit)
if args.v:
print(exploit)
exploitList.append(exploit)
db.bulkUpdate("d2sec", exploitList)
#update database info after successful program-run
db.setColUpdate('d2sec', last_modified)
bulk = []
for entry in progressbar(misp_last['response']):
# Get info
event = entry['Event']
attrs = event['Attribute']
CVEs = [x['value'] for x in attrs if x['type'] == 'vulnerability']
if len(CVEs) == 0: continue
threats = [
x['value'] for x in attrs
if x['category'] == 'Attribution' and x['type'] == 'threat-actor'
]
tags = [
x['value'] for x in attrs
if x['category'] == 'Other' and x['type'] == 'text'
]
tags.extend([
x['value'] for x in attrs
if x['category'] == 'External analysis' and x['type'] == 'text'
])
# Add info to each CVE
for cve in CVEs:
item = {'id': cve}
if len(threats) != 0: item['threats'] = threats
if len(tags) != 0: item['tags'] = tags
if len(item.keys()) > 1: bulk.append(item) # Avoid empty collections
db.bulkUpdate("user_misp", bulk)
#update database info after successful program-run
db.setColUpdate('user_misp', now.strftime("%a, %d %h %Y %H:%M:%S %Z"))
(Configuration.getRedisHost(), Configuration.getRedisPort()))
except Exception as e:
print(e)
sys.exit(1)
try:
(f, r) = Configuration.getFeedData('via4')
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(Configuration.getFeedURL("via4")))
# check modification date
lastmodified = parse_datetime(r.headers['last-modified'], ignoretz=True)
i = db.getLastModified("via4")
db.setColUpdateCurrentTime('via4')
if i is not None:
if lastmodified == i:
print("Not modified")
sys.exit(0)
data = json.loads(f.read().decode('utf-8'))
cves = data['cves']
bulk = [dict(val, id=key) for key, val in cves.items() if key]
db.bulkUpdate('via4', bulk)
db.setColInfo('via4', 'sources', data['metadata']['sources'])
db.setColInfo('via4', 'searchables', data['metadata']['searchables'])
#update database info after successful program-run
db.setColUpdate('via4', lastmodified)
db.setColUpdateCurrentTime('via4')
if name == 'capec:Attack_Pattern_Catalog':
self.Attack_Pattern_Catalog_tag = False
# dictionary
capecurl = Configuration.getCAPECDict()
# make parser
parser = make_parser()
ch = CapecHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(capecurl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl))
i = db.getLastModified('capec')
last_modified = parse_datetime(f.headers['last-modified'])
if i is not None:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
attacks=[]
for attack in progressbar(ch.capec):
attacks.append(attack)
db.bulkUpdate("capec", attacks)
#update database info after successful program-run
db.setColUpdate('capec', last_modified)
# dictionary
d2securl = Configuration.getd2secDict()
# make parser
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(d2securl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(d2securl))
last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified("d2sec")
if i is not None:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
exploitList=[]
for exploit in progressbar(ch.d2sec):
print (exploit)
if args.v:
print (exploit)
exploitList.append(exploit)
db.bulkUpdate("d2sec", exploitList)
#update database info after successful program-run
db.setColUpdate('d2sec', last_modified)
try:
f = Configuration.getFile(exploitdburl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(exploitdburl))
i = db.getLastModified('exploitdb')
if i is not None:
if f.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
if not os.path.exists(tmppath):
os.mkdir(tmppath)
csvfile = tmppath+'/exploitdb.csv'
with open(csvfile, 'wb') as fp:
shutil.copyfileobj(f, fp)
fp.close()
exploits=[]
with open(csvfile, newline='') as csvtoparse:
exploitcsv = csv.DictReader(csvtoparse, delimiter=',')
for row in exploitcsv:
exploits.append(row)
if args.v:
print ("{} ({}) Imported".format(row['id'],row['description']))
db.bulkUpdate('exploitdb', exploits)
# Update last-modified
db.setColUpdate('exploitdb', f.headers['last-modified'])
f = Configuration.getFile(exploitdburl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(exploitdburl))
i = db.getLastModified('exploitdb')
if i is not None:
if f.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
if not os.path.exists(tmppath):
os.mkdir(tmppath)
csvfile = tmppath + '/exploitdb.csv'
with open(csvfile, 'wb') as fp:
shutil.copyfileobj(f, fp)
fp.close()
exploits = []
with open(csvfile, newline='', encoding='utf-8') as csvtoparse:
exploitcsv = csv.DictReader(csvtoparse, delimiter=',')
for row in exploitcsv:
exploits.append(row)
if args.v:
print("{} ({}) Imported".format(row['id'], row['description']))
db.bulkUpdate('exploitdb', exploits)
# Update last-modified
db.setColUpdate('exploitdb', f.headers['last-modified'])
if 'message' in misp_last.keys():
if misp_last['message'] == 'No matches':
sys.exit(0)
elif misp_last['message'].startswith('Authentication failed.'):
print("MISP Authentication failed")
sys.exit(1)
if not 'response' in misp_last:
print("Error occured while fetching MISP data")
sys.exit(1)
bulk =[]
for entry in progressbar(misp_last['response']):
# Get info
event=entry['Event']
attrs=event['Attribute']
CVEs= [x['value'] for x in attrs if x['type'] == 'vulnerability']
if len(CVEs) == 0: continue
threats= [x['value'] for x in attrs if x['category'] == 'Attribution' and x['type'] == 'threat-actor']
tags = [x['value'] for x in attrs if x['category'] == 'Other' and x['type'] == 'text']
tags.extend([x['value'] for x in attrs if x['category'] == 'External analysis' and x['type'] == 'text'])
# Add info to each CVE
for cve in CVEs:
item={'id':cve}
if len(threats) !=0: item['threats'] = threats
if len(tags) !=0: item['tags'] = tags
if len(item.keys())>1: bulk.append(item) # Avoid empty collections
db.bulkUpdate("user_misp", bulk)
#update database info after successful program-run
db.setColUpdate('user_misp', now.strftime("%a, %d %h %Y %H:%M:%S %Z"))
