def bulkvFeedUpdate(dbpath, vfeedmap):
# connect to sqlite db
con = sqlite3.connect(dbpath+'/vfeed.db')
con.text_factory = lambda x: x.decode("utf-8", "ignore")
c = con.cursor()
# loop over sqlite db and insert into vfeed collection
for vmap in progressbar(vfeedmap):
e = c.execute('SELECT * FROM %s' % vmap)
names = list(map(lambda x: x[0], e.description))
bulk = colVFEED.initialize_ordered_bulk_op()
for r in e:
try:
if vmap == 'map_redhat_bugzilla':
icveid = names.index('redhatid')
else:
icveid = names.index("cveid")
except Exception as ex:
sys.exit('Exeption in %s: %s' % (vmap, ex))
continue
mapArray={}
for i in range(0,len(r)):
if not (names[i] == "cveid"):
mapArray[str(names[i])]=str(r[i])
if not vmap=='map_redhat_bugzilla':
bulk.find({'id': r[icveid]}).upsert().update({"$set":{vmap:mapArray}})
else:
bulk.find({'map_cve_redhat.redhatid': r[icveid]}).update({"$set":{vmap:mapArray}})
bulk.execute()
def cveItemsProcess(type, url, args):
if args.v:
print("%s... downloading: %s" % (type, url))
file = Configuration.getFile(url)
try:
(f, r) = file
except:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?" %
_url)
# get your parser on !!
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
if args.u:
i = db.getInfo("cves")
if args.u:
last_modified = parse_datetime(r.headers['last-modified'],
ignoretz=True)
if i is not None:
if last_modified == i['last-modified']:
print("Not modified")
sys.exit(0)
db.setColUpdate("cves", last_modified)
if args.v:
u_counter = 0
n_counter = 0
for item in progressbar(ch.cves):
if 'cvss' not in item:
item['cvss'] = None
else:
item['cvss'] = float(item['cvss'])
if 'cwe' not in item:
item['cwe'] = defaultvalue['cwe']
# check if already exists
x = db.getCVE(item['id'])
# if so, update the entry.
if x:
if args.v: u_counter += 1
db.updateCVE(item)
else:
if args.v: n_counter += 1
db.insertCVE(item)
if args.v:
print("New: %s Updated: %s" % (n_counter, u_counter))
print("")
def index(limit=5, cpe_lookup=False, verbose=False):
if limit == 0: limit = -1
data = DatabaseLayer().CVE.last(limit=limit)
for cve in progressbar(data, prefix="Processing"):
writer = get_schema_writer()
title = cve.summary[:70]
data = cve.summary
if cpe_lookup:
for cpe in cve.vulnerable_configuration:
data += " " + cpe.title
if verbose:
print('Indexing CVE-ID ' + str(cve.id) + ' ' + title)
writer.update_document(title=title, path=cve.id, content=data)
writer.commit()
def onDatabaseUpdate(self):
lastUpdate = db.p_readSetting(self.collectionName, "last_update")
now = datetime.utcnow().replace(tzinfo = pytz.utc)
if lastUpdate:
last = dateutil.parser.parse(lastUpdate)
delta = now - last
since = "%sm"%math.ceil(delta.total_seconds()/60)
else:
since = ""
if self.url and self.key:
try:
# Misp interface
misp = PyMISP(self.url, self.key, True, 'json')
except:
return "[-] Failed to connect to MISP. Wrong URL?"
try:
# Fetch data
misp_last = misp.download_last(since)
# Check data
if 'message' in misp_last.keys():
if misp_last['message'].lower().startswith('no matches'): return "[+] MISP collection updated (0 updates)"
elif misp_last['message'].startswith('Authentication failed.'): return "[-] MISP Authentication failed"
if not 'response' in misp_last: print(misp_last); return "[-] Error occured while fetching MISP data"
# Nothing wrong so far, so let's continue
bulk =[]
for entry in progressbar(misp_last['response']):
# Get info
attrs=entry['Event']['Attribute']
CVEs= [x['value'] for x in attrs if x['type'] == 'vulnerability']
if len(CVEs) == 0: continue
threats= [x['value'] for x in attrs if x['category'] == 'Attribution' and x['type'] == 'threat-actor']
tags = [x['value'] for x in attrs if x['category'] == 'Other' and x['type'] == 'text']
tags.extend([x['value'] for x in attrs if x['category'] == 'External analysis' and x['type'] == 'text'])
# Add info to each CVE
for cve in CVEs:
item={'id':cve}
if len(threats) !=0: item['threats'] = threats
if len(tags) !=0: item['tags'] = tags
if len(item.keys())>1: bulk.append(item) # Avoid empty collections
db.p_bulkUpdate(self.collectionName, "id", bulk)
#update database info after successful program-run
db.p_writeSetting(self.collectionName, "last_update", now.strftime("%a, %d %h %Y %H:%M:%S %Z"))
return "[+] MISP collection updated (%s updates)"%len(bulk)
except Exception as e: print(e);print(e);return "[-] Something went wrong..."
else: return "[-] MISP credentials not specified"
# dictionary
capecurl = Configuration.getCAPECDict()
# connect to db
db = Configuration.getMongoConnection()
capec = db.capec
info = db.info
# make parser
parser = make_parser()
ch = CapecHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(capecurl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl))
i = info.find_one({'db': 'capec'})
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = capec.initialize_ordered_bulk_op()
for attack in progressbar(ch.capec):
bulk.find({'id': attack['id']}).upsert().update({"$set": {'name': attack['name'], 'summary': attack['summary'], 'prerequisites': attack['prerequisites'], 'solutions': attack['solutions'], 'related_weakness': attack['related_weakness']}})
bulk.execute()
#update database info after successful program-run
info.update({'db': 'capec'}, {"$set": {'last-modified': f.headers['last-modified']}}, upsert=True)
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cwedict))
lastmodified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified('cwe')
if i is not None:
if lastmodified == i:
print("Not modified")
sys.exit(0)
# preparing xml by saving in a tempfile and unzipping
tmpdir = tempfile.gettempdir()
tmpfile = tempfile.NamedTemporaryFile()
cwezip = open(tmpfile.name, 'wb')
cwezip.write(f.read())
cwezip.close()
with zipfile.ZipFile(tmpfile.name) as z:
z.extractall(tmpdir)
z.close()
f = open(os.path.join(tmpdir, 'cwec_v2.8.xml'))
# parse xml and store in database
parser.parse(f)
cweList=[]
for cwe in progressbar(ch.cwe):
cwe['description_summary']=cwe['description_summary'].replace("\t\t\t\t\t", " ")
if args.v:
print (cwe)
cweList.append(cwe)
db.bulkUpdate('cwe', cweList)
#update database info after successful program-run
db.setColUpdate('cwe', lastmodified)
if not indexed:
indexed = 0
if icve and icpeo:
if icpeo == icve:
print("Not modified")
sys.exit(0)
cves = db.CVE.query(skip=indexed, sort=("Published", "asc"))
if not cves:
print("Empty collections, import skipped")
sys.exit(2)
unique = set()
for cve in progressbar(cves):
for cpe in cve.vulnerable_configuration:
unique.add(cpe.id)
indexed_cpe = set()
for cpe in unique:
if db.CPE.get(cpe):
unique.add(cpe)
for cpe in indexed_cpe:
unique.remove(cpe)
if len(unique) > 0:
db.CPE.alternative_upsert([CPE(x) for x in unique])
#update database info after successful program-run
db.CVE.updated(icve)
db.CPE.alternative_updated(icve, (indexed + len(cves)))
parser.setContentHandler(ch)
# Retrieve CAPECs from the configuration's capec url
try:
print("[+] Getting CAPEC XML file")
(f, r) = Configuration.getFeedData("capec")
except Exception as e:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?"
% (Configuration.getFeedURL("capec"))
)
db_last_modified = db.getLastModified("capec")
last_modified = parse_datetime(r.headers["last-modified"], ignoretz=True)
if db_last_modified is not None:
if last_modified == db_last_modified:
print("Not modified")
sys.exit(0)
# Parse XML and store in database
parser.parse(f)
attacks = []
for attack in progressbar(ch.capec):
attacks.append(attack)
print("[+] %d attacks in XML file" % (len(attacks)))
db.bulkUpdate("capec", attacks)
# Update database info after successful program-run
db.setColUpdate("capec", last_modified)
db = Configuration.getMongoConnection()
d2sec = db.d2sec
info = db.info
# make parser
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(d2securl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(d2securl))
i = dbLayer.getLastModified("d2sec")
if i is not None:
if f.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = d2sec.initialize_ordered_bulk_op()
for exploit in progressbar(ch.d2sec):
print (exploit)
if args.v:
print (exploit)
bulk.find({'id': exploit['id']}).upsert().update({"$set": {'id': exploit['id'], 'url': exploit['url'], 'name': exploit['name']}})
bulk.execute()
#update database info after successful program-run
dbLayer.setColUpdate('d2sec', f.headers['last-modified'])
self.vendor[-1]['statement'] = self.statement
# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
(f, r) = Configuration.getFeedData('vendor')
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(Configuration.getFeedURL('vendor')))
last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True)
i = db.getLastModified('vendor')
if i is not None:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
statements = []
for statement in progressbar(ch.vendor):
if args.v:
print(statement)
statements.append(statement)
db.bulkUpdate('vendor', statements)
#update database info after successful program-run
db.setColUpdate('vendor', last_modified)
'map_cve_osvdb', 'map_cve_gentoo', 'map_cve_oval', 'map_cve_iavm',
'map_cve_redhat', 'map_cve_mandriva', 'map_cve_saint',
'map_cve_scip', 'map_cve_aixapar', 'map_cve_ms', 'map_cve_suse',
'map_cve_certvn', 'map_cve_msf', 'map_cve_ubuntu', 'map_cve_cisco',
'map_cve_mskb', 'map_redhat_bugzilla', 'map_cve_debian', 'map_cve_nmap',
'map_cve_nessus', 'map_cve_vmware', 'map_cve_suricata',
'map_cve_hp', 'map_cve_bid', 'map_cve_snort']
# connect to sqlite database
con = sqlite3.connect('./tmp/vfeed.db')
con.text_factory = lambda x: x.decode("utf-8", "ignore")
c = con.cursor()
vfeed = db.vfeed
# read sqlite database and store in mongo database
for vmap in progressbar(vfeedmap):
e = c.execute('SELECT * FROM %s' % vmap)
names = list(map(lambda x: x[0], e.description))
bulk = vfeed.initialize_ordered_bulk_op()
for r in e:
try:
if vmap == 'map_redhat_bugzilla':
icveid = names.index('redhatid')
else:
icveid = names.index("cveid")
except Exception as ex:
sys.exit('Exeption in %s: %s' % (vmap, ex))
continue
mapArray={}
for i in range(0,len(r)):
if not (names[i] == "cveid"):
# Fetch data
misp_last = misp.download_last(since)
# Check data
if 'message' in misp_last.keys():
if misp_last['message'] == 'No matches':
sys.exit(0)
elif misp_last['message'].startswith('Authentication failed.'):
print("MISP Authentication failed")
sys.exit(1)
if not 'response' in misp_last:
print("Error occured while fetching MISP data")
sys.exit(1)
bulk = []
for entry in progressbar(misp_last['response']):
# Get info
event = entry['Event']
attrs = event['Attribute']
CVEs = [x['value'] for x in attrs if x['type'] == 'vulnerability']
if len(CVEs) == 0: continue
threats = [
x['value'] for x in attrs
if x['category'] == 'Attribution' and x['type'] == 'threat-actor'
]
tags = [
x['value'] for x in attrs
if x['category'] == 'Other' and x['type'] == 'text'
]
tags.extend([
x['value'] for x in attrs
def dumpallcveid(entry=None):
cveid = []
if entry is None:
for x in collection.find({}).sort('_id', 1):
cveid.append(x['id'])
else:
for x in collection.find({}).sort("Modified", -1).limit(int(entry)):
cveid.append(x['id'])
return cveid
def getcve(cveid=None):
if cveid is None:
return False
return collection.find_one({'id': cveid})
for cveid in progressbar(dumpallcveid(entry=args.l), prefix="Processing"):
writer = ix.writer()
item = getcve(cveid=cveid)
title = item['summary'][0:70]
if args.n:
for v in item['vulnerable_configuration']:
cpe = c.getcpe(cpeid=v).strip('\n')
item['summary'] += " " + cpe
if args.v:
print('Indexing CVE-ID ' + str(cveid) + ' ' + title)
writer.update_document(title=title, path=cveid, content=item['summary'])
writer.commit()
os.mkdir(indexpath)
if not exists_in(indexpath):
ix = create_in(indexpath, schema)
else:
ix = open_dir(indexpath)
def dumpallcveid(entry=None):
return db.getCVEID if not entry else db.getCVEIDs(int(entry))
def getcve(cveid=None):
if cveid is None:
return False
return db.getCVE(cveid)
for cveid in progressbar(dumpallcveid(entry=args.l),prefix="Processing"):
try:
writer = ix.writer()
except:
print ("Index is locked. Another db_fulltext process running?")
sys.exit(1)
item = getcve(cveid=cveid)
title = item['summary'][0:70]
if args.n:
for v in item['vulnerable_configuration']:
cpe = c.getcpe(cpeid=v).strip('\n')
item['summary'] += " " + cpe
if args.v:
print ('Indexing CVE-ID ' + str(cveid) + ' ' + title)
writer.update_document(title=title, path=cveid, content=item['summary'])
writer.commit()
parser.setContentHandler(ch)
db = DatabaseLayer()
# check modification date
try:
(f, r) = Configuration.getFeedData('cwe')
except Exception as e:
print(e)
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(Configuration.getFeedURL("cwe")))
lastmodified = parse_datetime(r.headers['last-modified'], ignoretz=True)
i = db.CWE.updated()
if i is not None:
if lastmodified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
cweList = []
for cwe in progressbar(ch.cwe):
data = CWE(cwe['id'], cwe['name'],
cwe['description_summary'].replace("\t\t\t\t\t", " "),
cwe['status'], cwe['weaknessabs'])
if args.v:
print(cwe)
cweList.append(data)
db.CWE.upsert(cweList)
#update database info after successful program-run
db.CWE.updated(lastmodified)
'map_cve_osvdb', 'map_cve_gentoo', 'map_cve_oval', 'map_cve_iavm',
'map_cve_redhat', 'map_cve_mandriva', 'map_cve_saint',
'map_cve_scip', 'map_cve_aixapar', 'map_cve_ms', 'map_cve_suse',
'map_cve_certvn', 'map_cve_msf', 'map_cve_ubuntu', 'map_cve_cisco',
'map_cve_mskb', 'map_redhat_bugzilla', 'map_cve_debian', 'map_cve_nmap',
'map_cve_nessus', 'map_cve_vmware', 'map_cve_suricata',
'map_cve_hp', 'map_cve_bid', 'map_cve_snort']
# connect to sqlite database
con = sqlite3.connect(tmppath+'/vfeed.db')
con.text_factory = lambda x: x.decode("utf-8", "ignore")
c = con.cursor()
vfeed = db.vfeed
# read sqlite database and store in mongo database
for vmap in progressbar(vfeedmap):
e = c.execute('SELECT * FROM %s' % vmap)
names = list(map(lambda x: x[0], e.description))
bulk = vfeed.initialize_ordered_bulk_op()
for r in e:
try:
if vmap == 'map_redhat_bugzilla':
icveid = names.index('redhatid')
else:
icveid = names.index("cveid")
except Exception as ex:
sys.exit('Exeption in %s: %s' % (vmap, ex))
continue
mapArray={}
for i in range(0,len(r)):
if not (names[i] == "cveid"):
# only get collection of new CVE's
collections = []
if date:
collections = cve.find({'last-modified': {'$gt': icve['last-modified']}})
else:
collections = cve.find({})
# check cpes for cves and parse and store missing cpes in cpeother
batch = []
# skip on empty collections
col=list(collections)
if not col:
print ("Empty collections, import skipped")
sys.exit(2)
for item in progressbar(col):
for cpeentry in item['vulnerable_configuration']:
checkdup = cpeother.find(({'id': cpeentry}))
if checkdup.count() <= 0:
entry = cpe.find(({'id': cpeentry}))
if entry.count() <= 0:
title = cpeentry
title = title[10:]
title = title.replace(':-:', ' ',10)
title = title.replace(':', ' ',10)
title = title.replace('_', ' ',10)
title = urllib.parse.unquote_plus(title)
title = title.title()
batch.append({'id': cpeentry, 'title': title})
if len(batch) != 0:
if 'cwe' not in item:
item['cwe'] = defaultvalue['cwe']
db.updateCVE(item)
else:
db.insertCVE(item)
# get the 'recent' file
getfile = file_prefix + file_rec + file_suffix
try:
(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
for item in progressbar(ch.cves):
# check if the CVE already exists.
x = db.getCVE(item['id'])
# if so, update the entry.
if x:
if args.v:
print("item found : " + item['id'])
if 'cvss' not in item:
item['cvss'] = None
else:
item['cvss'] = float(item['cvss'])
if 'cwe' not in item:
item['cwe'] = defaultvalue['cwe']
db.updateCVE(item)
# if not, create it.
else:
d2securl = Configuration.getd2secDict()
# make parser
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(d2securl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(d2securl))
last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified("d2sec")
if i is not None:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
exploitList = []
for exploit in progressbar(ch.d2sec):
print(exploit)
if args.v:
print(exploit)
exploitList.append(exploit)
db.bulkUpdate("d2sec", exploitList)
#update database info after successful program-run
db.setColUpdate('d2sec', last_modified)
# dict
cpedict = Configuration.getCPEDict()
# make parser
parser = make_parser()
ch = CPEHandler()
parser.setContentHandler(ch)
# check modification date
try:
f = Configuration.getFile(cpedict)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cpedict))
i = db.getLastModified('cpe')
if i is not None:
if f.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
cpeList=[]
for x in progressbar(ch.cpe):
x['id']= toStringFormattedCPE(x['name'])
x['title']=x['title'][0]
x['cpe_2_2'] = x.pop('name')
if not x['references']: x.pop('references')
cpeList.append(x)
db.bulkUpdate("cpe", cpeList)
#update database info after successful program-run
db.setColUpdate('cpe', f.headers['last-modified'])
self.vendor[-1]['statement'] = self.statement
# dictionary
vendordict = Configuration.getVendorDict()
# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
(f, r) = Configuration.getFile(vendordict, compressed = True)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict))
i = db.getLastModified('vendor')
if i is not None:
if r.headers['last-modified'] == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
parser.parse(f)
statements=[]
for statement in progressbar(ch.vendor):
if args.v:
print (statement)
statements.append(statement)
db.bulkUpdate('vendor', statements)
#update database info after successful program-run
db.setColUpdate('vendor', r.headers['last-modified'])
# only get collection of new CVE's
collections = []
if date:
collections = db.getCVEsNewerThan(icve)
else:
collections = db.getCVEs()
# check cpes for cves and parse and store missing cpes in cpeother
batch = []
# skip on empty collections
col = list(collections)
if not col:
print("Empty collections, import skipped")
sys.exit(2)
for item in progressbar(col):
for cpeentry in item['vulnerable_configuration']:
checkdup = db.getAlternativeCPE(cpeentry)
if checkdup and len(checkdup) <= 0:
entry = db.getCPE(cpeentry)
if entry and len(entry.count) <= 0:
title = cpeentry
title = title[10:]
title = title.replace(':-:', ' ', 10)
title = title.replace(':', ' ', 10)
title = title.replace('_', ' ', 10)
title = urllib.parse.unquote_plus(title)
title = title.title()
batch.append({'id': cpeentry, 'title': title})
if len(batch) != 0:
info = db.info
# make parser
parser = make_parser()
ch = CPEHandler()
parser.setContentHandler(ch)
# check modification date
f = urlopen(cpedict)
i = info.find_one({'db': 'cpe'})
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
sys.exit("Not modified")
# parse xml and store in database
parser.parse(f)
bulk = cpe.initialize_ordered_bulk_op()
for x in progressbar(ch.cpe):
name = toStringFormattedCPE(x['name'])
oldCPE = x['name']
title = x['title'][0]
if x['references']:
bulk.find({
'id': name
}).upsert().update({
"$set": {
'title': title,
'cpe_2_2': oldCPE,
'references': x['references']
}
})
else:
bulk.find({
db.insertCVE(item)
# get the 'recent' file
getfile = file_prefix + file_rec + file_suffix
try:
(f,
r) = Configuration.getFile(Configuration.getCVEDict() + getfile,
compressed=True)
except:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?"
% (Configuration.getCVEDict() + getfile))
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
for item in progressbar(ch.cves):
# check if the CVE already exists.
x = db.getCVE(item['id'])
# if so, update the entry.
if x:
if args.v:
print("item found : " + item['id'])
if 'cvss' not in item:
item['cvss'] = None
else:
item['cvss'] = float(item['cvss'])
if 'cwe' not in item:
item['cwe'] = defaultvalue['cwe']
db.updateCVE(item)
# if not, create it.
else:
# Fetch data
misp_last = misp.download_last(since)
# Check data
if 'message' in misp_last.keys():
if misp_last['message'] == 'No matches':
sys.exit(0)
elif misp_last['message'].startswith('Authentication failed.'):
print("MISP Authentication failed")
sys.exit(1)
if not 'response' in misp_last:
print("Error occured while fetching MISP data")
sys.exit(1)
bulk =[]
for entry in progressbar(misp_last['response']):
# Get info
event=entry['Event']
attrs=event['Attribute']
CVEs= [x['value'] for x in attrs if x['type'] == 'vulnerability']
if len(CVEs) == 0: continue
threats= [x['value'] for x in attrs if x['category'] == 'Attribution' and x['type'] == 'threat-actor']
tags = [x['value'] for x in attrs if x['category'] == 'Other' and x['type'] == 'text']
tags.extend([x['value'] for x in attrs if x['category'] == 'External analysis' and x['type'] == 'text'])
# Add info to each CVE
for cve in CVEs:
item={'id':cve}
if len(threats) !=0: item['threats'] = threats
if len(tags) !=0: item['tags'] = tags
if len(item.keys())>1: bulk.append(item) # Avoid empty collections
db.bulkUpdate("user_misp", bulk)