def cveItemsProcess(type, url, args):
if args.v:
print("%s... downloading: %s" % (type, url))
file = Configuration.getFile(url)
try:
(f, r) = file
except:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?" %
_url)
# get your parser on !!
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
if args.u:
i = db.getInfo("cves")
if args.u:
last_modified = parse_datetime(r.headers['last-modified'],
ignoretz=True)
if i is not None:
if last_modified == i['last-modified']:
print("Not modified")
sys.exit(0)
db.setColUpdate("cves", last_modified)
if args.v:
u_counter = 0
n_counter = 0
for item in progressbar(ch.cves):
if 'cvss' not in item:
item['cvss'] = None
else:
item['cvss'] = float(item['cvss'])
if 'cwe' not in item:
item['cwe'] = defaultvalue['cwe']
# check if already exists
x = db.getCVE(item['id'])
# if so, update the entry.
if x:
if args.v: u_counter += 1
db.updateCVE(item)
else:
if args.v: n_counter += 1
db.insertCVE(item)
if args.v:
print("New: %s Updated: %s" % (n_counter, u_counter))
print("")
cve['vulnerable_configuration_cpe_2_2'] = []
return cve
if __name__ == '__main__':
if args.u:
# get the 'modified' file
getfile = file_prefix + file_mod + file_suffix
try:
(f, r) = Configuration.getFile(
Configuration.getFeedURL('cve') + getfile)
except:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?"
% (Configuration.getFeedURL("cve") + getfile))
i = db.getInfo("cves")
last_modified = parse_datetime(r.headers['last-modified'],
ignoretz=True)
if i is not None:
if last_modified == i['last-modified'] and not args.f:
print("Not modified")
sys.exit(0)
db.setColUpdate("cves", last_modified)
cvej = json.loads(f.read())
for cveitem in cvej['CVE_Items']:
item = process_cve_item(item=cveitem)
# check if the CVE already exists.
x = db.getCVE(item['id'])
# if so, update the entry.
if x:
if args.v:
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
# start here if it's an update.
if args.u:
# get the 'modified' file
getfile = file_prefix + file_mod + file_suffix
try:
(f,
r) = Configuration.getFile(Configuration.getCVEDict() + getfile,
compressed=True)
except:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?"
% (Configuration.getCVEDict() + getfile))
i = db.getInfo("cve")
if i is not None:
if r.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
db.setColUpdate("cve", r.headers['last-modified'])
# get your parser on !!
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
for item in ch.cves:
# check if the CVE already exists.
x = db.getCVE(item['id'])
# if so, update the entry.
sys.path.append(os.path.join(runPath, ".."))
from pymongo import TEXT
import lib.DatabaseLayer as dbLayer
def setIndex(col, field, printSuccess = True):
try:
dbLayer.ensureIndex(col, field)
if printSuccess:
print('[+]Success to create index %s on %s' % (field, col))
except Exception as e:
print('[-]Failed to create index %s on %s: %s' % (col, field, e))
setIndex('cpe', 'id')
setIndex('cpeother', 'id')
setIndex('cves', 'id')
setIndex('cves', 'vulnerable_configuration')
setIndex('cves', 'Modified')
setIndex('cves', [("summary",TEXT)])
setIndex('vendor', 'id')
setIndex('via4', 'id')
setIndex('mgmt_whitelist', 'id')
setIndex('mgmt_blacklist', 'id')
setIndex('capec', 'related_weakness')
via4 = dbLayer.getInfo('via4')
if via4:
for index in via4.get('searchables', []):
setIndex('via4', index, False)
self.inPUBElem = 0
self.cves[-1]['Modified'] = self.PUB
if __name__ == '__main__':
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
# start here if it's an update.
if args.u:
# get the 'modified' file
getfile = file_prefix + file_mod + file_suffix
try:
(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
i = db.getInfo("cve")
if i is not None:
if r.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
db.setColUpdate("cve", r.headers['last-modified'])
# get your parser on !!
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
for item in ch.cves:
# check if the CVE already exists.
x = db.getCVE(item['id'])
# if so, update the entry.
return (datetime.datetime(1899, 12, 30) +
datetime.timedelta(days=xldate + 1462 * datemode))
# dictionary
msbulletinurl = Configuration.getMSBULLETINDict()
tmppath = Configuration.getTmpdir()
try:
f = Configuration.getFile(msbulletinurl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(msbulletinurl))
# check modification date
i = dbLayer.getInfo("ms")
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
if not os.path.exists(tmppath):
os.mkdir(tmppath)
with open(tmppath + '/BulletinSearch.xlsx', 'wb') as fp:
shutil.copyfileobj(f, fp)
fp.close()
# parse xlsx and store in database
wb = xlrd.open_workbook(tmppath + '/BulletinSearch.xlsx')
sh = wb.sheet_by_name('Bulletin Search')
header = [s.replace('\n', ' ') for s in sh.row_values(0)]
from pymongo import TEXT
import lib.DatabaseLayer as dbLayer
def setIndex(col, field, printSuccess=True):
try:
dbLayer.ensureIndex(col, field)
if printSuccess:
print('[+]Success to create index %s on %s' % (field, col))
except Exception as e:
print('[-]Failed to create index %s on %s: %s' % (col, field, e))
setIndex('cpe', 'id')
setIndex('cpeother', 'id')
setIndex('cves', 'id')
setIndex('cves', 'vulnerable_configuration')
setIndex('cves', 'Modified')
setIndex('cves', [("summary", TEXT)])
setIndex('via4', 'id')
setIndex('mgmt_whitelist', 'id')
setIndex('mgmt_blacklist', 'id')
setIndex('capec', 'related_weakness')
via4 = dbLayer.getInfo('via4')
if via4:
for index in via4.get('searchables', []):
setIndex('via4', index, False)
sys.path.append(os.path.join(runPath, ".."))
from pymongo import TEXT
import lib.DatabaseLayer as dbLayer
def setIndex(col, field, printSuccess=True):
try:
dbLayer.ensureIndex(col, field)
if printSuccess:
print('[+]Success to create index %s on %s' % (field, col))
except Exception as e:
print('[-]Failed to create index %s on %s: %s' % (col, field, e))
setIndex('cpe', 'id')
setIndex('cpeother', 'id')
setIndex('cves', 'id')
setIndex('cves', 'vulnerable_configuration')
setIndex('cves', 'Modified')
setIndex('cves', [("summary", TEXT)])
setIndex('vendor', 'id')
setIndex('via4', 'id')
setIndex('mgmt_whitelist', 'id')
setIndex('mgmt_blacklist', 'id')
setIndex('capec', 'related_weakness')
for index in dbLayer.getInfo('via4').get('searchables', []):
setIndex('via4', index, False)
self.inPUBElem = 0
self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True)
if __name__ == '__main__':
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
# start here if it's an update.
if args.u:
# get the 'modified' file
getfile = file_prefix + file_mod + file_suffix
try:
(f, r) = Configuration.getFile(Configuration.getFeedURL('cve') + getfile)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getFeedURL("cve") + getfile))
i = db.getInfo("cves")
last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True)
if i is not None:
if last_modified == i['last-modified']:
print("Not modified")
sys.exit(0)
db.setColUpdate("cves", last_modified)
# get your parser on !!
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
for item in ch.cves:
# check if the CVE already exists.
x = db.getCVE(item['id'])
return (
datetime.datetime(1899, 12, 30)
+ datetime.timedelta(days=xldate + 1462 * datemode)
)
# dictionary
msbulletinurl = Configuration.getMSBULLETINDict()
tmppath = Configuration.getTmpdir()
try:
f = Configuration.getFile(msbulletinurl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(msbulletinurl))
# check modification date
i=dbLayer.getInfo("ms")
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
if not os.path.exists(tmppath):
os.mkdir(tmppath)
with open(tmppath+'/BulletinSearch.xlsx', 'wb') as fp:
shutil.copyfileobj(f, fp)
fp.close()
# parse xlsx and store in database
wb = xlrd.open_workbook(tmppath+'/BulletinSearch.xlsx')
# Use first of the index (name should be Export Bulletin Search Spreadsh)
redis = Configuration.getRedisRefConnection()
try:
redis.info()
except:
sys.exit("Redis server not running on %s:%s"%(Configuration.getRedisHost(),Configuration.getRedisPort()))
except Exception as e:
print(e)
sys.exit(1)
try:
(f, r) = Configuration.getFeedData('via4')
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getFeedURL("via4")))
# check modification date
i=db.getInfo("via4")
if i is not None:
if r.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
data = json.loads(f.read().decode('utf-8'))
cves = data['cves']
bulk = [dict(val, id=key) for key, val in cves.items() if key]
db.bulkUpdate('via4', bulk)
db.setColInfo('via4', 'sources', data['metadata']['sources'])
db.setColInfo('via4', 'searchables', data['metadata']['searchables'])
#update database info after successful program-run
db.setColUpdate('via4', r.headers['last-modified'])
# connect to the DB.
db = Configuration.getMongoConnection()
collection = db.cves
info = db.info
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
# start here if it's an update.
if args.u:
# get the 'modified' file
getfile = file_prefix + file_mod + file_suffix
try:
f = Configuration.getFile(Configuration.getCVEDict() + getfile)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
i = dbLayer.getInfo("cve")
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
dbLayer.setColUpdate("cve", f.headers['last-modified'])
# get your parser on !!
parser = make_parser()
ch = CVEHandler()
parser.setContentHandler(ch)
parser.parse(f)
for item in ch.cves:
# check if the CVE already exists.
x = dbLayer.getCVE(item['id'])
# if so, update the entry.
redis.info()
except:
sys.exit("Redis server not running on %s:%s" %
(Configuration.getRedisHost(), Configuration.getRedisPort()))
except Exception as e:
print(e)
sys.exit(1)
try:
(f, r) = Configuration.getFeedData('via4')
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
(Configuration.getFeedURL("via4")))
# check modification date
i = db.getInfo("via4")
if i is not None:
if r.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
data = json.loads(f.read().decode('utf-8'))
cves = data['cves']
bulk = [dict(val, id=key) for key, val in cves.items() if key]
db.bulkUpdate('via4', bulk)
db.setColInfo('via4', 'sources', data['metadata']['sources'])
db.setColInfo('via4', 'searchables', data['metadata']['searchables'])
#update database info after successful program-run
db.setColUpdate('via4', r.headers['last-modified'])
import sys
runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(runPath, ".."))
from pymongo import TEXT
import lib.DatabaseLayer as dbLayer
def setIndex(col, field, printSuccess = True):
try:
dbLayer.ensureIndex(col, field)
if printSuccess:
print('[+]Success to create index %s on %s' % (field, col))
except Exception as e:
print('[-]Failed to create index %s on %s: %s' % (col, field, e))
setIndex('cpe', 'id')
setIndex('cpeother', 'id')
setIndex('cves', 'id')
setIndex('cves', 'vulnerable_configuration')
setIndex('cves', 'Modified')
setIndex('cves', [("summary",TEXT)])
setIndex('vendor', 'id')
setIndex('via4', 'id')
setIndex('mgmt_whitelist', 'id')
setIndex('mgmt_blacklist', 'id')
setIndex('capec', 'related_weakness')
for index in dbLayer.getInfo('via4').get('searchables', []):
setIndex('via4', index, False)
