def setup_module():
# create test account and collections, with appropriate permissions
admindb = db.ExistDB(server_url=EXISTDB_SERVER_URL,
username=EXISTDB_SERVER_ADMIN_USER,
password=EXISTDB_SERVER_ADMIN_PASSWORD)
# create non-admin test account
admindb.create_group(EXISTDB_TEST_GROUP)
admindb.create_account(EXISTDB_SERVER_USER, EXISTDB_SERVER_PASSWORD,
EXISTDB_TEST_GROUP)
admindb.createCollection('/db' + EXISTDB_TEST_BASECOLLECTION, True)
# test index config
test_cfg_collection = '/db/system/config/db/' + EXISTDB_TEST_BASECOLLECTION
admindb.createCollection(test_cfg_collection, True)
# make both collections owned by test group and group writable
admindb.query('sm:chgrp(xs:anyURI("/db%s"), "%s")' % \
(EXISTDB_TEST_BASECOLLECTION, EXISTDB_TEST_GROUP))
admindb.query('sm:chgrp(xs:anyURI("%s"), "%s")' % \
(test_cfg_collection, EXISTDB_TEST_GROUP))
admindb.query('sm:chmod(xs:anyURI("/db%s"), "rwxrwxrwx")' % \
(EXISTDB_TEST_BASECOLLECTION))
admindb.query('sm:chmod(xs:anyURI("%s"), "rwxrwxrwx")' % \
(test_cfg_collection))
def import_disa_iavm_cve():
flist=[]
exdb = db.ExistDB()
validateCollection(exdb,db_iavm_cve_disa_collection)
validateDataPath(iavm_cve_data_dir)
urls=[]
urls.append(("http://iasecontent.disa.mil/stigs/xml/iavm-to-cve%28u%29.xml","u_iavm-to-cve.xml"))
# -----------------------------------------------------------------------------
# download files even if they exist; NIST is constantly updating them
# -----------------------------------------------------------------------------
for url in urls:
u = url[0]
uname = url[1]
# if file does not exist, download
if (os.access(iavm_cve_data_dir, os.W_OK)):
try:
urllib.urlretrieve (u, iavm_cve_data_dir+uname)
try:
fo = open(iavm_cve_data_dir+uname, 'rb')
try:
if exdb.load(fo, db_iavm_cve_disa_collection+'/'+uname, True):
flist.append(uname+": data import successful")
except:
flist.append(uname+": data import failed")
fo.close()
except:
flist.append(uname+": file read failed")
except:
flist.append(uname+": file download failed")
else:
flist.append(uname+": file write failed")
return flist
def iavm_to_cpe_doc(flag=False):
exdb = db.ExistDB()
validateCollection(exdb, iavm_to_cpe_coll)
thisdoc = exdb.getDocument(iavm_to_cve_path)
root = etree.fromstring(thisdoc)
# get all IAVMs
find = etree.XPath('//IAVM')
iavms = find(root)
for i in range(len(iavms)):
print(iavms[i].find('./S').get('IAVM'))
if ((flag and iavms[i].find('./S').get('IAVM').startswith('2015-'))
or (not flag)):
iav_title = iavms[i].find('./S').get('Title')
cves = iavms[i].findall('./CVEs/CVENumber')
for j in range(len(cves)):
cpes = getCpeFromCve(cves[j].text)
if len(cpes) > 0:
cves[j].append(etree.Element('CPEs'))
cpes_elem = cves[j].find('./CPEs')
for v in cpes:
# apply vendor/reference filter
if filter_iavm_references(
v[0], iavms[i].find("./References")):
cpes_elem.append(etree.Element('Vendor', id=v[0]))
v_elem = cpes_elem.findall('./Vendor')[-1]
for cpe in v[1]:
# apply cpe/title filter
if filter_iavm_title(cpe, iav_title):
v_elem.append(etree.Element('CPE'))
this = v_elem.findall('./CPE')[-1]
this.text = cpe
# rewrite the above loop to just query once for each cve in the IAVMs
# create a dict for each cve with cpe values
# get all IAVMs
# this is 1800 CVEs at roughly 1.5 seconds = 45 minutes.
'''
find = etree.XPath('//CVENumber')
cvenodes = find(root)
cves=[]
for j in range(len(cvenodes)):
cves.append(cvenodes[j].text)
cves = list(set(cves)) # create unique list
cves.sort()
cve2cpe = {} # dictionary
for cve in cves:
print cve
cve2cpe[cve]=getCpeFromCve(cve)
'''
# print(etree.tostring(iavms[0],pretty_print=True))
# write xml doc to database
exdb.load(etree.tostring(root, pretty_print=True), iavm_to_cpe_path, True)
def importAdobeHtml():
exdb = db.ExistDB()
validateCollection(exdb, db_cvrf_adobe_collection)
flist = []
produrls = [
"https://helpx.adobe.com/security/products/reader.html",
"https://helpx.adobe.com/security/products/flash-player.html",
"https://helpx.adobe.com/security/products/shockwave.html",
]
apslinks = []
for pu in produrls:
content = urllib.urlopen(pu).read()
doc = fromstring(content)
doc.make_links_absolute("https://helpx.adobe.com")
for a in doc.xpath('//a[contains(@href,"/aps")]'):
apslinks.append(a.get('href'))
apslinks = list(set(list(apslinks)))
apslinks.sort()
# -----------------------------------------------------------------------------
# download files if they don't exist
# TODO: check for revisions and download those as well (check hashes?)
# -----------------------------------------------------------------------------
for u in apslinks:
uname = u.split('/')[-1]
# if file does not exist, download
if (not os.path.isfile(uname) and os.access(".", os.W_OK)):
print("downloading " + uname)
headers = {'User-Agent': 'Mozilla/5.0'}
req = urllib2.Request(u, None, headers)
#cvrfhtml = urllib2.urlopen(req).read()
urllib.urlretrieve(u, adobe_data_dir + uname)
#f = open(uname,'w')
#f.write(cvrfhtml)
#f.close()
try:
if int(uname[4:6]) > 13:
try:
translateAdobeHtmlToCvrf(adobe_data_dir + uname)
try:
cvrfname = uname.replace(".html", "-cvrf.xml")
fo = open(adobe_data_dir + cvrfname, 'rb')
if exdb.load(
fo,
db_cvrf_adobe_collection + '/' + cvrfname,
True):
flist.append(cvrfname +
": data import successful")
else:
flist.append(cvrfname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed")
except:
print("failed to translate: " + uname)
except:
pass
def import_cisco_cvrf():
flist = []
exdb = db.ExistDB()
validateCollection(exdb, db_cvrf_cisco_collection)
# -----------------------------------------------------------------------------
# get list of cvrf urls
# -----------------------------------------------------------------------------
nurl = "http://tools.cisco.com/security/center/cvrfListing.x"
request = urllib2.Request(nurl)
rawPage = urllib2.urlopen(request)
read = rawPage.read()
#print read
tree = etree.HTML(read)
tpath = "//a[contains(@href,'cvrf.xml')]"
findall = etree.ETXPath(tpath)
arefs = findall(tree)
urls = []
for a in arefs:
urls.append(a.get('href').replace('\t', '').replace('\n', ''))
# just for tracking for now, need to get cisco to fix or apply a fix
# i might ignore if it wasn't for poodle
badfiles = [
"/cisco-sa-20040420-tcp-nonios_cvrf.xml",
"cisco-sa-20120328-msdp_cvrf.xml",
"cisco-sa-20141015-poodle_cvrf.xml",
]
# -----------------------------------------------------------------------------
# download files if they don't exist
# -----------------------------------------------------------------------------
for u in urls:
uname = u.split('/')[-1]
# if file does not exist, download
#if (not os.path.isfile(cisco_data_dir+uname) and os.access(".", os.W_OK)):
if (os.access(".", os.W_OK)):
try:
print("downloading " + uname)
urllib.urlretrieve(u, cisco_data_dir + uname)
try:
fo = open(cisco_data_dir + uname, 'rb')
if exdb.load(fo, db_cvrf_cisco_collection + '/' + uname,
True):
flist.append(uname + ": data import successful")
else:
flist.append(uname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed")
except:
flist.append(uname + ": file download failed")
else:
flist.append(uname + ": file write failed")
return flist
def parse_disa_iavm_zip(fn):
if not disa_pki_flag:
return []
flist=[]
filen=open(fn,"rb")
exdb = db.ExistDB()
validateCollection(exdb,db_iavm_disa_collection)
#logger.debug(': '.join(['parse_zip',filen.name]))
#create zipfile object from passed in zip file object
z_file = zipfile.ZipFile(filen)
#create temporary directory
f_name = filen.name.split('/')[-1]
dir_name = f_name.replace('.zip', '')
tmp_dir = root_src_dir + dir_name + '/'
#logger.info(tmp_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
#extract files to tmp dir
z_file.extractall(tmp_dir)
# walk files in dir and add to database
# ValueError: too many values to unpack
# for root, dirs, files in os.walk(tmp_dir):
for src in glob.glob(tmp_dir+'/*/*.xml'):
print src
f = src.split(os.sep)[-1].split()[0]+'.xml'
#move tmp files to permanent location
#TODO: use static definition
dst = iavm_data_dir+f
try:
if os.path.exists(dst):
os.remove(dst)
shutil.move(src, dst)
#logger.debug(': '.join(['move_iavm',src, dst]))
#parse_xml(root+'/'+f) this is where I database boogie!
fo = open(dst, 'rb')
try:
if exdb.load(fo, db_iavm_disa_collection+'/'+f, True):
flist.append(f+": data import successful")
except:
flist.append(f+": data import failed")
fo.close()
except:
#logger.debug(': '.join(['move_iavm', 'FAILED',src, dst]))
flist.append(f+": file upload failed")
pass
flist.reverse()
return flist
def parse_msrc_cvrf_zip(fn):
flist = []
filen = open(fn, "rb")
exdb = db.ExistDB()
validateCollection(exdb, db_cvrf_microsoft_collection)
#logger.debug(': '.join(['parse_zip',filen.name]))
#create zipfile object from passed in zip file object
z_file = zipfile.ZipFile(filen)
#create temporary directory
f_name = filen.name.split('/')[-1]
dir_name = f_name.replace('.zip', '')
tmp_dir = root_src_dir + dir_name + '/'
#logger.info(tmp_dir)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
#extract files to tmp dir
z_file.extractall(tmp_dir)
#walk files in dir and add to database
#20140709rb: this is an awkward construction
for root, dirs, files in os.walk(tmp_dir):
for f in files:
#move tmp files to permanent location
#TODO: use static definition
src = root + '/' + f
dst = ms_data_dir + f
try:
if f[-4:].lower() == '.xml':
if os.path.exists(dst):
os.remove(dst)
shutil.move(src, dst)
#logger.debug(': '.join(['move_iavm',src, dst]))
#parse_xml(root+'/'+f) this is where I database boogie!
fo = open(dst, 'rb')
if exdb.load(fo, db_cvrf_microsoft_collection + '/' + f,
True):
flist.append(f + ": data import successful")
else:
flist.append(f + ": data import failed")
fo.close()
except:
#logger.debug(': '.join(['move_iavm', 'FAILED',src, dst]))
flist.append(f + ": file upload failed")
pass
flist.reverse()
return flist
def teardown_module():
# remove test account & collections
admindb = db.ExistDB(server_url=EXISTDB_SERVER_URL,
username=EXISTDB_SERVER_ADMIN_USER,
password=EXISTDB_SERVER_ADMIN_PASSWORD)
test_cfg_collection = '/db/system/config/db/' + EXISTDB_TEST_BASECOLLECTION
admindb.removeCollection(test_cfg_collection)
admindb.removeCollection(EXISTDB_TEST_BASECOLLECTION)
admindb.query('sm:remove-group("%s")' % EXISTDB_TEST_GROUP)
admindb.query('sm:remove-group("%s")' % EXISTDB_SERVER_USER)
admindb.query('sm:remove-account("%s")' % EXISTDB_SERVER_USER)
def import_redhat_cvrf():
flist = []
exdb = db.ExistDB()
validateCollection(exdb, db_cvrf_redhat_collection)
# -----------------------------------------------------------------------------
# get list of cvrf urls
# -----------------------------------------------------------------------------
# need User-Agent or Red Hat blocks request
nurl = "http://www.redhat.com/security/data/cvrf/index.txt"
headers = {'User-Agent': 'Mozilla/5.0'}
req = urllib2.Request(nurl, None, headers)
index = urllib2.urlopen(req).readlines()
urls = [
'http://www.redhat.com/security/data/cvrf/' + i.replace('\n', '')
for i in index
]
# -----------------------------------------------------------------------------
# download files if they don't exist
# TODO: check for revisions and download those as well (check hashes?)
# -----------------------------------------------------------------------------
for u in urls:
uname = u.split('/')[-1]
# if file does not exist, download
#if (not os.path.isfile(redhat_data_dir+uname) and os.access(redhat_data_dir, os.W_OK)):
if (os.access(redhat_data_dir, os.W_OK)):
try:
headers = {'User-Agent': 'Mozilla/5.0'}
req = urllib2.Request(u, None, headers)
cvrfxml = urllib2.urlopen(req).read()
#urllib.urlretrieve (u, redhat_data_dir+uname)
f = open(redhat_data_dir + uname, 'w')
f.write(cvrfxml)
f.close()
try:
fo = open(redhat_data_dir + uname, 'rb')
if exdb.load(fo, db_cvrf_redhat_collection + '/' + uname,
True):
flist.append(uname + ": data import successful")
else:
flist.append(uname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed")
except:
flist.append(uname + ": file download failed")
else:
flist.append(uname + ": file write failed")
return flist
def import_oracle_cvrf():
flist = []
exdb = db.ExistDB()
validateCollection(exdb, db_cvrf_oracle_collection)
# -----------------------------------------------------------------------------
# get list of cvrf urls
# -----------------------------------------------------------------------------
nurl = "http://www.oracle.com/ocom/groups/public/@otn/documents/webcontent/1932662.xml"
request = urllib2.Request(nurl)
rawPage = urllib2.urlopen(request)
read = rawPage.read()
#print read
root = etree.fromstring(read)
arefs = root.xpath("//link/text()")
urls = []
for a in arefs:
if "@otn" in a:
urls.append(a.replace('\t', '').replace('\n', '').replace(' ', ''))
# -----------------------------------------------------------------------------
# download files if they don't exist
# -----------------------------------------------------------------------------
for u in urls:
uname = u.split('/')[-1]
# if file does not exist, download
#if (not os.path.isfile(oracle_data_dir+uname) and os.access(oracle_data_dir, os.W_OK)):
if (os.access(oracle_data_dir, os.W_OK)):
try:
urllib.urlretrieve(u, oracle_data_dir + uname)
try:
fo = open(oracle_data_dir + uname)
if exdb.load(fo, db_cvrf_oracle_collection + '/' + uname,
True):
flist.append(uname + ": data import successful")
else:
flist.append(uname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed : " +
oracle_data_dir + uname)
except:
flist.append(uname + ": file download failed : " + u)
else:
flist.append(uname + ": file write failed : " + oracle_data_dir +
uname)
return flist
def import_redhat_oval():
flist = []
exdb = db.ExistDB()
# download
download_redhat_oval()
# get list of file names
for fname in glob.glob(redhat_oval_dir + "com.redhat.rhsa-*.xml"):
uname = fname.split(os.sep)[-1]
try:
fo = open(redhat_oval_dir + uname, 'rb')
if exdb.load(fo, db_oval_redhat_collection + '/' + uname, True):
flist.append(uname + ": data import successful")
else:
flist.append(uname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed")
return flist
def import_redhat_cpe():
flist = []
exdb = db.ExistDB()
validateCollection(exdb, db_cpe_redhat_collection)
validateDataPath(cpe_redhat_data_dir)
urls = []
urls.append(
"https://www.redhat.com/security/data/metrics/cpe-dictionary.xml")
# -----------------------------------------------------------------------------
# download files even if they exist; NIST is constantly updating them
# -----------------------------------------------------------------------------
for u in urls:
uname = u.split('/')[-1]
# if file does not exist, download
if (os.access(cpe_nist_data_dir, os.W_OK)):
try:
urllib.urlretrieve(u, cpe_redhat_data_dir + uname)
headers = {'User-Agent': 'Mozilla/5.0'}
req = urllib2.Request(u, None, headers)
cpexml = urllib2.urlopen(req).read()
#urllib.urlretrieve (u, cpe_redhat_data_dir+uname)
f = open(cpe_redhat_data_dir + uname, 'w')
f.write(cpexml)
f.close()
try:
fo = open(cpe_redhat_data_dir + uname, 'rb')
try:
if exdb.load(fo,
db_cpe_redhat_collection + '/' + uname,
True):
flist.append(uname + ": data import successful")
except:
flist.append(uname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed")
except:
flist.append(uname + ": file download failed")
else:
flist.append(uname + ": file write failed")
return flist
def import_nist_cce():
flist=[]
exdb = db.ExistDB()
validateCollection(exdb,db_cce_nist_collection)
validateDataPath(cce_data_dir)
urls=[]
urls.append("http://static.nvd.nist.gov/feeds/xml/cce/nvdcce-0.1-feed.xml.zip")
# -----------------------------------------------------------------------------
# download files even if they exist; NIST is constantly updating them
# -----------------------------------------------------------------------------
for u in urls:
uname = u.split('/')[-1]
# if file does not exist, download
if (os.access(cce_data_dir, os.W_OK)):
try:
urllib.urlretrieve (u, cce_data_dir+uname)
# unzip in place
zip = ZipFile(cce_data_dir+uname)
zip.extractall(cce_data_dir)
zip.close()
# remove zip file
os.remove(cce_data_dir+uname)
try:
xname = uname.replace('.zip','')
fo = open(cce_data_dir+xname, 'rb')
try:
if exdb.load(fo, db_cce_nist_collection+'/'+xname, True):
flist.append(xname+": data import successful")
except:
flist.append(xname+": data import failed")
fo.close()
except:
flist.append(uname+": file read failed")
except:
flist.append(uname+": file download failed")
else:
flist.append(uname+": file write failed")
return flist
def import_nist_cpe():
flist = []
exdb = db.ExistDB()
validateCollection(exdb, db_cpe_nist_collection)
validateDataPath(cpe_nist_data_dir)
urls = []
urls.append(
"http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml"
)
urls.append(
"http://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml"
)
# -----------------------------------------------------------------------------
# download files even if they exist; NIST is constantly updating them
# -----------------------------------------------------------------------------
for u in urls:
uname = u.split('/')[-1]
# if file does not exist, download
if (os.access(cpe_nist_data_dir, os.W_OK)):
try:
urllib.urlretrieve(u, cpe_nist_data_dir + uname)
try:
fo = open(cpe_nist_data_dir + uname, 'rb')
if exdb.load(fo, db_cpe_nist_collection + '/' + uname,
True):
flist.append(uname + ": data import successful")
else:
flist.append(uname + ": data import failed")
fo.close()
except:
flist.append(uname + ": file read failed")
except:
flist.append(uname + ": file download failed")
else:
flist.append(uname + ": file write failed")
return flist
def download_redhat_oval():
exdb = db.ExistDB()
validateCollection(exdb, db_oval_redhat_collection)
try:
dlurl = "https://www.redhat.com/security/data/oval/rhsa.tar.bz2"
headers = {'User-Agent': 'Mozilla/5.0'}
req = urllib2.Request(dlurl, None, headers)
dlreq = urllib2.urlopen(req)
CHUNK = 16 * 1024
with open(redhat_oval_dir + 'rhsa.tar.bz2', 'wb') as fp:
while True:
chunk = dlreq.read(CHUNK)
if not chunk: break
fp.write(chunk)
try:
tar = tarfile.open(redhat_oval_dir + 'rhsa.tar.bz2', 'r:bz2')
tar.extractall(redhat_oval_dir)
except:
print("import_redhat_oval: failed to uncompress tar.bz2 file")
except:
print("import_redhat_oval: failed to download file")
return
def importJuniperHtml():
exdb = db.ExistDB()
validateCollection(exdb,db_cvrf_juniper_collection)
flist=[]
produrls = [
"http://kb.juniper.net/InfoCenter/index?page=content&channel=SECURITY_ADVISORIES&cat=SIRT_ADVISORY&&actp=&sort=datemodified&dir=descending&max=1000&batch=1000&rss=true&itData.offset=0",
]
apslinks=[]
for pu in produrls:
content = urllib.urlopen(pu).read()
doc = fromstring(content)
doc.make_links_absolute("http://kb.juniper.net/InfoCenter/")
for a in doc.xpath('//a[contains(@href,"id=JSA") and contains(@href,"showDraft")]'):
apslinks.append(a.get('href'))
apslinks = list(set(list(apslinks)))
apslinks.sort()
# -----------------------------------------------------------------------------
# download files if they don't exist
# TODO: check for revisions and download those as well (check hashes?)
# -----------------------------------------------------------------------------
for u in apslinks:
uname = u.split('id=')[1].split('&')[0]+'.html'
# if file does not exist, download
if (not os.path.isfile(uname) and os.access(".", os.W_OK)):
print ("downloading "+uname)
headers = { 'User-Agent' : 'Mozilla/5.0' }
req = urllib2.Request(u, None, headers)
#cvrfhtml = urllib2.urlopen(req).read()
urllib.urlretrieve (u, juniper_data_dir+uname)
#f = open(uname,'w')
#f.write(cvrfhtml)
#f.close()
'''
def export_iavm_to_cpe_doc(fn):
exdb = db.ExistDB()
thisdoc = exdb.getDocument(iavm_to_cpe_path)
#f=open(fn,'w')
fn.write(thisdoc)
fn.close()
def __init__(self):
self.db = db.ExistDB()
def __init__(self):
self.db = db.ExistDB(server_url=EXISTDB_SERVER_URL)
def __init__(self):
self.db = db.ExistDB(server_url="http://localhost:8080/exist")
