Beispiel #1
0
def cveItemsProcess(type, url, args):
    if args.v:
        print("%s... downloading: %s" % (type, url))

    file = Configuration.getFile(url)
    try:
        (f, r) = file
    except:
        sys.exit(
            "Cannot open url %s. Bad URL or not connected to the internet?" %
            _url)

    # get your parser on !!
    parser = make_parser()
    ch = CVEHandler()
    parser.setContentHandler(ch)
    parser.parse(f)

    if args.u:
        i = db.getInfo("cves")
        if args.u:
            last_modified = parse_datetime(r.headers['last-modified'],
                                           ignoretz=True)
            if i is not None:
                if last_modified == i['last-modified']:
                    print("Not modified")
                    sys.exit(0)
            db.setColUpdate("cves", last_modified)

    if args.v:
        u_counter = 0
        n_counter = 0

    for item in progressbar(ch.cves):
        if 'cvss' not in item:
            item['cvss'] = None
        else:
            item['cvss'] = float(item['cvss'])
        if 'cwe' not in item:
            item['cwe'] = defaultvalue['cwe']

        # check if already exists
        x = db.getCVE(item['id'])
        # if so, update the entry.
        if x:
            if args.v: u_counter += 1
            db.updateCVE(item)
        else:
            if args.v: n_counter += 1
            db.insertCVE(item)

    if args.v:
        print("New: %s Updated: %s" % (n_counter, u_counter))
        print("")
    def endElement(self, name):
        if name == 'nvd:statement':
            self.statementtag = False
            self.statement = self.statement + self.statement.rstrip()
            self.vendor[-1]['statement'] = self.statement

# dictionary
vendordict = Configuration.getVendorDict()

# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
    (f, r) = Configuration.getFile(vendordict, compressed = True)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict))
i = db.getLastModified('vendor')
if i is not None:
    if r.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
statements=[]
for statement in progressbar(ch.vendor):
    if args.v:
        print (statement)
    statements.append(statement)
db.bulkUpdate('vendor', statements)
# dictionary
vendordict = Configuration.getVendorDict()

# connect to db
db = Configuration.getMongoConnection()
vendor = db.vendor
info = db.info

# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(vendordict)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict))
i = info.find_one({'db': 'vendor'})
if i is not None:
    if f.headers['last-modified'] == i['last-modified']:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = vendor.initialize_ordered_bulk_op()
for statement in progressbar(ch.vendor):
    if args.v:
        print (statement)
    bulk.find({'id': statement['id']}).upsert().update({"$set": {'statement': statement['statement'], 'id': statement['id'], 'organization': statement['organization'], 'contributor': statement['contributor'], 'lastmodified': statement['lastmodified']}})
bulk.execute()
        version_info += cpe["versionEndExcluding"]
    if "versionEndIncluding" in item:
        cpe["versionEndIncluding"] = item["versionEndIncluding"]
        version_info += cpe["versionEndIncluding"]

    sha1_hash = hashlib.sha1(cpe["cpe_2_2"].encode("utf-8") +
                             version_info.encode("utf-8")).hexdigest()
    cpe["id"] = sha1_hash

    return cpe


if __name__ == '__main__':
    if args.u:
        try:
            (f, r) = Configuration.getFile(Configuration.getFeedURL('cpe'))
        except:
            sys.exit(
                "Cannot open url %s. Bad URL or not connected to the internet?"
                % (Configuration.getFeedURL("cpe")))

        # check modification date
        i = db.getLastModified('cpe')
        last_modified = parse_datetime(r.headers['last-modified'],
                                       ignoretz=True)
        if i is not None:
            if last_modified == i and not args.f:
                print("Not modified")
                sys.exit(0)

        cpej = json.loads(f.read())
Beispiel #5
0
        if name == 'vuln:last-modified-datetime':
            self.inPUBElem = 0
            self.cves[-1]['Modified'] = self.PUB


if __name__ == '__main__':
    parser = make_parser()
    ch = CVEHandler()
    parser.setContentHandler(ch)
    # start here if it's an update.
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            (f,
             r) = Configuration.getFile(Configuration.getCVEDict() + getfile,
                                        compressed=True)
        except:
            sys.exit(
                "Cannot open url %s. Bad URL or not connected to the internet?"
                % (Configuration.getCVEDict() + getfile))
        i = db.getInfo("cve")
        if i is not None:
            if r.headers['last-modified'] == i['last-modified']:
                print("Not modified")
                sys.exit(0)
        db.setColUpdate("cve", r.headers['last-modified'])

        # get your parser on !!
        parser = make_parser()
        ch = CVEHandler()
        parser.setContentHandler(ch)
    sys.exit("Redis server not running on %s:%s"%(Configuration.getRedisHost(),Configuration.getRedisPort()))


RefUrl = Configuration.getRefURL()
tmppath = Configuration.getTmpdir()

argparser = argparse.ArgumentParser(description='Populate/update the NIST ref database')
argparser.add_argument('-v', action='store_true', help='verbose output', default=False)
args = argparser.parse_args()

if args.v:
    verbose = True

# check modification date
try:
    u = Configuration.getFile(RefUrl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(RefUrl))
i = db.getLastModified('ref')
if i is not None:
    if u.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)

# Create temp file and download and unpack database
if not os.path.exists(tmppath):
    os.mkdir(tmppath)

with open(tmppath+'/allrefmaps.zip', 'wb') as fp:
    shutil.copyfileobj(u, fp)
Beispiel #7
0
# dictionary
d2securl = Configuration.getd2secDict()

# connect to db
db = Configuration.getMongoConnection()
d2sec = db.d2sec
info = db.info

# make parser
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(d2securl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(d2securl))
i = dbLayer.getLastModified("d2sec")
if i is not None:
    if f.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = d2sec.initialize_ordered_bulk_op()
for exploit in progressbar(ch.d2sec):
    print (exploit)
    if args.v:
        print (exploit)
    bulk.find({'id': exploit['id']}).upsert().update({"$set": {'id': exploit['id'], 'url': exploit['url'], 'name': exploit['name']}})
Beispiel #8
0
            self.Attack_Pattern_Catalog_tag = False

# dictionary
capecurl = Configuration.getCAPECDict()
# connect to db
db = Configuration.getMongoConnection()
capec = db.capec
info = db.info

# make parser
parser = make_parser()
ch = CapecHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(capecurl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl))
i = info.find_one({'db': 'capec'})
if i is not None:
    if f.headers['last-modified'] == i['last-modified']:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = capec.initialize_ordered_bulk_op()
for attack in progressbar(ch.capec):
    bulk.find({'id': attack['id']}).upsert().update({"$set": {'name': attack['name'], 'summary': attack['summary'], 'prerequisites': attack['prerequisites'], 'solutions': attack['solutions'], 'related_weakness': attack['related_weakness']}})
bulk.execute()

#update database info after successful program-run
Beispiel #9
0

# dict
cpedict = Configuration.getCPEDict()
# connect to db
db = Configuration.getMongoConnection()
cpe = db.cpe
info = db.info

# make parser
parser = make_parser()
ch = CPEHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(cpedict)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (cpedict))
i = info.find_one({'db': 'cpe'})
if i is not None:
    if f.headers['last-modified'] == i['last-modified']:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = cpe.initialize_ordered_bulk_op()
for x in progressbar(ch.cpe):
    name = toStringFormattedCPE(x['name'])
    oldCPE = x['name']
    title = x['title'][0]
        if name == 'nvd:statement':
            self.statementtag = False
            self.statement = self.statement + self.statement.rstrip()
            self.vendor[-1]['statement'] = self.statement


# dictionary
vendordict = Configuration.getVendorDict()

# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
    (f, r) = Configuration.getFile(vendordict, compressed=True)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (vendordict))
i = db.getLastModified('vendor')
if i is not None:
    if r.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
statements = []
for statement in progressbar(ch.vendor):
    if args.v:
        print(statement)
    statements.append(statement)
import lib.DatabaseLayer as db

import csv
import argparse

# dictionary
exploitdburl = Configuration.getexploitdbDict()
tmppath = Configuration.getTmpdir()

argparser = argparse.ArgumentParser(description='Populate/update the exploitdb ref database')
argparser.add_argument('-v', action='store_true', help='verbose output', default=False)
args = argparser.parse_args()


try:
    f = Configuration.getFile(exploitdburl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(exploitdburl))

i = db.getLastModified('exploitdb')
if i is not None:
    if f.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)

if not os.path.exists(tmppath):
    os.mkdir(tmppath)

csvfile = tmppath+'/exploitdb.csv'
with open(csvfile, 'wb') as fp:
    shutil.copyfileobj(f, fp)
Beispiel #12
0
import argparse

# dictionary
exploitdburl = Configuration.getexploitdbDict()
tmppath = Configuration.getTmpdir()

argparser = argparse.ArgumentParser(
    description='Populate/update the exploitdb ref database')
argparser.add_argument('-v',
                       action='store_true',
                       help='verbose output',
                       default=False)
args = argparser.parse_args()

try:
    f = Configuration.getFile(exploitdburl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (exploitdburl))

i = db.getLastModified('exploitdb')
if i is not None:
    if f.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)

if not os.path.exists(tmppath):
    os.mkdir(tmppath)

csvfile = tmppath + '/exploitdb.csv'
with open(csvfile, 'wb') as fp:
Beispiel #13
0
            self.cves[-1]['Modified'] = self.PUB

if __name__ == '__main__':
    # connect to the DB.
    db = Configuration.getMongoConnection()
    collection = db.cves
    info = db.info
    parser = make_parser()
    ch = CVEHandler()
    parser.setContentHandler(ch)
    # start here if it's an update.
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            f = Configuration.getFile(Configuration.getCVEDict() + getfile)
        except:
            sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
        i = dbLayer.getInfo("cve")
        if i is not None:
            if f.headers['last-modified'] == i['last-modified']:
                print("Not modified")
                sys.exit(0)
        dbLayer.setColUpdate("cve", f.headers['last-modified'])

        # get your parser on !!
        parser = make_parser()
        ch = CVEHandler()
        parser.setContentHandler(ch)
        parser.parse(f)
        for item in ch.cves:
Beispiel #14
0
            self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True)
        if name == 'vuln:last-modified-datetime':
            self.inPUBElem = 0
            self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True)


if __name__ == '__main__':
    parser = make_parser()
    ch = CVEHandler()
    parser.setContentHandler(ch)
    # start here if it's an update.
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            (f, r) = Configuration.getFile(
                Configuration.getFeedURL('cve') + getfile)
        except:
            sys.exit(
                "Cannot open url %s. Bad URL or not connected to the internet?"
                % (Configuration.getFeedURL("cve") + getfile))
        i = db.getInfo("cve")
        last_modified = parse_datetime(r.headers['last-modified'],
                                       ignoretz=True)
        if i is not None:
            if last_modified == i['last-modified']:
                print("Not modified")
                sys.exit(0)
        db.setColUpdate("cve", last_modified)

        # get your parser on !!
        parser = make_parser()
Beispiel #15
0
    # datemode: 0 for 1900-based, 1 for 1904-based
    return (
        datetime.datetime(1899, 12, 30)
        + datetime.timedelta(days=xldate + 1462 * datemode)
        )

# dictionary
msbulletinurl = Configuration.getMSBULLETINDict()

# connect to db
db = Configuration.getMongoConnection()
msbulletin = db.ms
info = db.info

try:
    f = Configuration.getFile(msbulletinurl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(msbulletinurl))

# check modification date
i = info.find_one({'db': 'ms'})
if i is not None:
    if f.headers['last-modified'] == i['last-modified']:
        print("Not modified")
        sys.exit(0)

if not os.path.exists('./tmp'):
    os.mkdir('./tmp')
with open('./tmp/BulletinSearch.xls', 'wb') as fp:
    shutil.copyfileobj(f, fp)
     
tmppath = Configuration.getTmpdir()

argparser = argparse.ArgumentParser(
    description='Populate/update the NIST ref database')
argparser.add_argument('-v',
                       action='store_true',
                       help='verbose output',
                       default=False)
args = argparser.parse_args()

if args.v:
    verbose = True

# check modification date
try:
    u = Configuration.getFile(RefUrl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (RefUrl))
i = db.getLastModified('ref')
if i is not None:
    if u.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)

# Create temp file and download and unpack database
if not os.path.exists(tmppath):
    os.mkdir(tmppath)

with open(tmppath + '/allrefmaps.zip', 'wb') as fp:
    shutil.copyfileobj(u, fp)
Beispiel #17
0
            self.inDTElem = 0
            self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True)
        if name == 'vuln:last-modified-datetime':
            self.inPUBElem = 0
            self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True)

if __name__ == '__main__':
    parser = make_parser()
    ch = CVEHandler()
    parser.setContentHandler(ch)
    # start here if it's an update.
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            (f, r) = Configuration.getFile(Configuration.getFeedURL('cve') + getfile)
        except:
            sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getFeedURL("cve") + getfile))
        i = db.getInfo("cves")
        last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True)
        if i is not None:
            if last_modified == i['last-modified']:
                print("Not modified")
                sys.exit(0)
        db.setColUpdate("cves", last_modified)

        # get your parser on !!
        parser = make_parser()
        ch = CVEHandler()
        parser.setContentHandler(ch)
        parser.parse(f)
Beispiel #18
0
            self.inDTElem = 0
            self.cves[-1]['Published'] = self.DT
        if name == 'vuln:last-modified-datetime':
            self.inPUBElem = 0
            self.cves[-1]['Modified'] = self.PUB

if __name__ == '__main__':
    parser = make_parser()
    ch = CVEHandler()
    parser.setContentHandler(ch)
    # start here if it's an update.
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            (f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
        except:
            sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
        i = db.getInfo("cve")
        if i is not None:
            if r.headers['last-modified'] == i['last-modified']:
                print("Not modified")
                sys.exit(0)
        db.setColUpdate("cve", r.headers['last-modified'])

        # get your parser on !!
        parser = make_parser()
        ch = CVEHandler()
        parser.setContentHandler(ch)
        parser.parse(f)
        for item in ch.cves:
Beispiel #19
0
            self.exploittag = False
            self.refl = []
        if name == 'elliot':
            self.elliottag = False


# dictionary
d2securl = Configuration.getd2secDict()

# make parser
parser = make_parser()
ch = ExploitHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(d2securl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (d2securl))
last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified("d2sec")
if i is not None:
    if last_modified == i:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
exploitList = []
for exploit in progressbar(ch.d2sec):
    print(exploit)
    if args.v:
        elif name == 'references':
            self.referencestag = False
        elif name == 'reference':
            self.referencetag = False
            self.href = None

# dict
cpedict = Configuration.getCPEDict()

# make parser
parser = make_parser()
ch = CPEHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(cpedict)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cpedict))
i = db.getLastModified('cpe')
if i is not None:
    if f.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
cpeList=[]
for x in progressbar(ch.cpe):
  x['id']= toStringFormattedCPE(x['name'])
  x['title']=x['title'][0]
  x['cpe_2_2'] = x.pop('name')
  if not x['references']: x.pop('references')
Beispiel #21
0
    # datemode: 0 for 1900-based, 1 for 1904-based
    return (datetime.datetime(1899, 12, 30) +
            datetime.timedelta(days=xldate + 1462 * datemode))


# dictionary
msbulletinurl = Configuration.getMSBULLETINDict()
tmppath = Configuration.getTmpdir()

# connect to db
db = Configuration.getMongoConnection()
msbulletin = db.ms
info = db.info

try:
    f = Configuration.getFile(msbulletinurl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (msbulletinurl))

# check modification date
i = info.find_one({'db': 'ms'})
if i is not None:
    if f.headers['last-modified'] == i['last-modified']:
        print("Not modified")
        sys.exit(0)

if not os.path.exists(tmppath):
    os.mkdir(tmppath)
with open(tmppath + '/BulletinSearch.xlsx', 'wb') as fp:
    shutil.copyfileobj(f, fp)
Beispiel #22
0
import tarfile
import shutil
import sqlite3

from lib.ProgressBar import progressbar
from lib.Config import Configuration
import lib.DatabaseLayer as db

vFeedurl = Configuration.getvFeedURL()
vFeedstatus = Configuration.getvFeedStatus()
tmppath = os.path.join(runPath, "..", Configuration.getTmpdir())

# check modification date
try:
    u = Configuration.getFile(vFeedurl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vFeedurl))
i = db.getLastModified('vfeed')
if i is not None:
    if u.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)
# create temp file and download and unpack database
if not os.path.exists(tmppath):
    os.mkdir(tmppath)
with open(tmppath+'/vfeed.db.tgz', 'wb') as fp:
    shutil.copyfileobj(u, fp)
t = tarfile.open(name=tmppath+'/vfeed.db.tgz', mode='r')
t.extract('vfeed.db', path=tmppath)
t.close
Beispiel #23
0
                if cwe['lang'] == 'en':
                    cve['cwe'] = cwe['value']
        if not ('cwe' in cve):
            cve['cwe'] = defaultvalue['cwe']
    else:
        cve['cwe'] = defaultvalue['cwe']
    cve['vulnerable_configuration_cpe_2_2'] = []
    return cve


if __name__ == '__main__':
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            (f, r) = Configuration.getFile(
                Configuration.getFeedURL('cve') + getfile)
        except:
            sys.exit(
                "Cannot open url %s. Bad URL or not connected to the internet?"
                % (Configuration.getFeedURL("cve") + getfile))
        i = db.getInfo("cves")
        last_modified = parse_datetime(r.headers['last-modified'],
                                       ignoretz=True)
        if i is not None:
            if last_modified == i['last-modified'] and not args.f:
                print("Not modified")
                sys.exit(0)
        db.setColUpdate("cves", last_modified)
        cvej = json.loads(f.read())
        for cveitem in cvej['CVE_Items']:
            item = process_cve_item(item=cveitem)
Beispiel #24
0
from dateutil.parser import parse as parse_datetime

import tarfile
import shutil

from lib.Config import Configuration
import lib.DatabaseLayer as db

vFeedurl = Configuration.getvFeedURL()
vFeedstatus = Configuration.getvFeedStatus()
tmppath = os.path.join(runPath, "..", Configuration.getTmpdir())
print(tmppath)
# check modification date
try:
    u = Configuration.getFile(vFeedurl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vFeedurl))
last_modified = parse_datetime(u.headers['last-modified'], ignoretz=True)
i = db.getLastModified('vfeed')
if i is not None:
    if last_modified == i:
        print("Not modified")
        sys.exit(0)
# create temp file and download and unpack database
if not os.path.exists(tmppath):
    os.mkdir(tmppath)
with open(tmppath+'/vfeed.db.tgz', 'wb') as fp:
    shutil.copyfileobj(u, fp)
t = tarfile.open(name=tmppath+'/vfeed.db.tgz', mode='r:*')
t.extract('vfeed.db', path=tmppath)
Beispiel #25
0
        if name == 'capec:Attack_Patterns':
            self.Attack_Patterns_tag = False
        if name == 'capec:Attack_Pattern_Catalog':
            self.Attack_Pattern_Catalog_tag = False


# dictionary
capecurl = Configuration.getCAPECDict()

# make parser
parser = make_parser()
ch = CapecHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(capecurl)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (capecurl))
i = db.getLastModified('capec')
if i is not None:
    if f.headers['last-modified'] == i:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
attacks = []
for attack in progressbar(ch.capec):
    attacks.append(attack)
db.bulkUpdate("capec", attacks)
Beispiel #26
0
                if cwe['lang'] == 'en':
                    cve['cwe'] = cwe['value']
        if not ('cwe' in cve):
            cve['cwe'] = defaultvalue['cwe']
    else:
        cve['cwe'] = defaultvalue['cwe']
    cve['vulnerable_configuration_cpe_2_2'] = []
    return cve


if __name__ == '__main__':
    if args.u:
        # get the 'modified' file
        getfile = file_prefix + file_mod + file_suffix
        try:
            (f, r) = Configuration.getFile(
                Configuration.getFeedURL('cve') + getfile)
        except:
            sys.exit(
                "Cannot open url %s. Bad URL or not connected to the internet?"
                % (Configuration.getFeedURL("cve") + getfile))
        i = db.getInfo("cves")
        last_modified = parse_datetime(r.headers['last-modified'],
                                       ignoretz=True)
        if i is not None:
            if last_modified == i['last-modified'] and not args.f:
                print("Not modified")
                sys.exit(0)
        db.setColUpdate("cves", last_modified)
        cvej = json.loads(f.read())
        for cveitem in cvej['CVE_Items']:
            item = process_cve_item(item=cveitem)
# dictionary
vendordict = Configuration.getVendorDict()

# connect to db
db = Configuration.getMongoConnection()
vendor = db.vendor
info = db.info

# make parser
parser = make_parser()
ch = VendorHandler()
parser.setContentHandler(ch)
# check modification date
try:
    f = Configuration.getFile(vendordict)
except:
    sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" %
             (vendordict))
i = info.find_one({'db': 'vendor'})
if i is not None:
    if f.headers['last-modified'] == i['last-modified']:
        print("Not modified")
        sys.exit(0)
# parse xml and store in database
parser.parse(f)
bulk = vendor.initialize_ordered_bulk_op()
for statement in progressbar(ch.vendor):
    if args.v:
        print(statement)
    bulk.find({