Ejemplo n.º 1
0
def cpeItemPopulateCallBack(cpe: dict,
                            buffer: list,
                            bufferSize: int,
                            counter: dict,
                            flush: bool = False):
    if cpe:
        item = process_cpe_item(cpe)
        buffer.append(item)

    currentSize = len(buffer)

    if flush or currentSize >= bufferSize:
        db.bulkInsert("cpe", buffer)
        buffer.clear()

        counter["count"] = counter["count"] + currentSize
        print(
            str(datetime.now()) + " --> " + str(counter["count"]) +
            " CPEs inserted")
Ejemplo n.º 2
0
        cpeList = []
        for cpeitem in cpej["matches"]:
            item = process_cpe_item(cpeitem)
            cpeList.append(item)
        db.bulkUpdate("cpe", cpeList)

        #update database info after successful program-run
        db.setColUpdate('cpe', last_modified)
    elif args.p:
        c = db.getSize('cpe')
        if args.v:
            print(str(c))
        if c > 0 and args.a is False:
            print("Database already populated")
        else:
            print("Database population started")
            db.dropCollection("cpe")
            try:
                (f, r) = Configuration.getFile(Configuration.getFeedURL('cpe'))
            except:
                sys.exit(
                    "Cannot open url %s. Bad URL or not connected to the internet?"
                    % (Configuration.getFeedURL("cpe")))

            cpej = json.loads(f.read())
            cpeList = []
            for cpeitem in cpej["matches"]:
                item = process_cpe_item(cpeitem)
                cpeList.append(item)
            db.bulkInsert("cpe", cpeList)