def cpeItemPopulateCallBack(cpe: dict, buffer: list, bufferSize: int, counter: dict, flush: bool = False): if cpe: item = process_cpe_item(cpe) buffer.append(item) currentSize = len(buffer) if flush or currentSize >= bufferSize: db.bulkInsert("cpe", buffer) buffer.clear() counter["count"] = counter["count"] + currentSize print( str(datetime.now()) + " --> " + str(counter["count"]) + " CPEs inserted")
cpeList = [] for cpeitem in cpej["matches"]: item = process_cpe_item(cpeitem) cpeList.append(item) db.bulkUpdate("cpe", cpeList) #update database info after successful program-run db.setColUpdate('cpe', last_modified) elif args.p: c = db.getSize('cpe') if args.v: print(str(c)) if c > 0 and args.a is False: print("Database already populated") else: print("Database population started") db.dropCollection("cpe") try: (f, r) = Configuration.getFile(Configuration.getFeedURL('cpe')) except: sys.exit( "Cannot open url %s. Bad URL or not connected to the internet?" % (Configuration.getFeedURL("cpe"))) cpej = json.loads(f.read()) cpeList = [] for cpeitem in cpej["matches"]: item = process_cpe_item(cpeitem) cpeList.append(item) db.bulkInsert("cpe", cpeList)