def cpeItemPopulateCallBack(cpe: dict,
buffer: list,
bufferSize: int,
counter: dict,
flush: bool = False):
if cpe:
item = process_cpe_item(cpe)
buffer.append(item)
currentSize = len(buffer)
if flush or currentSize >= bufferSize:
db.bulkInsert("cpe", buffer)
buffer.clear()
counter["count"] = counter["count"] + currentSize
print(
str(datetime.now()) + " --> " + str(counter["count"]) +
" CPEs inserted")
cpeList = []
for cpeitem in cpej["matches"]:
item = process_cpe_item(cpeitem)
cpeList.append(item)
db.bulkUpdate("cpe", cpeList)
#update database info after successful program-run
db.setColUpdate('cpe', last_modified)
elif args.p:
c = db.getSize('cpe')
if args.v:
print(str(c))
if c > 0 and args.a is False:
print("Database already populated")
else:
print("Database population started")
db.dropCollection("cpe")
try:
(f, r) = Configuration.getFile(Configuration.getFeedURL('cpe'))
except:
sys.exit(
"Cannot open url %s. Bad URL or not connected to the internet?"
% (Configuration.getFeedURL("cpe")))
cpej = json.loads(f.read())
cpeList = []
for cpeitem in cpej["matches"]:
item = process_cpe_item(cpeitem)
cpeList.append(item)
db.bulkInsert("cpe", cpeList)