usecache.registerPacket(packet)
if queue.empty() and usecache.dirty:
usecache.saveToMongo()
queue.task_done()
def initializeThreads(usecache):
t = Thread(target=worker, args=(usecache,))
t.daemon = True
t.start()
if __name__ == "__main__":
c = Connection()
# True=assume logical link between "root" non-dependent packages
c.drop_database("snm-tacc")
usecache = UsageCache(openOrCreate(c, "snm-tacc"), True, "TACC", useWeakDeps=True)
initializeThreads(usecache)
rownum = 0
try:
for raw in c["snm-tacc-raw"]["scimapInfo"].find(timeout=False):
rownum += 1
queue.put(raw)
while (not(queue.empty())):
time.sleep(1)
print "Spooling through queued packets at row#", rownum
queue.join()
print "Saving"
if usecache.dirty:
usecache.saveToMongo()
print "Saved."
c = Connection()
if len(sys.argv) < 4:
print "Usage:", sys.argv[
0], "<sqlite repo db>", "<mongo db>", "<appinfo file>"
quit()
sqlitedb = sys.argv[1]
snmdb = sys.argv[2]
appinfo = sys.argv[3]
if (not os.path.isfile(sqlitedb)):
print sqlitedb, "does not exist."
quit()
rs = RepoScrape(sqlitedb)
rs.makeAppInfo()
rs.writeAppInfo(appinfo)
# True=assume logical link between "root" non-dependent packages
c.drop_database(snmdb)
usecache = UsageCache(openOrCreate(c, snmdb), True, "R")
initializeThreads(usecache)
for raw in c["snm-raw-records"]["scimapInfo"].find():
queue.put(raw)
finalizeThreads()
usecache.insertGitData(rs)
usecache.insertPublicationData(rs)
try:
record = json.loads(data, object_hook=scrub_dots)
record["receivedEpoch"] = int(time.time())
registerParsed(c, record, ip, usecache, dbraw)
except Exception as e:
print "Error: " + str(e)
(r1, r2, r3) = sys.exc_info()
print traceback.format_exception(r1, r2, r3)
def initializeThreads(usecache):
t = Thread(target=worker, args=(usecache, ))
t.daemon = True
t.start()
def finalizeThreads():
"""Used by taccParse and reprocess"""
while (not (queue.empty())):
time.sleep(1)
if __name__ == "__main__":
c = Connection()
# True=assume logical link between "root" non-dependent packages
usecache = UsageCache(openOrCreate(c, "snm-r"), True, "R")
initializeThreads(usecache)
await ()
if __name__ == "__main__":
c = Connection()
if len(sys.argv) < 4:
print "Usage:", sys.argv[0], "<sqlite repo db>", "<mongo db>", "<appinfo file>"
quit()
sqlitedb = sys.argv[1]
snmdb = sys.argv[2]
appinfo = sys.argv[3]
if (not os.path.isfile(sqlitedb)):
print sqlitedb, "does not exist."
quit()
rs = RepoScrape(sqlitedb)
rs.makeAppInfo()
rs.writeAppInfo(appinfo)
# True=assume logical link between "root" non-dependent packages
c.drop_database(snmdb)
usecache = UsageCache(openOrCreate(c, snmdb), True, "R")
initializeThreads(usecache)
for raw in c["snm-raw-records"]["scimapInfo"].find():
queue.put(raw)
finalizeThreads()
usecache.insertGitData(rs)
usecache.insertPublicationData(rs)
