usecache.registerPacket(packet) if queue.empty() and usecache.dirty: usecache.saveToMongo() queue.task_done() def initializeThreads(usecache): t = Thread(target=worker, args=(usecache,)) t.daemon = True t.start() if __name__ == "__main__": c = Connection() # True=assume logical link between "root" non-dependent packages c.drop_database("snm-tacc") usecache = UsageCache(openOrCreate(c, "snm-tacc"), True, "TACC", useWeakDeps=True) initializeThreads(usecache) rownum = 0 try: for raw in c["snm-tacc-raw"]["scimapInfo"].find(timeout=False): rownum += 1 queue.put(raw) while (not(queue.empty())): time.sleep(1) print "Spooling through queued packets at row#", rownum queue.join() print "Saving" if usecache.dirty: usecache.saveToMongo() print "Saved."
c = Connection() if len(sys.argv) < 4: print "Usage:", sys.argv[ 0], "<sqlite repo db>", "<mongo db>", "<appinfo file>" quit() sqlitedb = sys.argv[1] snmdb = sys.argv[2] appinfo = sys.argv[3] if (not os.path.isfile(sqlitedb)): print sqlitedb, "does not exist." quit() rs = RepoScrape(sqlitedb) rs.makeAppInfo() rs.writeAppInfo(appinfo) # True=assume logical link between "root" non-dependent packages c.drop_database(snmdb) usecache = UsageCache(openOrCreate(c, snmdb), True, "R") initializeThreads(usecache) for raw in c["snm-raw-records"]["scimapInfo"].find(): queue.put(raw) finalizeThreads() usecache.insertGitData(rs) usecache.insertPublicationData(rs)
try: record = json.loads(data, object_hook=scrub_dots) record["receivedEpoch"] = int(time.time()) registerParsed(c, record, ip, usecache, dbraw) except Exception as e: print "Error: " + str(e) (r1, r2, r3) = sys.exc_info() print traceback.format_exception(r1, r2, r3) def initializeThreads(usecache): t = Thread(target=worker, args=(usecache, )) t.daemon = True t.start() def finalizeThreads(): """Used by taccParse and reprocess""" while (not (queue.empty())): time.sleep(1) if __name__ == "__main__": c = Connection() # True=assume logical link between "root" non-dependent packages usecache = UsageCache(openOrCreate(c, "snm-r"), True, "R") initializeThreads(usecache) await ()
if __name__ == "__main__": c = Connection() if len(sys.argv) < 4: print "Usage:", sys.argv[0], "<sqlite repo db>", "<mongo db>", "<appinfo file>" quit() sqlitedb = sys.argv[1] snmdb = sys.argv[2] appinfo = sys.argv[3] if (not os.path.isfile(sqlitedb)): print sqlitedb, "does not exist." quit() rs = RepoScrape(sqlitedb) rs.makeAppInfo() rs.writeAppInfo(appinfo) # True=assume logical link between "root" non-dependent packages c.drop_database(snmdb) usecache = UsageCache(openOrCreate(c, snmdb), True, "R") initializeThreads(usecache) for raw in c["snm-raw-records"]["scimapInfo"].find(): queue.put(raw) finalizeThreads() usecache.insertGitData(rs) usecache.insertPublicationData(rs)