def main(argv=None): # IGNORE:C0111 '''Command line options.''' if argv: sys.argv.extend(argv) try: # Setup argument parser parser = ArgumentParser(description=''' Read data from the Meetup API and write it do a MongoDB database. Each run of this program creates a new batch of data identified by a batchID. The default database is MUGS. You can change this by using the --host parameter and specifying a different database in the mongodb URI. If you use the --pro arguement your API key must be a meetup pro account API key. If not the api calls to the pro interface will fail. If you are and adminstrator on the pro account you should use the --admin flag to give you access to the admin APIs. ''') # # MongoDB Args parser.add_argument('--host', default="mongodb://localhost:27017/MUGS", help='URI to connect to : [default: %(default)s]') parser.add_argument("--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") parser.add_argument("-v", "--version", action='version', version=__programName__ + " " + __version__) parser.add_argument( '--trialrun', action="store_true", default=False, help='Trial run, no updates [default: %(default)s]') parser.add_argument( '--mugs', nargs="+", help='Process MUGs list list mugs by name [default: %(default)s]') parser.add_argument( "--pro", default=False, action="store_true", help="use if you have a pro account uses pro API calls") parser.add_argument( "--admin", default=False, action="store_true", help="Some calls are only available to admin users") parser.add_argument( "--database", default="MUGS", help="Default database name to write to [default: %(default)s]") parser.add_argument('--phases', nargs="+", choices=[ "groups", "members", "attendees", "upcomingevents", "pastevents" ], default=["all"], help='execution phases') parser.add_argument( '--loglevel', default="INFO", choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"], help='Logging level [default: %(default)s]') parser.add_argument('--apikey', default=None, help='Default API key for meetup') parser.add_argument( '--urlfile', help= "File containing a list of MUG URLs to be used to parse data [ default: %(default)s]" ) # Process arguments args = parser.parse_args() apikey = "" if args.apikey: apikey = args.apikey else: apikey = get_meetup_key() verbose = args.verbose format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(format=format_string, level=LoggingLevel(args.loglevel)) # Turn off logging for requests logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) if verbose > 0: logging.info("Verbose mode on") if args.urlfile: if not os.path.isfile(args.urlfile): print("No such file --urlfile '%s'" % args.urlfile) sys.exit(1) if args.mugs: mugList = args.mugs else: mugList = [] if args.pro: nopro = False else: nopro = True mdb = MUGAlyserMongoDB(args.host) audit = Audit(mdb) batchID = audit.startBatch( { "args": vars(args), "version": __programName__ + " " + __version__, "pro_account": args.pro }, trial=args.trialrun, apikey=apikey) start = datetime.utcnow() logging.info("Started MUG processing for batch ID: %i", batchID) logging.info("Writing to database : '%s'", mdb.database().name) if nopro: logging.info("Using standard API calls (no pro account API key)") if args.urlfile: logging.info("Reading groups from: '%s'", args.urlfile) with open(args.urlfile) as f: mugList = f.read().splitlines() else: logging.info("Using pro API calls (pro account API key)") if nopro: logging.info("Processing %i MUG URLS", len(mugList)) else: mugList = list(MeetupAPI().get_pro_group_names()) writer = MeetupWriter(audit, mdb, mugList, apikey) if "all" in args.phases: phases = ["groups", "members", "upcomingevents", "pastevents"] if args.admin: phases.append("attendees") else: phases = args.phases if "groups" in phases: logging.info("processing group info for %i groups: nopro=%s", len(mugList), nopro) writer.processGroups(nopro) phases.remove("groups") if "members" in phases: logging.info("processing members info for %i groups: nopro=%s", len(mugList), nopro) writer.processMembers(nopro) phases.remove("members") for i in mugList: writer.capture_snapshot(i, args.admin, phases) audit.endBatch(batchID) end = datetime.utcnow() elapsed = end - start logging.info("MUG processing took %s for BatchID : %i", elapsed, batchID) except KeyboardInterrupt: print("Keyboard interrupt : Exiting...") sys.exit(2) except pymongo.errors.ServerSelectionTimeoutError, e: print("Failed to connect to MongoDB Server (server timeout): %s" % e) sys.exit(2)
class Test_audit(unittest.TestCase): def setUp(self): self._mdb = MUGAlyserMongoDB(uri="mongodb://localhost/TEST_AUDIT") self._audit = Audit(self._mdb) def tearDown(self): self._mdb.client().drop_database("TEST_AUDIT") pass #@unittest.skip def test_incrementID(self): batchID = self._audit.incrementBatchID() curID = self._audit.getCurrentBatchID() self.assertEqual(batchID, curID) newID = self._audit.incrementBatchID() self.assertEqual(batchID + 1, newID) def test_getCurrentValidBatchID(self): batchID1 = self._audit.startBatch(doc={"test": "doc"}, trial=True) self._audit.endBatch(batchID1) #self.assertRaises( ValueError, self._audit.getCurrentValidBatchID ) batchID2 = self._audit.startBatch( { "args": "arg list", "version": __programName__ + " " + __version__ }, trial=False, apikey=get_meetup_key()) self._audit.endBatch(batchID2) self.assertEqual(batchID2, self._audit.getCurrentValidBatchID()) batchID3 = self._audit.startBatch(doc={"test": "doc"}, trial=True) self._audit.endBatch(batchID3) self.assertEqual(batchID2, self._audit.getCurrentValidBatchID()) def test_batch(self): batchIDs = [x for x in self._audit.getBatchIDs()] thisBatchID = self._audit.startBatch(doc={"test": "doc"}, trial=True) newBatchIDs = [x for x in self._audit.getBatchIDs()] self.assertEqual(len(batchIDs) + 1, len(newBatchIDs)) self.assertTrue(thisBatchID in newBatchIDs) self._audit.endBatch(thisBatchID) #@unittest.skip def test_IDs(self): self.assertRaises(ValueError, self._audit.getCurrentBatchID) self.assertRaises(ValueError, self._audit.getLastBatchID) self.assertFalse(self._audit.inBatch()) batchID = self._audit.startBatch({}) self.assertTrue(self._audit.inBatch()) self.assertEquals(1, self._audit.getCurrentBatchID()) self._audit.endBatch(batchID) batch = self._audit.getBatch(batchID) self.assertTrue("start" in batch) self.assertTrue("end" in batch) self.assertTrue("info" in batch) self.assertTrue("batchID" in batch) self.assertFalse(self._audit.incomplete(batchID)) batchID = self._audit.startBatch({}) self.assertTrue(self._audit.inBatch()) self.assertEquals(2, self._audit.getCurrentBatchID()) self._audit.endBatch(batchID) self.assertFalse(self._audit.inBatch()) #@unittest.skip def test_start_end_batch(self): batchID = self._audit.startBatch({}) self.assertTrue(self._audit.incomplete(batchID)) self._audit.endBatch(batchID) self.assertFalse(self._audit.incomplete(batchID))