def main(argv=None): # IGNORE:C0111
'''Command line options.'''
if argv:
sys.argv.extend(argv)
try:
# Setup argument parser
parser = ArgumentParser(description='''
Read data from the Meetup API and write it do a MongoDB database. Each run of this program
creates a new batch of data identified by a batchID. The default database is MUGS. You can change
this by using the --host parameter and specifying a different database in the mongodb URI.
If you use the --pro arguement your API key must be a meetup pro account API key. If not the api
calls to the pro interface will fail.
If you are and adminstrator on the pro account you should use the --admin flag to give you
access to the admin APIs.
''')
#
# MongoDB Args
parser.add_argument('--host',
default="mongodb://localhost:27017/MUGS",
help='URI to connect to : [default: %(default)s]')
parser.add_argument("--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument("-v",
"--version",
action='version',
version=__programName__ + " " + __version__)
parser.add_argument(
'--trialrun',
action="store_true",
default=False,
help='Trial run, no updates [default: %(default)s]')
parser.add_argument(
'--mugs',
nargs="+",
help='Process MUGs list list mugs by name [default: %(default)s]')
parser.add_argument(
"--pro",
default=False,
action="store_true",
help="use if you have a pro account uses pro API calls")
parser.add_argument(
"--admin",
default=False,
action="store_true",
help="Some calls are only available to admin users")
parser.add_argument(
"--database",
default="MUGS",
help="Default database name to write to [default: %(default)s]")
parser.add_argument('--phases',
nargs="+",
choices=[
"groups", "members", "attendees",
"upcomingevents", "pastevents"
],
default=["all"],
help='execution phases')
parser.add_argument(
'--loglevel',
default="INFO",
choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
help='Logging level [default: %(default)s]')
parser.add_argument('--apikey',
default=None,
help='Default API key for meetup')
parser.add_argument(
'--urlfile',
help=
"File containing a list of MUG URLs to be used to parse data [ default: %(default)s]"
)
# Process arguments
args = parser.parse_args()
apikey = ""
if args.apikey:
apikey = args.apikey
else:
apikey = get_meetup_key()
verbose = args.verbose
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string,
level=LoggingLevel(args.loglevel))
# Turn off logging for requests
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
if verbose > 0:
logging.info("Verbose mode on")
if args.urlfile:
if not os.path.isfile(args.urlfile):
print("No such file --urlfile '%s'" % args.urlfile)
sys.exit(1)
if args.mugs:
mugList = args.mugs
else:
mugList = []
if args.pro:
nopro = False
else:
nopro = True
mdb = MUGAlyserMongoDB(args.host)
audit = Audit(mdb)
batchID = audit.startBatch(
{
"args": vars(args),
"version": __programName__ + " " + __version__,
"pro_account": args.pro
},
trial=args.trialrun,
apikey=apikey)
start = datetime.utcnow()
logging.info("Started MUG processing for batch ID: %i", batchID)
logging.info("Writing to database : '%s'", mdb.database().name)
if nopro:
logging.info("Using standard API calls (no pro account API key)")
if args.urlfile:
logging.info("Reading groups from: '%s'", args.urlfile)
with open(args.urlfile) as f:
mugList = f.read().splitlines()
else:
logging.info("Using pro API calls (pro account API key)")
if nopro:
logging.info("Processing %i MUG URLS", len(mugList))
else:
mugList = list(MeetupAPI().get_pro_group_names())
writer = MeetupWriter(audit, mdb, mugList, apikey)
if "all" in args.phases:
phases = ["groups", "members", "upcomingevents", "pastevents"]
if args.admin:
phases.append("attendees")
else:
phases = args.phases
if "groups" in phases:
logging.info("processing group info for %i groups: nopro=%s",
len(mugList), nopro)
writer.processGroups(nopro)
phases.remove("groups")
if "members" in phases:
logging.info("processing members info for %i groups: nopro=%s",
len(mugList), nopro)
writer.processMembers(nopro)
phases.remove("members")
for i in mugList:
writer.capture_snapshot(i, args.admin, phases)
audit.endBatch(batchID)
end = datetime.utcnow()
elapsed = end - start
logging.info("MUG processing took %s for BatchID : %i", elapsed,
batchID)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except pymongo.errors.ServerSelectionTimeoutError, e:
print("Failed to connect to MongoDB Server (server timeout): %s" % e)
sys.exit(2)
class Test_audit(unittest.TestCase):
def setUp(self):
self._mdb = MUGAlyserMongoDB(uri="mongodb://localhost/TEST_AUDIT")
self._audit = Audit(self._mdb)
def tearDown(self):
self._mdb.client().drop_database("TEST_AUDIT")
pass
#@unittest.skip
def test_incrementID(self):
batchID = self._audit.incrementBatchID()
curID = self._audit.getCurrentBatchID()
self.assertEqual(batchID, curID)
newID = self._audit.incrementBatchID()
self.assertEqual(batchID + 1, newID)
def test_getCurrentValidBatchID(self):
batchID1 = self._audit.startBatch(doc={"test": "doc"}, trial=True)
self._audit.endBatch(batchID1)
#self.assertRaises( ValueError, self._audit.getCurrentValidBatchID )
batchID2 = self._audit.startBatch(
{
"args": "arg list",
"version": __programName__ + " " + __version__
},
trial=False,
apikey=get_meetup_key())
self._audit.endBatch(batchID2)
self.assertEqual(batchID2, self._audit.getCurrentValidBatchID())
batchID3 = self._audit.startBatch(doc={"test": "doc"}, trial=True)
self._audit.endBatch(batchID3)
self.assertEqual(batchID2, self._audit.getCurrentValidBatchID())
def test_batch(self):
batchIDs = [x for x in self._audit.getBatchIDs()]
thisBatchID = self._audit.startBatch(doc={"test": "doc"}, trial=True)
newBatchIDs = [x for x in self._audit.getBatchIDs()]
self.assertEqual(len(batchIDs) + 1, len(newBatchIDs))
self.assertTrue(thisBatchID in newBatchIDs)
self._audit.endBatch(thisBatchID)
#@unittest.skip
def test_IDs(self):
self.assertRaises(ValueError, self._audit.getCurrentBatchID)
self.assertRaises(ValueError, self._audit.getLastBatchID)
self.assertFalse(self._audit.inBatch())
batchID = self._audit.startBatch({})
self.assertTrue(self._audit.inBatch())
self.assertEquals(1, self._audit.getCurrentBatchID())
self._audit.endBatch(batchID)
batch = self._audit.getBatch(batchID)
self.assertTrue("start" in batch)
self.assertTrue("end" in batch)
self.assertTrue("info" in batch)
self.assertTrue("batchID" in batch)
self.assertFalse(self._audit.incomplete(batchID))
batchID = self._audit.startBatch({})
self.assertTrue(self._audit.inBatch())
self.assertEquals(2, self._audit.getCurrentBatchID())
self._audit.endBatch(batchID)
self.assertFalse(self._audit.inBatch())
#@unittest.skip
def test_start_end_batch(self):
batchID = self._audit.startBatch({})
self.assertTrue(self._audit.incomplete(batchID))
self._audit.endBatch(batchID)
self.assertFalse(self._audit.incomplete(batchID))
