class Test(unittest.TestCase):
def tearDown(self):
self._mdb.client().drop_database("TESTWRITER")
def test_write_group(self):
self._mdb = MUGAlyserMongoDB("mongodb://localhost:27017/TESTWRITER")
self._audit = Audit(self._mdb)
batchID = self._audit.start_batch({"test": 1})
self._writer = MeetupWriter(get_meetup_key(), batchID, self._mdb)
self._writer.write_groups("nopro", ["DublinMUG"])
self._writer.write_groups("pro", ["DublinMUG"])
self.assertTrue(self._mdb.groupsCollection().find_one(
{"group.urlname": "DublinMUG"}))
self.assertTrue(self._mdb.proGroupsCollection().find_one(
{"group.urlname": "DublinMUG"}))
self._audit.end_batch(batchID)
def testProcessMembers(self):
self._mdb = MUGAlyserMongoDB("mongodb://localhost:27017/TESTWRITER")
self._audit = Audit(self._mdb)
batchID = self._audit.start_batch({"test": 2})
self._writer = MeetupWriter(get_meetup_key(), batchID, self._mdb)
self._writer.write_members("pro", ["DublinMUG"])
self._writer.write_members("nopro", ["DublinMUG"])
self.assertTrue(self._mdb.proMembersCollection().find_one(
{"member.member_name": "Joe Drumgoole"}))
self.assertTrue(self._mdb.membersCollection().find_one(
{"member.name": "Joe Drumgoole"}))
self._audit.end_batch(batchID)
class Test_audit(unittest.TestCase):
def setUp(self):
self._mdb = MUGAlyserMongoDB( uri="mongodb://localhost/TEST_AUDIT" )
self._audit = Audit( self._mdb )
def tearDown(self):
self._mdb.client().drop_database( "TEST_AUDIT" )
#@unittest.skip
def test_get_current_batch_id(self):
self.assertFalse( self._audit.in_batch())
batch_id = self._audit.start_batch( doc = { "test" : "doc"})
self.assertTrue( self._audit.in_batch())
self._audit.end_batch( batch_id )
self.assertTrue( self._audit.get_batch( batch_id ))
self.assertFalse( self._audit.in_batch())
self.assertEqual( batch_id, self._audit.get_last_valid_batch_id())
def test_get_valid_batches(self):
id1 = self._audit.start_batch( doc = { "test" : "doc"})
id2 = self._audit.start_batch( doc = { "test" : "doc"})
self.assertTrue( self._audit.in_batch())
self._audit.end_batch( id2 )
self.assertTrue( self._audit.in_batch())
self._audit.end_batch( id1 )
batch = self._audit.get_batch_end( id1 )
self.assertGreaterEqual( batch[ 'end'], parse( "1-Jun-2017", ) )
self.assertFalse( self._audit.in_batch())
idlist = list( self._audit.get_valid_batch_ids())
self.assertTrue( id1 in idlist )
self.assertTrue( id2 in idlist )
def test_get_last_batch_id(self):
id1 = self._audit.start_batch( doc = { "test" : "doc"})
id2 = self._audit.start_batch( doc = { "test" : "doc"})
self.assertEqual( 101, self._audit.get_last_batch_id())
self._audit.end_batch( id2 )
self.assertEqual( 101, self._audit.get_last_batch_id())
self._audit.end_batch( id1 )
id1 = self._audit.start_batch( doc = { "test" : "doc"})
self.assertEqual( 102, self._audit.get_last_batch_id())
self._audit.end_batch( id1 )
def test_pro_batch_id(self):
def mugalyser(argv=None): # IGNORE:C0111
'''Command line options.'''
try:
# Setup argument parser
parser = ArgumentParser(description='''
Read data from the Meetup API and write it do a MongoDB database. Each run of this program
creates a new batch of data identified by a batchID. The default database is MUGS. You can change
this by using the --host parameter and specifying a different database in the mongodb URI.
If you use the --pro arguement your API key must be a meetup pro account API key. If not the api
calls to the pro interface will fail.
If you are and adminstrator on the pro account you should use the --admin flag to give you
access to the admin APIs.
''')
#
# MongoDB Args
parser.add_argument('--host',
default="mongodb://localhost:27017/MUGS",
help='URI to connect to : [default: %(default)s]')
parser.add_argument("-v",
"--version",
action='version',
version=__programName__ + " " + __version__)
parser.add_argument(
'--mugs',
nargs="+",
default=[],
help='Process MUGs list list mugs by name [default: %(default)s]')
parser.add_argument("--collect",
choices=["pro", "nopro", "all"],
default="all",
help="Use pro API calls, no pro API calls or both")
parser.add_argument(
"--admin",
default=False,
action="store_true",
help=
"Some calls are only available to admin users, use this if you are not an admin"
)
parser.add_argument(
"--database",
default="MUGS",
help="Default database name to write to [default: %(default)s]")
parser.add_argument('--phases',
nargs="+",
choices=[
"groups", "members", "attendees",
"upcomingevents", "pastevents"
],
default=["all"],
help='execution phases')
parser.add_argument(
'--loglevel',
default="INFO",
choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
help='Logging level [default: %(default)s]')
parser.add_argument('--apikey',
default=None,
help='Default API key for meetup')
parser.add_argument("--batchname",
default=__programName__,
help="Batch name used in creating audit batches")
parser.add_argument(
'--urlfile',
help=
"File containing a list of MUG URLs to be used to parse data [ default: %(default)s]"
)
parser.add_argument(
"--drop",
default=False,
action="store_true",
help="drop the database before writing data [default: %(default)s]"
)
parser.add_argument("--organizer_id",
type=int,
help="Organizer ID is required for non pro groups")
# Process arguments
args = parser.parse_args(argv)
apikey = ""
if args.apikey:
apikey = args.apikey
else:
apikey = get_meetup_key()
mugalyser_logger = Logger(__programName__, args.loglevel)
# mugalyser_logger.add_stream_handler( args.loglevel )
mugalyser_logger.add_file_handler("mugalyser.log", args.loglevel)
api = MeetupAPI(apikey, reshape=True)
logger = mugalyser_logger.log()
# Turn off logging for requests
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
mdb = MUGAlyserMongoDB(uri=args.host, database_name=args.database)
if args.drop:
logger.warn(f"Dropping database:'{args.database}'")
mdb.drop(args.database)
audit = Audit(mdb)
batchID = audit.start_batch({
"args": vars(args),
"version": __programName__ + " " + __version__,
"name": args.batchname
})
start = datetime.utcnow()
logger.info("Started MUG processing for batch ID: %i", batchID)
logger.info("Writing to database : '%s'", mdb.database().name)
group_dict = {}
count = 0
group_list = []
if args.mugs:
for url in args.mugs:
group_list.append(api.get_group(url))
else:
group_list = list(api.get_groups())
for url, group in group_list:
#print(f"Checking:{group['urlname']}")
urlname = group['urlname']
url, full_group = api.get_group(urlname)
if args.collect in ["pro", "all"]:
if "pro_network" in full_group and full_group["pro_network"][
"name"] == "MongoDB":
count = count + 1
logger.info(
f"{count}. Processing pro group: {group['urlname']}")
group_dict[urlname] = full_group
if args.collect in ["nopro", "all"]:
if args.organizer_id:
if full_group["organizer"]["id"] == args.organizer_id:
count = count + 1
logger.info(
f"{count}. Processing normal group: {group['urlname']}"
)
group_dict[urlname] = full_group
else:
logger.error(
"You must specify --organizer_id when collecting nopro groups"
)
sys.exit(1)
if args.urlfile:
urlfile = os.path.abspath(args.urlfile)
logger.info("Reading groups from: '%s'", urlfile)
with open(urlfile) as f:
lines = f.read().splitlines()
# string comments
regex = "^\s*#.*|^\s*$" # comments with # or blank lines
for i in lines:
clean_line = i.rstrip()
if not re.match(regex, clean_line):
group_dict[clean_line] = None
# scoop up any command line args
for i in args.mugs:
group_dict[i] = None
writer = MeetupWriter(apikey, batchID, mdb, reshape=True)
if "all" in args.phases:
phases = ["groups", "members", "upcomingevents", "pastevents"]
else:
phases = args.phases
if args.admin:
logger.info("--admin : we will collect attendee info")
phases.append("attendees")
else:
logger.info("No admin account")
logger.info(
"We will not collect attendee info: ignoring attendees")
logger.info("Processing phases: %s", phases)
if "groups" in phases:
logger.info("processing group info for %i groups: collect=%s",
len(group_dict), args.collect)
writer.write_groups(group_dict.keys())
phases.remove("groups")
if "members" in phases:
logger.info("processing members info for %i groups: collect=%s",
len(group_dict), args.collect)
writer.write_members(group_dict.keys())
phases.remove("members")
for i in group_dict.keys():
writer.capture_snapshot(i, args.admin, phases)
audit.end_batch(batchID)
end = datetime.utcnow()
elapsed = end - start
logger.info("MUG processing took %s for BatchID : %i", elapsed,
batchID)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except pymongo.errors.ServerSelectionTimeoutError as e:
print("Failed to connect to MongoDB Server (server timeout): %s" % e)
sys.exit(2)