class Test_meetup_api(unittest.TestCase):
def setUp(self):
self._api = MeetupAPI()
def tearDown(self):
pass
def test_get_group(self):
g = self._api.get_group("DublinMUG")
self.assertTrue("city" in g and g["city"] == u"Dublin")
self.assertTrue("timezone" in g and g["timezone"] == u"Europe/Dublin")
self.assertTrue(g["urlname"] == u"DublinMUG")
self.assertTrue(g["id"] == 3478392)
self.assertTrue(g["country"] == u"IE")
#pprint.pprint( g )
self.assertTrue("location" in g)
def test_get_pro_groups(self):
g = self._api.get_pro_groups()
groups = list(g)
self.assertGreaterEqual(len(groups), 114)
def test_get_past_events(self):
g = self._api.get_past_events("DublinMUG")
events = list(g)
self.assertGreaterEqual(len(events), 29)
event = events[0]
#self.assertEqual( event[ "created"], 1335802792000 )
self.assertEqual(event["event_url"],
u'https://www.meetup.com/DublinMUG/events/62760772/')
def test_get_all_attendees(self):
attendees = self._api.get_all_attendees(
["DublinMUG", "London-MongoDB-User-Group"], items=400)
attendees = list(attendees)
self.assertGreaterEqual(len(attendees), 1306)
(attendee, event) = attendees[0]
self.assertTrue(u"member" in attendee)
self.assertTrue(u"rsvp" in attendee)
self.assertTrue(u"status" in attendee)
self.assertTrue(u"announced" in event)
self.assertTrue(u"group" in event)
self.assertTrue(u"name" in event)
self.assertEqual(event["rsvp_limit"], 80)
def test_get_aamember_by_id(self):
member = self._api.get_member_by_id(210984049)
self.assertEqual(member["name"], u"Julio Román")
#print( member[ "name"] )
self.assertEqual(type(member["name"]), types.UnicodeType)
def __init__(self, audit, mdb, urls, apikey= get_meetup_key(), unordered=True ):
'''
Write contents of meetup API to MongoDB
'''
self._mdb = mdb
self._meetup_api = MeetupAPI( apikey )
self._audit = audit
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
self._mugs = []
self._unordered = unordered
self._urls = urls
def __init__(self, apikey, batch_ID, mdb, reshape=True, unordered=True):
"""Write contents of meetup API to MongoDB"""
self._mdb = mdb
self._meetup_api = MeetupAPI(apikey, reshape=reshape)
self._batch_ID = batch_ID
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
# self._mugs = []
self._unordered = unordered
self._members_set = set()
self._logger = logging.getLogger(__programName__)
class Test(unittest.TestCase):
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
def test_get_member_by_url(self):
members = self._api.get_members(
["DublinMUG", "London-MongoDB-User-Group"])
self.assertGreaterEqual(len(list(members)), 2470)
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
if argv:
sys.argv.extend(argv)
try:
# Setup argument parser
parser = ArgumentParser(description='''
Read data from the Meetup API and write it do a MongoDB database. Each run of this program
creates a new batch of data identified by a batchID. The default database is MUGS. You can change
this by using the --host parameter and specifying a different database in the mongodb URI.
If you use the --pro arguement your API key must be a meetup pro account API key. If not the api
calls to the pro interface will fail.
If you are and adminstrator on the pro account you should use the --admin flag to give you
access to the admin APIs.
''')
#
# MongoDB Args
parser.add_argument('--host',
default="mongodb://localhost:27017/MUGS",
help='URI to connect to : [default: %(default)s]')
parser.add_argument("--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument("-v",
"--version",
action='version',
version=__programName__ + " " + __version__)
parser.add_argument(
'--trialrun',
action="store_true",
default=False,
help='Trial run, no updates [default: %(default)s]')
parser.add_argument(
'--mugs',
nargs="+",
help='Process MUGs list list mugs by name [default: %(default)s]')
parser.add_argument(
"--pro",
default=False,
action="store_true",
help="use if you have a pro account uses pro API calls")
parser.add_argument(
"--admin",
default=False,
action="store_true",
help="Some calls are only available to admin users")
parser.add_argument(
"--database",
default="MUGS",
help="Default database name to write to [default: %(default)s]")
parser.add_argument('--phases',
nargs="+",
choices=[
"groups", "members", "attendees",
"upcomingevents", "pastevents"
],
default=["all"],
help='execution phases')
parser.add_argument(
'--loglevel',
default="INFO",
choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
help='Logging level [default: %(default)s]')
parser.add_argument('--apikey',
default=None,
help='Default API key for meetup')
parser.add_argument(
'--urlfile',
help=
"File containing a list of MUG URLs to be used to parse data [ default: %(default)s]"
)
# Process arguments
args = parser.parse_args()
apikey = ""
if args.apikey:
apikey = args.apikey
else:
apikey = get_meetup_key()
verbose = args.verbose
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string,
level=LoggingLevel(args.loglevel))
# Turn off logging for requests
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
if verbose > 0:
logging.info("Verbose mode on")
if args.urlfile:
if not os.path.isfile(args.urlfile):
print("No such file --urlfile '%s'" % args.urlfile)
sys.exit(1)
if args.mugs:
mugList = args.mugs
else:
mugList = []
if args.pro:
nopro = False
else:
nopro = True
mdb = MUGAlyserMongoDB(args.host)
audit = Audit(mdb)
batchID = audit.startBatch(
{
"args": vars(args),
"version": __programName__ + " " + __version__,
"pro_account": args.pro
},
trial=args.trialrun,
apikey=apikey)
start = datetime.utcnow()
logging.info("Started MUG processing for batch ID: %i", batchID)
logging.info("Writing to database : '%s'", mdb.database().name)
if nopro:
logging.info("Using standard API calls (no pro account API key)")
if args.urlfile:
logging.info("Reading groups from: '%s'", args.urlfile)
with open(args.urlfile) as f:
mugList = f.read().splitlines()
else:
logging.info("Using pro API calls (pro account API key)")
if nopro:
logging.info("Processing %i MUG URLS", len(mugList))
else:
mugList = list(MeetupAPI().get_pro_group_names())
writer = MeetupWriter(audit, mdb, mugList, apikey)
if "all" in args.phases:
phases = ["groups", "members", "upcomingevents", "pastevents"]
if args.admin:
phases.append("attendees")
else:
phases = args.phases
if "groups" in phases:
logging.info("processing group info for %i groups: nopro=%s",
len(mugList), nopro)
writer.processGroups(nopro)
phases.remove("groups")
if "members" in phases:
logging.info("processing members info for %i groups: nopro=%s",
len(mugList), nopro)
writer.processMembers(nopro)
phases.remove("members")
for i in mugList:
writer.capture_snapshot(i, args.admin, phases)
audit.endBatch(batchID)
end = datetime.utcnow()
elapsed = end - start
logging.info("MUG processing took %s for BatchID : %i", elapsed,
batchID)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except pymongo.errors.ServerSelectionTimeoutError, e:
print("Failed to connect to MongoDB Server (server timeout): %s" % e)
sys.exit(2)
def setUp(self):
self._api = MeetupAPI()
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
class Test_meetup_api(unittest.TestCase):
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
def tearDown(self):
pass
def test_get_group(self):
g = self._api.get_group("DublinMUG")
self.assertTrue("city" in g and g["city"] == u"Dublin")
self.assertTrue("timezone" in g and g["timezone"] == u"Europe/Dublin")
self.assertTrue(g["urlname"] == u"DublinMUG")
self.assertTrue(g["id"] == 3478392)
self.assertTrue(g["country"] == u"IE")
#pprint.pprint( g )
self.assertTrue("location" in g)
self.assertTrue(g.has_key("created"))
def test_get_pro_groups(self):
g = self._api.get_pro_groups()
count = 0
for i in g:
self.assertTrue("rsvps_per_event" in i)
self.assertTrue("pro_join_date" in i)
self.assertTrue("founded_date" in i)
self.assertTrue(isinstance(i["pro_join_date"], datetime))
self.assertTrue(isinstance(i["founded_date"], datetime))
count = count + 1
self.assertGreaterEqual(count, 116)
def test_get_past_events(self):
g = self._api.get_past_events("DublinMUG")
events = list(g)
self.assertGreaterEqual(len(events), 29)
event = events[0]
#self.assertEqual( event[ "created"], 1335802792000 )
self.assertEqual(event["event_url"],
u'https://www.meetup.com/DublinMUG/events/62760772/')
self.assertTrue(isinstance(event["created"], datetime))
def test_get_all_attendees(self):
attendees = self._api.get_all_attendees(
["DublinMUG", "London-MongoDB-User-Group"])
attendees = list(attendees)
self.assertGreaterEqual(len(attendees), 1306)
(attendee, event) = attendees[0]
self.assertTrue(u"member" in attendee)
self.assertTrue(u"rsvp" in attendee)
self.assertTrue(u"status" in attendee)
self.assertTrue(u"announced" in event)
self.assertTrue(u"group" in event)
self.assertTrue(u"name" in event)
self.assertEqual(event["rsvp_limit"], 80)
def test_get_member_by_id(self):
member = self._api.get_member_by_id(210984049)
self.assertEqual(member["name"], u"Julio Román")
#print( member[ "name"] )
self.assertEqual(type(member["name"]), types.UnicodeType)
def test_get_member_by_url(self):
members = self._api.get_members(
["DublinMUG", "London-MongoDB-User-Group"])
self.assertGreaterEqual((sum(1 for _ in members)), 2465)
def test_get_members(self):
members = self._api.get_members(["DublinMUG"])
count = 0
for _ in members:
count = count + 1
#print( "%i %s" % (count, i ))
#print( i )
self.assertGreaterEqual(count, 844)
members = list(self._api.get_pro_members())
self.assertGreaterEqual((sum(1 for _ in members)), 17400)
class MeetupWriter(object):
"""
A class that reads data about MUGS from the Meetup API using the MeetupAPI class and writes that
data to a MongoDB collection. Supports pro and no pro APIs
The process function is a generic reader function that takes a retrieval generator (provided by
the MeetupAPI class and a processing function. It iterates over the docs returned by the
retrieval generator and transforms then with "processFunc". The results are returned in an
embedded document with the key "newFieldname".
"""
INSERT_SIZE = 1000
def _addTimestamp(self, doc):
if "timestamp" in doc:
raise ValueError("cannot add timestamp, \
'timestamp' field already exists")
if "batchID" in doc:
raise ValueError("cannot add batchID, \
'batchID' field already exists")
doc["timestamp"] = datetime.datetime.utcnow()
doc["batchID"] = self._batch_ID
return doc
def __init__(self, apikey, batch_ID, mdb, reshape=True, unordered=True):
"""Write contents of meetup API to MongoDB"""
self._mdb = mdb
self._meetup_api = MeetupAPI(apikey, reshape=reshape)
self._batch_ID = batch_ID
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
# self._mugs = []
self._unordered = unordered
self._members_set = set()
self._logger = logging.getLogger(__programName__)
def update_members(self, retrievalGenerator, processFunc):
'''
For nopro collections we count the members in each group. To avoid double counting
we use update to overwrite previous records with the same member.
'''
docs = []
count = 0
# print( "update_members")
for url, i in retrievalGenerator:
# ignore already inserted members a member may be in multiple
# groups.
if self._members.find_one({
"batchID": self._batch_ID,
"id": i["id"]
}):
continue
else:
# print( "inserting : %s" %i["id"] )
count = count + 1
docs.append(processFunc(i))
if count == 500:
self._members.insert_many(docs)
docs = []
count = 0
if count > 0:
self._members.insert_many(docs)
docs = []
count = 0
def write(self, collection, retrievalGenerator, processFunc):
"""
:param collection: The collection to write too
:param retrievalGenerator: a generator that produces docs
:param processFunc: a preprocessing function for the docs to be written
:return: The number of docs written
"""
'''
Use retrievalGenerator to get a single
document (this should be a generator function). Use processFunc to tranform the
document into a new doc (it should take a doc and return a doc).
Write the new doc using the newFieldName.
Write is done using a generator as well. The write receiver accumulates writes until a threshold
is reached and then writes them as a batch using BatchWriter.
'''
docs = []
count = 0
# print( "write")
for url, i in retrievalGenerator:
docs.append(processFunc(i))
if len(docs) == MeetupWriter.INSERT_SIZE:
count = count + len(docs)
# print( "inserted 500")
collection.insert_many(docs)
docs = []
if len(docs) > 0:
collection.insert_many(docs)
count = count + len(docs)
return count
def write_Attendees(self, group):
writer = self._meetup_api.get_attendees(group)
self.write(self._attendees, writer, self._addTimestamp)
# def write_group(self, url_name, groupName="group"):
# group = self._meetup_api.get_group( url_name )
# newDoc = self._addTimestamp( groupName, group )
# self._groups.insert_one( newDoc )
# return newDoc
# def updateGroup(self, groupName, doc ):
# self._mugs.append( doc[ "urlname" ])
#
# return self._addTimestamp( groupName, doc )
def write_nopro_groups(self, mug_list):
groups = self._meetup_api.get_groups_by_url(mug_list)
self.write(self._groups, groups, self._addTimestamp)
def select_groups(self, groups, urls):
for url, g in groups:
if g["urlname"] in urls:
# print(g["urlname"])
yield url, g
# def write_pro_groups(self, urls):
#
# groups = self._meetup_api.get_pro_groups()
# self.write(self._pro_groups, self.select_groups(groups, urls), self._addTimestamp)
def write_groups(self, urls):
"""
The old pro API has been disabled by the numbskulls at Meetup so no both
pro and no pro APIs use the same get_group call.
:param urls: List of urlnames to get group info for
:return: No of groups written
"""
groups = self._meetup_api.get_groups_by_url(urls)
return self.write(self._groups, groups, self._addTimestamp)
# self.write_nopro_groups(urls)
# if collect == "nopro":
# self.write_nopro_groups(urls)
# elif collect == "pro":
# self.write_pro_groups(urls)
# else:
# self.write_pro_groups(urls)
# self.write_nopro_groups(urls)
def write_PastEvents(self, url_name):
pastEvents = self._meetup_api.get_past_events(url_name)
self.write(self._pastEvents, pastEvents, self._addTimestamp)
def write_UpcomingEvents(self, url_name):
upcomingEvents = self._meetup_api.get_upcoming_events(url_name)
self.write(self._upcomingEvents, upcomingEvents, self._addTimestamp)
# def write_pro_members(self):
# members = self._meetup_api.get_pro_members()
# self.write(self._pro_members, members, self._addTimestamp)
#
# def write_nopro_members(self, urls):
# members = self._meetup_api.get_members(urls)
# self.update_members(members, self._addTimestamp)
def write_members(self, urls):
members = self._meetup_api.get_members(urls)
self.update_members(members, self._addTimestamp)
# if collect == "nopro":
# self.write_nopro_members(urls)
# elif collect == "pro":
# self.write_pro_members()
# else:
# self.write_pro_members()
# self.write_nopro_members(urls)
# def mug_list(self):
# return self._mugs
def capture_snapshot(self, url_name, admin_arg, phases):
try:
for i in phases:
if i == "pastevents":
self._logger.info("process past events for : '%s'",
url_name)
self.write_PastEvents(url_name)
elif i == "upcomingevents":
self._logger.info("process upcoming events for : '%s'",
url_name)
self.write_UpcomingEvents(url_name)
elif i == "attendees":
if admin_arg:
self._logger.info(
"process attendees : '%s'", url_name)
self.write_Attendees(url_name)
else:
self._logger.warning(
"You have not specified the admin arg")
self._logger.warning(
"You must be a meetup admin user to request attendees"
)
self._logger.warning("Ignoring phase 'attendees'")
else:
self._logger.warn(
"ignoring phase '%s': not a valid execution phase", i)
except HTTPError as e:
self._logger.fatal("Stopped processing: %s", e)
raise
class MeetupWriter(object):
'''
A class that reads data about MUGS from the Meetup API using the MeetupAPI class and writes that
data to a MongoDB collection. Supports pro and no pro APIs
'''
def __init__(self, audit, mdb, urls, apikey= get_meetup_key(), unordered=True ):
'''
Write contents of meetup API to MongoDB
'''
self._mdb = mdb
self._meetup_api = MeetupAPI( apikey )
self._audit = audit
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
self._mugs = []
self._unordered = unordered
self._urls = urls
def process(self, collection, retrievalGenerator, processFunc, newFieldName ):
'''
Call batchWriter with a collection. Use retrievalGenerator to get a single
document (this should be a generator function). Use processFunc to tranform the
document into a new doc (it should take a doc and return a doc).
Write the new doc using the newFieldName.
Write is done using a generator as well. The write receiver accumulates writes until a threshold
is reached and then writes them as a batch using BatchWriter.
'''
bw = BatchWriter( collection, processFunc, newFieldName, orderedWrites=self._unordered )
writer = bw.bulkWrite( writeLimit=1)
for i in retrievalGenerator :
writer.send( i )
def processAttendees( self, group ):
writer = self._meetup_api.get_attendees( group )
newWriter = mergeEvents( writer )
self.process( self._attendees, newWriter, self._audit.addTimestamp, "info" )
def processGroup(self, url_name, groupName="group"):
group = self._meetup_api.get_group( url_name )
newDoc = self._audit.addTimestamp( groupName, group )
self._groups.insert_one( newDoc )
return newDoc
def updateGroup(self, groupName, doc ):
self._mugs.append( doc[ "urlname" ])
return self._audit.addTimestamp( groupName, doc )
def processGroups(self, nopro ):
if nopro:
groups = self.get_groups()
else:
groups = self._meetup_api.get_pro_groups()
self.process( self._groups, groups, self.updateGroup, "group" )
def get_groups(self ):
for i in self._urls:
yield self._meetup_api.get_group( i )
def processPastEvents(self, url_name ):
pastEvents = self._meetup_api.get_past_events( url_name )
self.process( self._pastEvents, pastEvents, self._audit.addTimestamp, "event" )
def processUpcomingEvents(self, url_name ):
upcomingEvents = self._meetup_api.get_upcoming_events( url_name )
self.process( self._upcomingEvents, upcomingEvents, self._audit.addTimestamp, "event" )
def processMembers( self, nopro=True ):
if nopro:
members = self.get_members()
else:
members = self._meetup_api.get_pro_members()
self.process( self._members, members, self._audit.addTimestamp, "member" )
def get_members(self ):
for i in self._urls:
for member in self._meetup_api.get_members( i ):
# if member.has_key( "name" ) :
# print( member[ "name"] )
# else:
# pprint.pprint( member )
yield member
def mug_list(self):
return self._mugs
def capture_snapshot(self, url_name, admin_arg, phases ):
try :
for i in phases:
if i == "pastevents" :
logging.info( "process past events for : '%s'", url_name )
self.processPastEvents( url_name )
elif i == "upcomingevents" :
logging.info( "process upcoming events for : '%s'", url_name )
self.processUpcomingEvents( url_name )
elif i == "attendees" :
if admin_arg:
logging.info( "process attendees : '%s'", url_name )
self.processAttendees( url_name )
else:
logging.warn( "You have not specified the admin arg")
logging.warn( "You must be a meetup admin user to request attendees")
logging.warn( "Ignoring phase 'attendees")
else:
logging.warn( "ignoring phase '%s': not a valid execution phase", i )
except HTTPError, e :
logging.fatal( "Stopped processing: %s", e )
raise
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI()
else:
m = MeetupAPI(apikey=args.apikey)
for i in range(args.loop):
if args.member_id:
member = m.get_member_by_id(args.member_id)
if member.has_key("name"):
print(member["name"])
else:
print(member["member_name"])
if args.groups:
for i in args.mugs:
mug = m.get_group(i)
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
for i in args.mugs:
it = m.get_members(i)
count = 0
name = ""
mid = ""
for j in it:
#pprint.pprint( i )
count = count + 1
if j.has_key("name"):
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
past_events = m.get_past_events(args.pastevents)
printCursor(past_events)
if args.upcomingevents:
upcoming_events = m.get_upcoming_events(args.upcomingevents)
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument(
"--allgroups",
default=False,
action="store_true",
help="list group data for all groups for this API key")
parser.add_argument(
"--progroups",
default=False,
action="store_true",
help="list group data for all pro groups for this API key")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
parser.add_argument("--reshape",
default=False,
action="store_true",
help="Reshape output for BSON")
parser.add_argument("--req",
default=False,
action="store_true",
help="Report underlying request URL to meetup")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI(apikey=get_meetup_key(), reshape=args.reshape)
else:
m = MeetupAPI(apikey=args.apikey, reshape=args.reshape)
for i in range(args.loop):
if args.member_id and not (args.progroups or args.allgroups):
(url, member) = m.get_member_by_id(args.member_id)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(member)
if args.allgroups:
member_total = 0
for group_count, g in enumerate(m.get_groups(), 1):
#pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g[1]
full_group = m.get_group(group['urlname'])[1]
if full_group["organizer"]["id"] == args.member_id:
pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.progroups:
member_total = 0
group_count = 0
for url, g in m.get_groups():
print(f"URL :{url}")
pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g
url, full_group = m.get_group(group['urlname'])
if "pro_network" in full_group and full_group[
"pro_network"]["name"] == "MongoDB":
#pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
group_count = group_count + 1
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.groups:
for i in args.mugs:
(url, mug) = m.get_group(i)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
# it = m.get_members( args.mugs )
count = 0
name = ""
mid = ""
for (url, j) in m.get_members(args.mugs):
if args.req:
print("req: '{}'".format(url))
count = count + 1
if "name" in j:
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
(url, past_events) = m.get_past_events(args.pastevents)
if args.req:
print("req: '{}'".format(url))
printCursor(past_events)
if args.upcomingevents:
(url,
upcoming_events) = m.get_upcoming_events(args.upcomingevents)
if args.req:
print("req: '{}'".format(url))
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
return 0
def mugalyser(argv=None): # IGNORE:C0111
'''Command line options.'''
try:
# Setup argument parser
parser = ArgumentParser(description='''
Read data from the Meetup API and write it do a MongoDB database. Each run of this program
creates a new batch of data identified by a batchID. The default database is MUGS. You can change
this by using the --host parameter and specifying a different database in the mongodb URI.
If you use the --pro arguement your API key must be a meetup pro account API key. If not the api
calls to the pro interface will fail.
If you are and adminstrator on the pro account you should use the --admin flag to give you
access to the admin APIs.
''')
#
# MongoDB Args
parser.add_argument('--host',
default="mongodb://localhost:27017/MUGS",
help='URI to connect to : [default: %(default)s]')
parser.add_argument("-v",
"--version",
action='version',
version=__programName__ + " " + __version__)
parser.add_argument(
'--mugs',
nargs="+",
default=[],
help='Process MUGs list list mugs by name [default: %(default)s]')
parser.add_argument("--collect",
choices=["pro", "nopro", "all"],
default="all",
help="Use pro API calls, no pro API calls or both")
parser.add_argument(
"--admin",
default=False,
action="store_true",
help=
"Some calls are only available to admin users, use this if you are not an admin"
)
parser.add_argument(
"--database",
default="MUGS",
help="Default database name to write to [default: %(default)s]")
parser.add_argument('--phases',
nargs="+",
choices=[
"groups", "members", "attendees",
"upcomingevents", "pastevents"
],
default=["all"],
help='execution phases')
parser.add_argument(
'--loglevel',
default="INFO",
choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
help='Logging level [default: %(default)s]')
parser.add_argument('--apikey',
default=None,
help='Default API key for meetup')
parser.add_argument("--batchname",
default=__programName__,
help="Batch name used in creating audit batches")
parser.add_argument(
'--urlfile',
help=
"File containing a list of MUG URLs to be used to parse data [ default: %(default)s]"
)
parser.add_argument(
"--drop",
default=False,
action="store_true",
help="drop the database before writing data [default: %(default)s]"
)
parser.add_argument("--organizer_id",
type=int,
help="Organizer ID is required for non pro groups")
# Process arguments
args = parser.parse_args(argv)
apikey = ""
if args.apikey:
apikey = args.apikey
else:
apikey = get_meetup_key()
mugalyser_logger = Logger(__programName__, args.loglevel)
# mugalyser_logger.add_stream_handler( args.loglevel )
mugalyser_logger.add_file_handler("mugalyser.log", args.loglevel)
api = MeetupAPI(apikey, reshape=True)
logger = mugalyser_logger.log()
# Turn off logging for requests
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
mdb = MUGAlyserMongoDB(uri=args.host, database_name=args.database)
if args.drop:
logger.warn(f"Dropping database:'{args.database}'")
mdb.drop(args.database)
audit = Audit(mdb)
batchID = audit.start_batch({
"args": vars(args),
"version": __programName__ + " " + __version__,
"name": args.batchname
})
start = datetime.utcnow()
logger.info("Started MUG processing for batch ID: %i", batchID)
logger.info("Writing to database : '%s'", mdb.database().name)
group_dict = {}
count = 0
group_list = []
if args.mugs:
for url in args.mugs:
group_list.append(api.get_group(url))
else:
group_list = list(api.get_groups())
for url, group in group_list:
#print(f"Checking:{group['urlname']}")
urlname = group['urlname']
url, full_group = api.get_group(urlname)
if args.collect in ["pro", "all"]:
if "pro_network" in full_group and full_group["pro_network"][
"name"] == "MongoDB":
count = count + 1
logger.info(
f"{count}. Processing pro group: {group['urlname']}")
group_dict[urlname] = full_group
if args.collect in ["nopro", "all"]:
if args.organizer_id:
if full_group["organizer"]["id"] == args.organizer_id:
count = count + 1
logger.info(
f"{count}. Processing normal group: {group['urlname']}"
)
group_dict[urlname] = full_group
else:
logger.error(
"You must specify --organizer_id when collecting nopro groups"
)
sys.exit(1)
if args.urlfile:
urlfile = os.path.abspath(args.urlfile)
logger.info("Reading groups from: '%s'", urlfile)
with open(urlfile) as f:
lines = f.read().splitlines()
# string comments
regex = "^\s*#.*|^\s*$" # comments with # or blank lines
for i in lines:
clean_line = i.rstrip()
if not re.match(regex, clean_line):
group_dict[clean_line] = None
# scoop up any command line args
for i in args.mugs:
group_dict[i] = None
writer = MeetupWriter(apikey, batchID, mdb, reshape=True)
if "all" in args.phases:
phases = ["groups", "members", "upcomingevents", "pastevents"]
else:
phases = args.phases
if args.admin:
logger.info("--admin : we will collect attendee info")
phases.append("attendees")
else:
logger.info("No admin account")
logger.info(
"We will not collect attendee info: ignoring attendees")
logger.info("Processing phases: %s", phases)
if "groups" in phases:
logger.info("processing group info for %i groups: collect=%s",
len(group_dict), args.collect)
writer.write_groups(group_dict.keys())
phases.remove("groups")
if "members" in phases:
logger.info("processing members info for %i groups: collect=%s",
len(group_dict), args.collect)
writer.write_members(group_dict.keys())
phases.remove("members")
for i in group_dict.keys():
writer.capture_snapshot(i, args.admin, phases)
audit.end_batch(batchID)
end = datetime.utcnow()
elapsed = end - start
logger.info("MUG processing took %s for BatchID : %i", elapsed,
batchID)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except pymongo.errors.ServerSelectionTimeoutError as e:
print("Failed to connect to MongoDB Server (server timeout): %s" % e)
sys.exit(2)