class Test(unittest.TestCase):
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
def test_get_member_by_url(self):
members = self._api.get_members(
["DublinMUG", "London-MongoDB-User-Group"])
self.assertGreaterEqual(len(list(members)), 2470)
class Test_meetup_api(unittest.TestCase):
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
def tearDown(self):
pass
def test_get_group(self):
g = self._api.get_group("DublinMUG")
self.assertTrue("city" in g and g["city"] == u"Dublin")
self.assertTrue("timezone" in g and g["timezone"] == u"Europe/Dublin")
self.assertTrue(g["urlname"] == u"DublinMUG")
self.assertTrue(g["id"] == 3478392)
self.assertTrue(g["country"] == u"IE")
#pprint.pprint( g )
self.assertTrue("location" in g)
self.assertTrue(g.has_key("created"))
def test_get_pro_groups(self):
g = self._api.get_pro_groups()
count = 0
for i in g:
self.assertTrue("rsvps_per_event" in i)
self.assertTrue("pro_join_date" in i)
self.assertTrue("founded_date" in i)
self.assertTrue(isinstance(i["pro_join_date"], datetime))
self.assertTrue(isinstance(i["founded_date"], datetime))
count = count + 1
self.assertGreaterEqual(count, 116)
def test_get_past_events(self):
g = self._api.get_past_events("DublinMUG")
events = list(g)
self.assertGreaterEqual(len(events), 29)
event = events[0]
#self.assertEqual( event[ "created"], 1335802792000 )
self.assertEqual(event["event_url"],
u'https://www.meetup.com/DublinMUG/events/62760772/')
self.assertTrue(isinstance(event["created"], datetime))
def test_get_all_attendees(self):
attendees = self._api.get_all_attendees(
["DublinMUG", "London-MongoDB-User-Group"])
attendees = list(attendees)
self.assertGreaterEqual(len(attendees), 1306)
(attendee, event) = attendees[0]
self.assertTrue(u"member" in attendee)
self.assertTrue(u"rsvp" in attendee)
self.assertTrue(u"status" in attendee)
self.assertTrue(u"announced" in event)
self.assertTrue(u"group" in event)
self.assertTrue(u"name" in event)
self.assertEqual(event["rsvp_limit"], 80)
def test_get_member_by_id(self):
member = self._api.get_member_by_id(210984049)
self.assertEqual(member["name"], u"Julio Román")
#print( member[ "name"] )
self.assertEqual(type(member["name"]), types.UnicodeType)
def test_get_member_by_url(self):
members = self._api.get_members(
["DublinMUG", "London-MongoDB-User-Group"])
self.assertGreaterEqual((sum(1 for _ in members)), 2465)
def test_get_members(self):
members = self._api.get_members(["DublinMUG"])
count = 0
for _ in members:
count = count + 1
#print( "%i %s" % (count, i ))
#print( i )
self.assertGreaterEqual(count, 844)
members = list(self._api.get_pro_members())
self.assertGreaterEqual((sum(1 for _ in members)), 17400)
class MeetupWriter(object):
"""
A class that reads data about MUGS from the Meetup API using the MeetupAPI class and writes that
data to a MongoDB collection. Supports pro and no pro APIs
The process function is a generic reader function that takes a retrieval generator (provided by
the MeetupAPI class and a processing function. It iterates over the docs returned by the
retrieval generator and transforms then with "processFunc". The results are returned in an
embedded document with the key "newFieldname".
"""
INSERT_SIZE = 1000
def _addTimestamp(self, doc):
if "timestamp" in doc:
raise ValueError("cannot add timestamp, \
'timestamp' field already exists")
if "batchID" in doc:
raise ValueError("cannot add batchID, \
'batchID' field already exists")
doc["timestamp"] = datetime.datetime.utcnow()
doc["batchID"] = self._batch_ID
return doc
def __init__(self, apikey, batch_ID, mdb, reshape=True, unordered=True):
"""Write contents of meetup API to MongoDB"""
self._mdb = mdb
self._meetup_api = MeetupAPI(apikey, reshape=reshape)
self._batch_ID = batch_ID
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
# self._mugs = []
self._unordered = unordered
self._members_set = set()
self._logger = logging.getLogger(__programName__)
def update_members(self, retrievalGenerator, processFunc):
'''
For nopro collections we count the members in each group. To avoid double counting
we use update to overwrite previous records with the same member.
'''
docs = []
count = 0
# print( "update_members")
for url, i in retrievalGenerator:
# ignore already inserted members a member may be in multiple
# groups.
if self._members.find_one({
"batchID": self._batch_ID,
"id": i["id"]
}):
continue
else:
# print( "inserting : %s" %i["id"] )
count = count + 1
docs.append(processFunc(i))
if count == 500:
self._members.insert_many(docs)
docs = []
count = 0
if count > 0:
self._members.insert_many(docs)
docs = []
count = 0
def write(self, collection, retrievalGenerator, processFunc):
"""
:param collection: The collection to write too
:param retrievalGenerator: a generator that produces docs
:param processFunc: a preprocessing function for the docs to be written
:return: The number of docs written
"""
'''
Use retrievalGenerator to get a single
document (this should be a generator function). Use processFunc to tranform the
document into a new doc (it should take a doc and return a doc).
Write the new doc using the newFieldName.
Write is done using a generator as well. The write receiver accumulates writes until a threshold
is reached and then writes them as a batch using BatchWriter.
'''
docs = []
count = 0
# print( "write")
for url, i in retrievalGenerator:
docs.append(processFunc(i))
if len(docs) == MeetupWriter.INSERT_SIZE:
count = count + len(docs)
# print( "inserted 500")
collection.insert_many(docs)
docs = []
if len(docs) > 0:
collection.insert_many(docs)
count = count + len(docs)
return count
def write_Attendees(self, group):
writer = self._meetup_api.get_attendees(group)
self.write(self._attendees, writer, self._addTimestamp)
# def write_group(self, url_name, groupName="group"):
# group = self._meetup_api.get_group( url_name )
# newDoc = self._addTimestamp( groupName, group )
# self._groups.insert_one( newDoc )
# return newDoc
# def updateGroup(self, groupName, doc ):
# self._mugs.append( doc[ "urlname" ])
#
# return self._addTimestamp( groupName, doc )
def write_nopro_groups(self, mug_list):
groups = self._meetup_api.get_groups_by_url(mug_list)
self.write(self._groups, groups, self._addTimestamp)
def select_groups(self, groups, urls):
for url, g in groups:
if g["urlname"] in urls:
# print(g["urlname"])
yield url, g
# def write_pro_groups(self, urls):
#
# groups = self._meetup_api.get_pro_groups()
# self.write(self._pro_groups, self.select_groups(groups, urls), self._addTimestamp)
def write_groups(self, urls):
"""
The old pro API has been disabled by the numbskulls at Meetup so no both
pro and no pro APIs use the same get_group call.
:param urls: List of urlnames to get group info for
:return: No of groups written
"""
groups = self._meetup_api.get_groups_by_url(urls)
return self.write(self._groups, groups, self._addTimestamp)
# self.write_nopro_groups(urls)
# if collect == "nopro":
# self.write_nopro_groups(urls)
# elif collect == "pro":
# self.write_pro_groups(urls)
# else:
# self.write_pro_groups(urls)
# self.write_nopro_groups(urls)
def write_PastEvents(self, url_name):
pastEvents = self._meetup_api.get_past_events(url_name)
self.write(self._pastEvents, pastEvents, self._addTimestamp)
def write_UpcomingEvents(self, url_name):
upcomingEvents = self._meetup_api.get_upcoming_events(url_name)
self.write(self._upcomingEvents, upcomingEvents, self._addTimestamp)
# def write_pro_members(self):
# members = self._meetup_api.get_pro_members()
# self.write(self._pro_members, members, self._addTimestamp)
#
# def write_nopro_members(self, urls):
# members = self._meetup_api.get_members(urls)
# self.update_members(members, self._addTimestamp)
def write_members(self, urls):
members = self._meetup_api.get_members(urls)
self.update_members(members, self._addTimestamp)
# if collect == "nopro":
# self.write_nopro_members(urls)
# elif collect == "pro":
# self.write_pro_members()
# else:
# self.write_pro_members()
# self.write_nopro_members(urls)
# def mug_list(self):
# return self._mugs
def capture_snapshot(self, url_name, admin_arg, phases):
try:
for i in phases:
if i == "pastevents":
self._logger.info("process past events for : '%s'",
url_name)
self.write_PastEvents(url_name)
elif i == "upcomingevents":
self._logger.info("process upcoming events for : '%s'",
url_name)
self.write_UpcomingEvents(url_name)
elif i == "attendees":
if admin_arg:
self._logger.info(
"process attendees : '%s'", url_name)
self.write_Attendees(url_name)
else:
self._logger.warning(
"You have not specified the admin arg")
self._logger.warning(
"You must be a meetup admin user to request attendees"
)
self._logger.warning("Ignoring phase 'attendees'")
else:
self._logger.warn(
"ignoring phase '%s': not a valid execution phase", i)
except HTTPError as e:
self._logger.fatal("Stopped processing: %s", e)
raise
class MeetupWriter(object):
'''
A class that reads data about MUGS from the Meetup API using the MeetupAPI class and writes that
data to a MongoDB collection. Supports pro and no pro APIs
'''
def __init__(self, audit, mdb, urls, apikey= get_meetup_key(), unordered=True ):
'''
Write contents of meetup API to MongoDB
'''
self._mdb = mdb
self._meetup_api = MeetupAPI( apikey )
self._audit = audit
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
self._mugs = []
self._unordered = unordered
self._urls = urls
def process(self, collection, retrievalGenerator, processFunc, newFieldName ):
'''
Call batchWriter with a collection. Use retrievalGenerator to get a single
document (this should be a generator function). Use processFunc to tranform the
document into a new doc (it should take a doc and return a doc).
Write the new doc using the newFieldName.
Write is done using a generator as well. The write receiver accumulates writes until a threshold
is reached and then writes them as a batch using BatchWriter.
'''
bw = BatchWriter( collection, processFunc, newFieldName, orderedWrites=self._unordered )
writer = bw.bulkWrite( writeLimit=1)
for i in retrievalGenerator :
writer.send( i )
def processAttendees( self, group ):
writer = self._meetup_api.get_attendees( group )
newWriter = mergeEvents( writer )
self.process( self._attendees, newWriter, self._audit.addTimestamp, "info" )
def processGroup(self, url_name, groupName="group"):
group = self._meetup_api.get_group( url_name )
newDoc = self._audit.addTimestamp( groupName, group )
self._groups.insert_one( newDoc )
return newDoc
def updateGroup(self, groupName, doc ):
self._mugs.append( doc[ "urlname" ])
return self._audit.addTimestamp( groupName, doc )
def processGroups(self, nopro ):
if nopro:
groups = self.get_groups()
else:
groups = self._meetup_api.get_pro_groups()
self.process( self._groups, groups, self.updateGroup, "group" )
def get_groups(self ):
for i in self._urls:
yield self._meetup_api.get_group( i )
def processPastEvents(self, url_name ):
pastEvents = self._meetup_api.get_past_events( url_name )
self.process( self._pastEvents, pastEvents, self._audit.addTimestamp, "event" )
def processUpcomingEvents(self, url_name ):
upcomingEvents = self._meetup_api.get_upcoming_events( url_name )
self.process( self._upcomingEvents, upcomingEvents, self._audit.addTimestamp, "event" )
def processMembers( self, nopro=True ):
if nopro:
members = self.get_members()
else:
members = self._meetup_api.get_pro_members()
self.process( self._members, members, self._audit.addTimestamp, "member" )
def get_members(self ):
for i in self._urls:
for member in self._meetup_api.get_members( i ):
# if member.has_key( "name" ) :
# print( member[ "name"] )
# else:
# pprint.pprint( member )
yield member
def mug_list(self):
return self._mugs
def capture_snapshot(self, url_name, admin_arg, phases ):
try :
for i in phases:
if i == "pastevents" :
logging.info( "process past events for : '%s'", url_name )
self.processPastEvents( url_name )
elif i == "upcomingevents" :
logging.info( "process upcoming events for : '%s'", url_name )
self.processUpcomingEvents( url_name )
elif i == "attendees" :
if admin_arg:
logging.info( "process attendees : '%s'", url_name )
self.processAttendees( url_name )
else:
logging.warn( "You have not specified the admin arg")
logging.warn( "You must be a meetup admin user to request attendees")
logging.warn( "Ignoring phase 'attendees")
else:
logging.warn( "ignoring phase '%s': not a valid execution phase", i )
except HTTPError, e :
logging.fatal( "Stopped processing: %s", e )
raise
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI()
else:
m = MeetupAPI(apikey=args.apikey)
for i in range(args.loop):
if args.member_id:
member = m.get_member_by_id(args.member_id)
if member.has_key("name"):
print(member["name"])
else:
print(member["member_name"])
if args.groups:
for i in args.mugs:
mug = m.get_group(i)
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
for i in args.mugs:
it = m.get_members(i)
count = 0
name = ""
mid = ""
for j in it:
#pprint.pprint( i )
count = count + 1
if j.has_key("name"):
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
past_events = m.get_past_events(args.pastevents)
printCursor(past_events)
if args.upcomingevents:
upcoming_events = m.get_upcoming_events(args.upcomingevents)
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument(
"--allgroups",
default=False,
action="store_true",
help="list group data for all groups for this API key")
parser.add_argument(
"--progroups",
default=False,
action="store_true",
help="list group data for all pro groups for this API key")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
parser.add_argument("--reshape",
default=False,
action="store_true",
help="Reshape output for BSON")
parser.add_argument("--req",
default=False,
action="store_true",
help="Report underlying request URL to meetup")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI(apikey=get_meetup_key(), reshape=args.reshape)
else:
m = MeetupAPI(apikey=args.apikey, reshape=args.reshape)
for i in range(args.loop):
if args.member_id and not (args.progroups or args.allgroups):
(url, member) = m.get_member_by_id(args.member_id)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(member)
if args.allgroups:
member_total = 0
for group_count, g in enumerate(m.get_groups(), 1):
#pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g[1]
full_group = m.get_group(group['urlname'])[1]
if full_group["organizer"]["id"] == args.member_id:
pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.progroups:
member_total = 0
group_count = 0
for url, g in m.get_groups():
print(f"URL :{url}")
pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g
url, full_group = m.get_group(group['urlname'])
if "pro_network" in full_group and full_group[
"pro_network"]["name"] == "MongoDB":
#pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
group_count = group_count + 1
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.groups:
for i in args.mugs:
(url, mug) = m.get_group(i)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
# it = m.get_members( args.mugs )
count = 0
name = ""
mid = ""
for (url, j) in m.get_members(args.mugs):
if args.req:
print("req: '{}'".format(url))
count = count + 1
if "name" in j:
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
(url, past_events) = m.get_past_events(args.pastevents)
if args.req:
print("req: '{}'".format(url))
printCursor(past_events)
if args.upcomingevents:
(url,
upcoming_events) = m.get_upcoming_events(args.upcomingevents)
if args.req:
print("req: '{}'".format(url))
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
return 0