class Test_meetup_api(unittest.TestCase):
def setUp(self):
self._api = MeetupAPI()
def tearDown(self):
pass
def test_get_group(self):
g = self._api.get_group("DublinMUG")
self.assertTrue("city" in g and g["city"] == u"Dublin")
self.assertTrue("timezone" in g and g["timezone"] == u"Europe/Dublin")
self.assertTrue(g["urlname"] == u"DublinMUG")
self.assertTrue(g["id"] == 3478392)
self.assertTrue(g["country"] == u"IE")
#pprint.pprint( g )
self.assertTrue("location" in g)
def test_get_pro_groups(self):
g = self._api.get_pro_groups()
groups = list(g)
self.assertGreaterEqual(len(groups), 114)
def test_get_past_events(self):
g = self._api.get_past_events("DublinMUG")
events = list(g)
self.assertGreaterEqual(len(events), 29)
event = events[0]
#self.assertEqual( event[ "created"], 1335802792000 )
self.assertEqual(event["event_url"],
u'https://www.meetup.com/DublinMUG/events/62760772/')
def test_get_all_attendees(self):
attendees = self._api.get_all_attendees(
["DublinMUG", "London-MongoDB-User-Group"], items=400)
attendees = list(attendees)
self.assertGreaterEqual(len(attendees), 1306)
(attendee, event) = attendees[0]
self.assertTrue(u"member" in attendee)
self.assertTrue(u"rsvp" in attendee)
self.assertTrue(u"status" in attendee)
self.assertTrue(u"announced" in event)
self.assertTrue(u"group" in event)
self.assertTrue(u"name" in event)
self.assertEqual(event["rsvp_limit"], 80)
def test_get_aamember_by_id(self):
member = self._api.get_member_by_id(210984049)
self.assertEqual(member["name"], u"Julio Román")
#print( member[ "name"] )
self.assertEqual(type(member["name"]), types.UnicodeType)
class Test_meetup_api(unittest.TestCase):
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
def tearDown(self):
pass
def test_get_group(self):
g = self._api.get_group("DublinMUG")
self.assertTrue("city" in g and g["city"] == u"Dublin")
self.assertTrue("timezone" in g and g["timezone"] == u"Europe/Dublin")
self.assertTrue(g["urlname"] == u"DublinMUG")
self.assertTrue(g["id"] == 3478392)
self.assertTrue(g["country"] == u"IE")
#pprint.pprint( g )
self.assertTrue("location" in g)
self.assertTrue(g.has_key("created"))
def test_get_pro_groups(self):
g = self._api.get_pro_groups()
count = 0
for i in g:
self.assertTrue("rsvps_per_event" in i)
self.assertTrue("pro_join_date" in i)
self.assertTrue("founded_date" in i)
self.assertTrue(isinstance(i["pro_join_date"], datetime))
self.assertTrue(isinstance(i["founded_date"], datetime))
count = count + 1
self.assertGreaterEqual(count, 116)
def test_get_past_events(self):
g = self._api.get_past_events("DublinMUG")
events = list(g)
self.assertGreaterEqual(len(events), 29)
event = events[0]
#self.assertEqual( event[ "created"], 1335802792000 )
self.assertEqual(event["event_url"],
u'https://www.meetup.com/DublinMUG/events/62760772/')
self.assertTrue(isinstance(event["created"], datetime))
def test_get_all_attendees(self):
attendees = self._api.get_all_attendees(
["DublinMUG", "London-MongoDB-User-Group"])
attendees = list(attendees)
self.assertGreaterEqual(len(attendees), 1306)
(attendee, event) = attendees[0]
self.assertTrue(u"member" in attendee)
self.assertTrue(u"rsvp" in attendee)
self.assertTrue(u"status" in attendee)
self.assertTrue(u"announced" in event)
self.assertTrue(u"group" in event)
self.assertTrue(u"name" in event)
self.assertEqual(event["rsvp_limit"], 80)
def test_get_member_by_id(self):
member = self._api.get_member_by_id(210984049)
self.assertEqual(member["name"], u"Julio Román")
#print( member[ "name"] )
self.assertEqual(type(member["name"]), types.UnicodeType)
def test_get_member_by_url(self):
members = self._api.get_members(
["DublinMUG", "London-MongoDB-User-Group"])
self.assertGreaterEqual((sum(1 for _ in members)), 2465)
def test_get_members(self):
members = self._api.get_members(["DublinMUG"])
count = 0
for _ in members:
count = count + 1
#print( "%i %s" % (count, i ))
#print( i )
self.assertGreaterEqual(count, 844)
members = list(self._api.get_pro_members())
self.assertGreaterEqual((sum(1 for _ in members)), 17400)
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI()
else:
m = MeetupAPI(apikey=args.apikey)
for i in range(args.loop):
if args.member_id:
member = m.get_member_by_id(args.member_id)
if member.has_key("name"):
print(member["name"])
else:
print(member["member_name"])
if args.groups:
for i in args.mugs:
mug = m.get_group(i)
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
for i in args.mugs:
it = m.get_members(i)
count = 0
name = ""
mid = ""
for j in it:
#pprint.pprint( i )
count = count + 1
if j.has_key("name"):
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
past_events = m.get_past_events(args.pastevents)
printCursor(past_events)
if args.upcomingevents:
upcoming_events = m.get_upcoming_events(args.upcomingevents)
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
class MeetupWriter(object):
'''
A class that reads data about MUGS from the Meetup API using the MeetupAPI class and writes that
data to a MongoDB collection. Supports pro and no pro APIs
'''
def __init__(self, audit, mdb, urls, apikey= get_meetup_key(), unordered=True ):
'''
Write contents of meetup API to MongoDB
'''
self._mdb = mdb
self._meetup_api = MeetupAPI( apikey )
self._audit = audit
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
self._mugs = []
self._unordered = unordered
self._urls = urls
def process(self, collection, retrievalGenerator, processFunc, newFieldName ):
'''
Call batchWriter with a collection. Use retrievalGenerator to get a single
document (this should be a generator function). Use processFunc to tranform the
document into a new doc (it should take a doc and return a doc).
Write the new doc using the newFieldName.
Write is done using a generator as well. The write receiver accumulates writes until a threshold
is reached and then writes them as a batch using BatchWriter.
'''
bw = BatchWriter( collection, processFunc, newFieldName, orderedWrites=self._unordered )
writer = bw.bulkWrite( writeLimit=1)
for i in retrievalGenerator :
writer.send( i )
def processAttendees( self, group ):
writer = self._meetup_api.get_attendees( group )
newWriter = mergeEvents( writer )
self.process( self._attendees, newWriter, self._audit.addTimestamp, "info" )
def processGroup(self, url_name, groupName="group"):
group = self._meetup_api.get_group( url_name )
newDoc = self._audit.addTimestamp( groupName, group )
self._groups.insert_one( newDoc )
return newDoc
def updateGroup(self, groupName, doc ):
self._mugs.append( doc[ "urlname" ])
return self._audit.addTimestamp( groupName, doc )
def processGroups(self, nopro ):
if nopro:
groups = self.get_groups()
else:
groups = self._meetup_api.get_pro_groups()
self.process( self._groups, groups, self.updateGroup, "group" )
def get_groups(self ):
for i in self._urls:
yield self._meetup_api.get_group( i )
def processPastEvents(self, url_name ):
pastEvents = self._meetup_api.get_past_events( url_name )
self.process( self._pastEvents, pastEvents, self._audit.addTimestamp, "event" )
def processUpcomingEvents(self, url_name ):
upcomingEvents = self._meetup_api.get_upcoming_events( url_name )
self.process( self._upcomingEvents, upcomingEvents, self._audit.addTimestamp, "event" )
def processMembers( self, nopro=True ):
if nopro:
members = self.get_members()
else:
members = self._meetup_api.get_pro_members()
self.process( self._members, members, self._audit.addTimestamp, "member" )
def get_members(self ):
for i in self._urls:
for member in self._meetup_api.get_members( i ):
# if member.has_key( "name" ) :
# print( member[ "name"] )
# else:
# pprint.pprint( member )
yield member
def mug_list(self):
return self._mugs
def capture_snapshot(self, url_name, admin_arg, phases ):
try :
for i in phases:
if i == "pastevents" :
logging.info( "process past events for : '%s'", url_name )
self.processPastEvents( url_name )
elif i == "upcomingevents" :
logging.info( "process upcoming events for : '%s'", url_name )
self.processUpcomingEvents( url_name )
elif i == "attendees" :
if admin_arg:
logging.info( "process attendees : '%s'", url_name )
self.processAttendees( url_name )
else:
logging.warn( "You have not specified the admin arg")
logging.warn( "You must be a meetup admin user to request attendees")
logging.warn( "Ignoring phase 'attendees")
else:
logging.warn( "ignoring phase '%s': not a valid execution phase", i )
except HTTPError, e :
logging.fatal( "Stopped processing: %s", e )
raise
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument(
"--allgroups",
default=False,
action="store_true",
help="list group data for all groups for this API key")
parser.add_argument(
"--progroups",
default=False,
action="store_true",
help="list group data for all pro groups for this API key")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
parser.add_argument("--reshape",
default=False,
action="store_true",
help="Reshape output for BSON")
parser.add_argument("--req",
default=False,
action="store_true",
help="Report underlying request URL to meetup")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI(apikey=get_meetup_key(), reshape=args.reshape)
else:
m = MeetupAPI(apikey=args.apikey, reshape=args.reshape)
for i in range(args.loop):
if args.member_id and not (args.progroups or args.allgroups):
(url, member) = m.get_member_by_id(args.member_id)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(member)
if args.allgroups:
member_total = 0
for group_count, g in enumerate(m.get_groups(), 1):
#pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g[1]
full_group = m.get_group(group['urlname'])[1]
if full_group["organizer"]["id"] == args.member_id:
pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.progroups:
member_total = 0
group_count = 0
for url, g in m.get_groups():
print(f"URL :{url}")
pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g
url, full_group = m.get_group(group['urlname'])
if "pro_network" in full_group and full_group[
"pro_network"]["name"] == "MongoDB":
#pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
group_count = group_count + 1
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.groups:
for i in args.mugs:
(url, mug) = m.get_group(i)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
# it = m.get_members( args.mugs )
count = 0
name = ""
mid = ""
for (url, j) in m.get_members(args.mugs):
if args.req:
print("req: '{}'".format(url))
count = count + 1
if "name" in j:
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
(url, past_events) = m.get_past_events(args.pastevents)
if args.req:
print("req: '{}'".format(url))
printCursor(past_events)
if args.upcomingevents:
(url,
upcoming_events) = m.get_upcoming_events(args.upcomingevents)
if args.req:
print("req: '{}'".format(url))
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
return 0