def getCurrentValidBatchID( self ):
curBatch = self._auditCollection.find( { "apikey" : get_meetup_key(),
"end" : { "$type" : "date" }, #17 BSON type for timestamp
"trial" : False } ).sort( "batchID", pymongo.DESCENDING ).limit( 1 )
if curBatch is None :
raise ValueError( "No current valid batch ID" )
else:
try :
x= curBatch.next()[ "batchID"]
return x
except StopIteration :
raise ValueError( "Have you set a valid API key? (APIKEY='%s')" % get_meetup_key())
def __init__(self, apikey=get_meetup_key(), items=100):
'''
Constructor
'''
self._api = "https://api.meetup.com/"
self._params = {}
self._params["key"] = apikey
self._items = str(items)
self._requester = PaginatedRequest(self._items)
def testProcessMembers(self):
self._mdb = MUGAlyserMongoDB("mongodb://localhost:27017/TESTWRITER")
self._audit = Audit(self._mdb)
batchID = self._audit.start_batch({"test": 2})
self._writer = MeetupWriter(get_meetup_key(), batchID, self._mdb)
self._writer.write_members("pro", ["DublinMUG"])
self._writer.write_members("nopro", ["DublinMUG"])
self.assertTrue(self._mdb.proMembersCollection().find_one(
{"member.member_name": "Joe Drumgoole"}))
self.assertTrue(self._mdb.membersCollection().find_one(
{"member.name": "Joe Drumgoole"}))
self._audit.end_batch(batchID)
def test_write_group(self):
self._mdb = MUGAlyserMongoDB("mongodb://localhost:27017/TESTWRITER")
self._audit = Audit(self._mdb)
batchID = self._audit.start_batch({"test": 1})
self._writer = MeetupWriter(get_meetup_key(), batchID, self._mdb)
self._writer.write_groups("nopro", ["DublinMUG"])
self._writer.write_groups("pro", ["DublinMUG"])
self.assertTrue(self._mdb.groupsCollection().find_one(
{"group.urlname": "DublinMUG"}))
self.assertTrue(self._mdb.proGroupsCollection().find_one(
{"group.urlname": "DublinMUG"}))
self._audit.end_batch(batchID)
def getCurrentValidBatches( self, start=None, end=None ):
agg = Agg( self._auditCollection )
agg.addMatch({ "apikey" : get_meetup_key(),
"end" : { "$ne" : None },
"trial" : False } )
agg.addRangeMatch( "start", start, end)
agg.addSort( Sorter( batchID = pymongo.DESCENDING ))
# cursor = self._auditCollection.find( { "apikey" : get_meetup_key(),
# "end" : { "$ne" : None },
# "trial" : False } ).sort( "batchID", pymongo.DESCENDING )
return agg
def __init__(self, audit, mdb, urls, apikey= get_meetup_key(), unordered=True ):
'''
Write contents of meetup API to MongoDB
'''
self._mdb = mdb
self._meetup_api = MeetupAPI( apikey )
self._audit = audit
self._groups = self._mdb.groupsCollection()
self._members = self._mdb.membersCollection()
self._attendees = self._mdb.attendeesCollection()
self._pastEvents = self._mdb.pastEventsCollection()
self._upcomingEvents = self._mdb.upcomingEventsCollection()
self._mugs = []
self._unordered = unordered
self._urls = urls
def startBatch(self, doc, name=None, trial=False, apikey = get_meetup_key()):
thisBatchID = self.incrementBatchID()
if name is None :
name = "Standard Batch: %i" % thisBatchID
self._auditCollection.insert_one( { "batchID" : thisBatchID,
"start" : datetime.now(),
"trial" : trial,
"end" : None,
"name" : name,
"apikey" : apikey,
"info" : doc })
self._currentBatchID = thisBatchID
return thisBatchID
def test_getCurrentValidBatchID(self):
batchID1 = self._audit.startBatch(doc={"test": "doc"}, trial=True)
self._audit.endBatch(batchID1)
#self.assertRaises( ValueError, self._audit.getCurrentValidBatchID )
batchID2 = self._audit.startBatch(
{
"args": "arg list",
"version": __programName__ + " " + __version__
},
trial=False,
apikey=get_meetup_key())
self._audit.endBatch(batchID2)
self.assertEqual(batchID2, self._audit.getCurrentValidBatchID())
batchID3 = self._audit.startBatch(doc={"test": "doc"}, trial=True)
self._audit.endBatch(batchID3)
self.assertEqual(batchID2, self._audit.getCurrentValidBatchID())
def test_api_fail(self):
params={}
params[ "key" ] = get_meetup_key()
#params[ "group_urlname" ] = "MongoDBGlasgow"
params[ "group_urlname" ] = "London-MongoDB-User-Group"
params[ "page"] = 10
count = 0
print( "requests.get(%s, %s)" % ( "https://api.meetup.com/2/members", params ))
r = requests.get( "https://api.meetup.com/2/members", params )
count = count + 1
print( "requests count : %i" % count )
print( r.url )
_ = r.headers
body = r.json()
print( "results: %i" % len( body[ "results"]))
for i in body[ "results"]:
#print( i )
pass
#print( body[ "meta"])
while ( body[ 'meta' ][ "next" ] != "" ) :
print( "requests.get(%s)" % body['meta'][ 'next' ] )
r = requests.get( body['meta'][ 'next' ] )
count = count + 1
print( "requests count : %i" % count )
print( "next: %s" % r.url )
print( "body: '%s'" % r.text )
body = r.json()
print( "results: %i" % len( body ))
for i in body[ "results"]:
#print( i )
pass
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
if argv:
sys.argv.extend(argv)
try:
# Setup argument parser
parser = ArgumentParser(description='''
Read data from the Meetup API and write it do a MongoDB database. Each run of this program
creates a new batch of data identified by a batchID. The default database is MUGS. You can change
this by using the --host parameter and specifying a different database in the mongodb URI.
If you use the --pro arguement your API key must be a meetup pro account API key. If not the api
calls to the pro interface will fail.
If you are and adminstrator on the pro account you should use the --admin flag to give you
access to the admin APIs.
''')
#
# MongoDB Args
parser.add_argument('--host',
default="mongodb://localhost:27017/MUGS",
help='URI to connect to : [default: %(default)s]')
parser.add_argument("--verbose",
dest="verbose",
action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument("-v",
"--version",
action='version',
version=__programName__ + " " + __version__)
parser.add_argument(
'--trialrun',
action="store_true",
default=False,
help='Trial run, no updates [default: %(default)s]')
parser.add_argument(
'--mugs',
nargs="+",
help='Process MUGs list list mugs by name [default: %(default)s]')
parser.add_argument(
"--pro",
default=False,
action="store_true",
help="use if you have a pro account uses pro API calls")
parser.add_argument(
"--admin",
default=False,
action="store_true",
help="Some calls are only available to admin users")
parser.add_argument(
"--database",
default="MUGS",
help="Default database name to write to [default: %(default)s]")
parser.add_argument('--phases',
nargs="+",
choices=[
"groups", "members", "attendees",
"upcomingevents", "pastevents"
],
default=["all"],
help='execution phases')
parser.add_argument(
'--loglevel',
default="INFO",
choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
help='Logging level [default: %(default)s]')
parser.add_argument('--apikey',
default=None,
help='Default API key for meetup')
parser.add_argument(
'--urlfile',
help=
"File containing a list of MUG URLs to be used to parse data [ default: %(default)s]"
)
# Process arguments
args = parser.parse_args()
apikey = ""
if args.apikey:
apikey = args.apikey
else:
apikey = get_meetup_key()
verbose = args.verbose
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string,
level=LoggingLevel(args.loglevel))
# Turn off logging for requests
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
if verbose > 0:
logging.info("Verbose mode on")
if args.urlfile:
if not os.path.isfile(args.urlfile):
print("No such file --urlfile '%s'" % args.urlfile)
sys.exit(1)
if args.mugs:
mugList = args.mugs
else:
mugList = []
if args.pro:
nopro = False
else:
nopro = True
mdb = MUGAlyserMongoDB(args.host)
audit = Audit(mdb)
batchID = audit.startBatch(
{
"args": vars(args),
"version": __programName__ + " " + __version__,
"pro_account": args.pro
},
trial=args.trialrun,
apikey=apikey)
start = datetime.utcnow()
logging.info("Started MUG processing for batch ID: %i", batchID)
logging.info("Writing to database : '%s'", mdb.database().name)
if nopro:
logging.info("Using standard API calls (no pro account API key)")
if args.urlfile:
logging.info("Reading groups from: '%s'", args.urlfile)
with open(args.urlfile) as f:
mugList = f.read().splitlines()
else:
logging.info("Using pro API calls (pro account API key)")
if nopro:
logging.info("Processing %i MUG URLS", len(mugList))
else:
mugList = list(MeetupAPI().get_pro_group_names())
writer = MeetupWriter(audit, mdb, mugList, apikey)
if "all" in args.phases:
phases = ["groups", "members", "upcomingevents", "pastevents"]
if args.admin:
phases.append("attendees")
else:
phases = args.phases
if "groups" in phases:
logging.info("processing group info for %i groups: nopro=%s",
len(mugList), nopro)
writer.processGroups(nopro)
phases.remove("groups")
if "members" in phases:
logging.info("processing members info for %i groups: nopro=%s",
len(mugList), nopro)
writer.processMembers(nopro)
phases.remove("members")
for i in mugList:
writer.capture_snapshot(i, args.admin, phases)
audit.endBatch(batchID)
end = datetime.utcnow()
elapsed = end - start
logging.info("MUG processing took %s for BatchID : %i", elapsed,
batchID)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except pymongo.errors.ServerSelectionTimeoutError, e:
print("Failed to connect to MongoDB Server (server timeout): %s" % e)
sys.exit(2)
def setUp(self):
apikey = get_meetup_key()
self._api = MeetupAPI(apikey, reshape=True)
def apikey():
get_dns_name()
with cd( "GIT/MUGAlyser/mugalyser") :
run( "python makeapikeyfile_main.py %s" % get_meetup_key())
def main():
try:
parser = ArgumentParser(
description="A direct interface to the meetup API")
parser.add_argument("--apikey",
default="",
help="API Key to use for Calls")
parser.add_argument("-i",
"--member_id",
type=int,
help="Retrieve information for a specific ID")
parser.add_argument("-g", "--mugs", nargs="+", help="Get Info for MUG")
parser.add_argument("--members",
default=False,
action="store_true",
help="list all members of a list of groups")
parser.add_argument("-l",
"--listgroups",
action="store_true",
default=False,
help="List all groups")
parser.add_argument("--groups",
default=False,
action="store_true",
help="list group data for groups")
parser.add_argument(
"--allgroups",
default=False,
action="store_true",
help="list group data for all groups for this API key")
parser.add_argument(
"--progroups",
default=False,
action="store_true",
help="list group data for all pro groups for this API key")
parser.add_argument("-u",
"--urlnames",
action="store_true",
default=False,
help="List all groups by URL name")
parser.add_argument("--pastevents",
nargs="+",
default=[],
help="Get past events for MUG")
parser.add_argument("--upcomingevents",
nargs="+",
default=[],
help="Get upcoming events for MUG")
parser.add_argument("--attendees",
nargs="+",
default=[],
help="Get attendees for list of groups")
parser.add_argument("--loop",
type=int,
default=1,
help="Loop call for --loop times")
parser.add_argument("--reshape",
default=False,
action="store_true",
help="Reshape output for BSON")
parser.add_argument("--req",
default=False,
action="store_true",
help="Report underlying request URL to meetup")
# Process arguments
args = parser.parse_args()
format_string = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=format_string, level=logging.INFO)
if args.apikey == "":
m = MeetupAPI(apikey=get_meetup_key(), reshape=args.reshape)
else:
m = MeetupAPI(apikey=args.apikey, reshape=args.reshape)
for i in range(args.loop):
if args.member_id and not (args.progroups or args.allgroups):
(url, member) = m.get_member_by_id(args.member_id)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(member)
if args.allgroups:
member_total = 0
for group_count, g in enumerate(m.get_groups(), 1):
#pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g[1]
full_group = m.get_group(group['urlname'])[1]
if full_group["organizer"]["id"] == args.member_id:
pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.progroups:
member_total = 0
group_count = 0
for url, g in m.get_groups():
print(f"URL :{url}")
pprint.pprint(g)
#print(f"{g[1]['urlname']}")
group = g
url, full_group = m.get_group(group['urlname'])
if "pro_network" in full_group and full_group[
"pro_network"]["name"] == "MongoDB":
#pprint.pprint(full_group)
print(
f"{full_group['urlname']}, {full_group['members']}"
)
member_total = member_total + full_group['members']
group_count = group_count + 1
print(f"{group_count} groups in total")
print(f"Total members: {member_total}")
if args.groups:
for i in args.mugs:
(url, mug) = m.get_group(i)
if args.req:
print("req: '{}'".format(url))
pprint.pprint(mug)
if args.members:
print("args.members: %s" % args.mugs)
# it = m.get_members( args.mugs )
count = 0
name = ""
mid = ""
for (url, j) in m.get_members(args.mugs):
if args.req:
print("req: '{}'".format(url))
count = count + 1
if "name" in j:
name = j["name"]
mid = j["id"]
else:
name = j["member_name"]
mid = j["member_id"]
print(u"{:30}, {:20}, {:20}, {:20}".format(
name, i, j["country"], mid))
print("%i total" % count)
if args.pastevents:
(url, past_events) = m.get_past_events(args.pastevents)
if args.req:
print("req: '{}'".format(url))
printCursor(past_events)
if args.upcomingevents:
(url,
upcoming_events) = m.get_upcoming_events(args.upcomingevents)
if args.req:
print("req: '{}'".format(url))
printCursor(upcoming_events)
if args.attendees:
attendees = m.get_all_attendees(args.attendees)
printCursor(attendees)
if args.listgroups:
printCursor(m.get_pro_groups())
if args.urlnames:
for i in m.get_pro_group_names():
print(i)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print_exception(exc_type, exc_value, exc_traceback)
indent = len("mug_info_main") * " "
sys.stderr.write("mug_info_main" + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help\n")
return 2
return 0
def mugalyser(argv=None): # IGNORE:C0111
'''Command line options.'''
try:
# Setup argument parser
parser = ArgumentParser(description='''
Read data from the Meetup API and write it do a MongoDB database. Each run of this program
creates a new batch of data identified by a batchID. The default database is MUGS. You can change
this by using the --host parameter and specifying a different database in the mongodb URI.
If you use the --pro arguement your API key must be a meetup pro account API key. If not the api
calls to the pro interface will fail.
If you are and adminstrator on the pro account you should use the --admin flag to give you
access to the admin APIs.
''')
#
# MongoDB Args
parser.add_argument('--host',
default="mongodb://localhost:27017/MUGS",
help='URI to connect to : [default: %(default)s]')
parser.add_argument("-v",
"--version",
action='version',
version=__programName__ + " " + __version__)
parser.add_argument(
'--mugs',
nargs="+",
default=[],
help='Process MUGs list list mugs by name [default: %(default)s]')
parser.add_argument("--collect",
choices=["pro", "nopro", "all"],
default="all",
help="Use pro API calls, no pro API calls or both")
parser.add_argument(
"--admin",
default=False,
action="store_true",
help=
"Some calls are only available to admin users, use this if you are not an admin"
)
parser.add_argument(
"--database",
default="MUGS",
help="Default database name to write to [default: %(default)s]")
parser.add_argument('--phases',
nargs="+",
choices=[
"groups", "members", "attendees",
"upcomingevents", "pastevents"
],
default=["all"],
help='execution phases')
parser.add_argument(
'--loglevel',
default="INFO",
choices=["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"],
help='Logging level [default: %(default)s]')
parser.add_argument('--apikey',
default=None,
help='Default API key for meetup')
parser.add_argument("--batchname",
default=__programName__,
help="Batch name used in creating audit batches")
parser.add_argument(
'--urlfile',
help=
"File containing a list of MUG URLs to be used to parse data [ default: %(default)s]"
)
parser.add_argument(
"--drop",
default=False,
action="store_true",
help="drop the database before writing data [default: %(default)s]"
)
parser.add_argument("--organizer_id",
type=int,
help="Organizer ID is required for non pro groups")
# Process arguments
args = parser.parse_args(argv)
apikey = ""
if args.apikey:
apikey = args.apikey
else:
apikey = get_meetup_key()
mugalyser_logger = Logger(__programName__, args.loglevel)
# mugalyser_logger.add_stream_handler( args.loglevel )
mugalyser_logger.add_file_handler("mugalyser.log", args.loglevel)
api = MeetupAPI(apikey, reshape=True)
logger = mugalyser_logger.log()
# Turn off logging for requests
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
mdb = MUGAlyserMongoDB(uri=args.host, database_name=args.database)
if args.drop:
logger.warn(f"Dropping database:'{args.database}'")
mdb.drop(args.database)
audit = Audit(mdb)
batchID = audit.start_batch({
"args": vars(args),
"version": __programName__ + " " + __version__,
"name": args.batchname
})
start = datetime.utcnow()
logger.info("Started MUG processing for batch ID: %i", batchID)
logger.info("Writing to database : '%s'", mdb.database().name)
group_dict = {}
count = 0
group_list = []
if args.mugs:
for url in args.mugs:
group_list.append(api.get_group(url))
else:
group_list = list(api.get_groups())
for url, group in group_list:
#print(f"Checking:{group['urlname']}")
urlname = group['urlname']
url, full_group = api.get_group(urlname)
if args.collect in ["pro", "all"]:
if "pro_network" in full_group and full_group["pro_network"][
"name"] == "MongoDB":
count = count + 1
logger.info(
f"{count}. Processing pro group: {group['urlname']}")
group_dict[urlname] = full_group
if args.collect in ["nopro", "all"]:
if args.organizer_id:
if full_group["organizer"]["id"] == args.organizer_id:
count = count + 1
logger.info(
f"{count}. Processing normal group: {group['urlname']}"
)
group_dict[urlname] = full_group
else:
logger.error(
"You must specify --organizer_id when collecting nopro groups"
)
sys.exit(1)
if args.urlfile:
urlfile = os.path.abspath(args.urlfile)
logger.info("Reading groups from: '%s'", urlfile)
with open(urlfile) as f:
lines = f.read().splitlines()
# string comments
regex = "^\s*#.*|^\s*$" # comments with # or blank lines
for i in lines:
clean_line = i.rstrip()
if not re.match(regex, clean_line):
group_dict[clean_line] = None
# scoop up any command line args
for i in args.mugs:
group_dict[i] = None
writer = MeetupWriter(apikey, batchID, mdb, reshape=True)
if "all" in args.phases:
phases = ["groups", "members", "upcomingevents", "pastevents"]
else:
phases = args.phases
if args.admin:
logger.info("--admin : we will collect attendee info")
phases.append("attendees")
else:
logger.info("No admin account")
logger.info(
"We will not collect attendee info: ignoring attendees")
logger.info("Processing phases: %s", phases)
if "groups" in phases:
logger.info("processing group info for %i groups: collect=%s",
len(group_dict), args.collect)
writer.write_groups(group_dict.keys())
phases.remove("groups")
if "members" in phases:
logger.info("processing members info for %i groups: collect=%s",
len(group_dict), args.collect)
writer.write_members(group_dict.keys())
phases.remove("members")
for i in group_dict.keys():
writer.capture_snapshot(i, args.admin, phases)
audit.end_batch(batchID)
end = datetime.utcnow()
elapsed = end - start
logger.info("MUG processing took %s for BatchID : %i", elapsed,
batchID)
except KeyboardInterrupt:
print("Keyboard interrupt : Exiting...")
sys.exit(2)
except pymongo.errors.ServerSelectionTimeoutError as e:
print("Failed to connect to MongoDB Server (server timeout): %s" % e)
sys.exit(2)