def get_rsvp_by_event(self, urls, filename="rsvp_events"): agg = Agg(self._mdb.pastEventsCollection()) agg.addMatch({ "batchID": self._batchID, "event.group.urlname": { "$in": urls } }) if self._start_date or self._end_date: agg.addRangeMatch("event.time", self._start_date, self._end_date) agg.addGroup({ "_id": "$event.group.urlname", "rsvp_count": { "$sum": "$event.yes_rsvp_count" } }) if self._sorter: agg.addSort(self._sorter) if self._view: agg.create_view(self._mdb.database(), "rsvps_by_event_view") formatter = CursorFormatter(agg.aggregate(), filename, self._format) formatter.output(fieldNames=["_id", "rsvp_count"], limit=self._limit) if filename != "-": self._files.append(filename)
def get_group_totals(self, urls, filename=None): ''' get the total number of RSVPs by group. ''' agg = Agg(self._mdb.pastEventsCollection()) agg.addMatch({ "batchID": self._batchID, "event.status": "past", "event.group.urlname": { "$in": urls } }) if self._start_date or self._end_date: agg.addRangeMatch("groups.founded_date", self._start_date, self._end_date) agg.addGroup({ "_id": { "urlname": "$event.group.urlname", "year": { "$year": "$event.time" } }, "event_count": { "$sum": 1 }, "rsvp_count": { "$sum": "$event.yes_rsvp_count" } }) agg.addProject({ "_id": 0, "group": "$_id.urlname", "year": "$_id.year", "event_count": 1, "rsvp_count": 1 }) if self._sorter: agg.addSort(self._sorter) if filename: self._filename = filename if self._view: agg.create_view(self._mdb.database(), "group_totals_view") formatter = CursorFormatter(agg, self._filename, self._format) filename = formatter.output( fieldNames=["year", "group", "event_count", "rsvp_count"], limit=self._limit) if self._filename != "-": self._files.append(self._filename)
class Pro_Members(Members): def __init__(self, mdb): ''' Constructor ''' super(Pro_Members, self).__init__(mdb, "pro_members") self._membersAgg = Agg(self._collection) self._membersAgg.addMatch({"member.member_name": {"$exists": 1}}) self._membersAgg.addProject({"_id": 0, "name": "$member.member_name"}) self._membersAgg.addGroup({"_id": "$name", "occurences": {"$sum": 1}}) self._membersAgg.addSort( Sorter(occurences=pymongo.DESCENDING)) # largest first self._memberCount = 0 def get_group_members(self, url_name, q=None): ''' returns a MongoDB cursor. ''' query = {"member.chapters": {"$elemMatch": {"urlname": url_name}}} if q: query.update(q) return self.find(query) def count_members(self, groups): total = 0 for i in groups: count = self.get_group_members(i).count() total = total + count return count def get_many_group_members(self, groups, query=None): ''' returns a generator ''' return itertools.chain( *[self.get_group_members(i, query) for i in groups]) def joined_by_year(self): agg_pipe = Agg(self._collection) agg_pipe.addMatch({"batchID": self._audit.get_last_valid_batch_id()}) agg_pipe.addProject({ "_id": 0, "member_id": "$member.member_id", "member_name": "$member.member_name", "year": { "$year": "$member.join_time" }, }) agg_pipe.addGroup({"_id": "$year", "total_registered": {"$sum": 1}}) return agg_pipe.aggregate() def distinct_members(self): return self._collection.distinct("member.member_name")
def get_rsvps(self, urls, filename=None): ''' Lookup RSVPs by user. So for each user collect how many events they RSVPed to. ''' agg = Agg(self._mdb.attendeesCollection()) agg.addMatch({ "batchID": self._batchID, "info.event.group.urlname": { "$in": urls } }) if self._start_date or self._end_date: agg.addRangeMatch("info.event_time", self._start_date, self._end_date) agg.addProject({ "_id": 0, "attendee": "$info.attendee.member.name", "group": "$info.event.group.urlname", "event_name": "$info.event.name" }) agg.addGroup({ "_id": { "attendee": "$attendee", "group": "$group" }, "event_count": { "$sum": 1 } }) agg.addProject({ "_id": 0, "attendee": "$_id.attendee", "group": "$_id.group", "event_count": 1 }) if self._sorter: agg.addSort(self._sorter) if filename: self._filename = filename if self._view: agg.create_view(self._mdb.database(), "rsvps_view") formatter = CursorFormatter(agg, self._filename, self._format) filename = formatter.output( fieldNames=["attendee", "group", "event_count"], limit=self._limit) if self._filename != "-": self._files.append(self._filename)
def joined_by_year(self): agg_pipe = Agg(self._collection) agg_pipe.addMatch({"batchID": self._audit.get_last_valid_batch_id()}) agg_pipe.addProject({ "_id": 0, "member_id": "$member.member_id", "member_name": "$member.member_name", "year": { "$year": "$member.join_time" }, }) agg_pipe.addGroup({"_id": "$year", "total_registered": {"$sum": 1}}) return agg_pipe.aggregate()
def get_active_users(self, urls, filename=None): ''' We define an active user as somebody who has rsvp'd to at least one event in the last six months. ''' agg = Agg(self._mdb.attendeesCollection()) agg.addMatch({ "batchID": self._batchID, "info.event.group.urlname": { "$in": urls }, "info.attendee.rsvp.response": "yes" }) if self._start_date or self._end_date: agg.addRangeMatch("info.event_time", self._start_date, self._end_date) # agg.addProject( { "_id" : 0, # "name" : "$info.attendee.member.name", # "urlname" : "$info.event.group.urlname", # "event_name" : "$info.event.name" }) agg.addGroup({ "_id": "$info.attendee.member.name", "count": { "$sum": 1 }, "groups": { "$addToSet": "$info.event.group.urlname" } }) if self._sorter: agg.addSort(self._sorter) if filename: self._filename = filename if self._view: agg.create_view(self._mdb.database(), "active_users_view") formatter = CursorFormatter(agg, self._filename, self._format) filename = formatter.output(fieldNames=["_id", "count", "groups"], limit=self._limit) if self._filename != "-": self._files.append(self._filename)
def get_total_events( self, urls, when="past", filename=None ) : agg = None if when == "past" : agg = Agg( self._mdb.pastEventsCollection()) elif when == "upcoming" : agg = Agg( self._mdb.upcomingEventsCollection()) if self._start_date or self._end_date : agg.addRangeMatch( "event.time", self._start_date, self._end_date ) agg.addMatch({ "batchID" : self._batchID, "event.status" : when, "event.group.urlname" : { "$in" : urls }} ) agg.addProject( { "_id" : 0, "group" : "$event.group.urlname", "rsvp" : "$event.group.yes_rsvp_count", "month" : { "$month" : "$event.time" }, "year" : { "$year" : "$event.time" }}) agg.addGroup( { "_id" : { "month" : "$month", "year" : "$year" }, "count" : { "$sum": 1 }}) agg.addProject( { "month" : "$_id.month", "year" : "$_id.year", "count" : "$count"} ) if self._sorter: agg.addSort( self._sorter) if self._view : agg.create_view( self._mdb.database(), "total_events" ) if filename : self._filename = filename formatter = CursorFormatter( agg, self._filename, self._format ) filename = formatter.output( fieldNames= [ "month", "year", "count" ], limit=self._limit) if filename != "-": self._files.append( filename )
def get_RSVP_history(self, urls, filename=None): ''' Get the list of past events for groups (urls) and report on the date of the event and the number of RSVPs. ''' agg = Agg(self._mdb.pastEventsCollection()) agg.addMatch({"event.group.urlname": {"$in": urls}}) if self._start_date or self._end_date: agg.addRangeMatch("event.time", self._start_date, self._end_date) agg.addProject({ "timestamp": "$event.time", "event": "$event.name", "country": "$event.venue.country", "rsvp_count": "$event.yes_rsvp_count" }) agg.addMatch({"timestamp": {"$type": "date"}}) agg.addGroup({ "_id": "$timestamp", #"event" : { "$addToSet" : { "event" : "$event", "country" : "$country" }}, "rsvp_count": { "$sum": "$rsvp_count" } }) if self._sorter: agg.addSort(self._sorter) if filename: self._filename = filename if self._view: agg.create_view(self._mdb.database(), "rsvps_view") formatter = CursorFormatter(agg.aggregate(), self._filename, self._format) filename = formatter.output(fieldNames=["_id", "rsvp_count"], datemap=["_id"], limit=self._limit) if filename != "-": self._files.append(filename)
def get_organisers(self, urls, filename=None): ''' Get all the members of all the groups (urls). Range is join_time. ''' agg = Agg(self._mdb.proGroupsCollection()) agg.addMatch({"batchID": self._batchID}) agg.addMatch({"group.urlname": {"$in": urls}}) if self._start_date or self._end_date: agg.addRangeMatch("group.created", self._start_date, self._end_date) agg.addUnwind("$group.organizers") agg.addGroup({ "_id": "$group.organizers", "groups": { "$addToSet": "$group.urlname" } }) agg.addProject({ "_id": 0, "name": "$_id.name", "role": "$_id.permission", "groups": "$groups" "" }) print(agg) if self._sorter: agg.addSort(self._sorter) if self._view: agg.create_view(self._mdb.database(), "new_members_view") formatter = CursorFormatter(agg, filename, self._format) formatter.output(fieldNames=["name", "role", "groups"], limit=self._limit) if filename != "-": self._files.append(filename)
def get_job_groups( self, filename=None ): ''' Find all the jobs in SFCD ''' agg = Agg( self._collection ) # [ { "$group" : { "_id" : "$Job Function", "count" : { "$sum" : 1 }}} ] agg.addGroup( { "_id" : "$Job Function", "count" : { "$sum" : 1 }}) if self._sorter: agg.addSort( self._sorter ) print( agg ) for i in agg() : print( i ) formatter = CursorFormatter( agg, filename, self._format ) formatter.output( fieldNames= [ "_id", "count" ], limit=self._limit ) if filename != "-": self._files.append( filename )
def get_member_history(self, urls, filename=None): ''' Got into every batch and see what the member count was for each group (URL) this uses all the batches to get a history of a group. Range is used to select batches in this case via the "timestamp" field. ''' audit = Audit(self._mdb) validBatches = list(audit.getCurrentValidBatchIDs()) agg = Agg(self._mdb.groupsCollection()) if self._start_date or self._end_date: agg.addRangeMatch("timestamp", self._start_date, self._end_date) agg.addMatch({ "batchID": { "$in": validBatches }, "group.urlname": { "$in": urls } }) agg.addProject({ "_id": 0, "timestamp": 1, "batchID": 1, "urlname": "$group.urlname", #"count" : "$group.members" } ) "count": Agg.ifNull("$group.member_count", "$group.members") }) agg.addGroup({ "_id": { "ts": "$timestamp", "batchID": "$batchID" }, "groups": { "$addToSet": "$urlname" }, "count": { "$sum": "$count" } }) #CursorFormatter( agg.aggregate()).output() if self._sorter: agg.addSort(self._sorter) #CursorFormatter( agg.aggregate()).output() if filename: self._filename = filename if self._view: agg.create_view(self._mdb.database(), "groups_view") formatter = CursorFormatter(agg.aggregate(), self._filename, self._format) formatter.output(fieldNames=["_id", "groups", "count"], datemap=["_id.ts"], limit=self._limit) if filename != "-": self._files.append(filename)
class Members(MUGData): ''' classdocs ''' def __init__(self, mdb, collection_name=None): ''' Constructor ''' if collection_name is None: collection_name = "members" super(Members, self).__init__(mdb, collection_name) self._membersAgg = Agg(self._collection) self._membersAgg.addMatch({"member.name": {"$exists": 1}}) self._membersAgg.addProject({"_id": 0, "name": "$member.name"}) self._membersAgg.addGroup({"_id": "$name", "occurences": {"$sum": 1}}) self._membersAgg.addSort( Sorter(occurences=pymongo.DESCENDING)) # largest first self._memberCount = 0 def get_all_members(self, query=None): ''' Query meetup API for multiple groups. ''' return self.find(query) def get_by_name(self, name): member = self.find_one({"member.member_name": name}) if member is None: return None else: return member["member"] def get_by_ID(self, member_id): val = self.find_one({"member.member_id": member_id}) if val is None: return val else: return val["member"] def get_by_join_date(self, start, end): return self.find({"member.join_time": {"$gte": start, "$lte": end}}) def joined_by_year(self): agg_pipe = Agg(self._collection) agg_pipe.addMatch({"batchID": self._audit.get_last_valid_batch_id()}) agg_pipe.addProject({ "_id": 0, "member_id": "$member.member_id", "member_name": "$member.member_name", "year": { "$year": "$member.join_time" }, }) agg_pipe.addGroup({"_id": "$year", "total_registered": {"$sum": 1}}) return agg_pipe.aggregate() def distinct_members(self): return self._collection.distinct("member.name") def get_members(self): return self._membersAgg.aggregate() def summary(self, doc): return "name: %s, id: %s, country: %s" % (doc["member"]["member_name"], doc["member"]["member_id"], doc["member"]["country"]) def one_line(self, doc): return "name : %s, id: %s" % (doc["member"]["member_name"], doc["member"]["member_id"])