Esempio n. 1
0
    def get_active_users(self, urls, filename=None):
        '''
        We define an active user as somebody who has rsvp'd to at least one event in the last six months.
        '''
        agg = Agg(self._mdb.attendeesCollection())

        agg.addMatch({
            "batchID": self._batchID,
            "info.event.group.urlname": {
                "$in": urls
            },
            "info.attendee.rsvp.response": "yes"
        })

        if self._start_date or self._end_date:
            agg.addRangeMatch("info.event.time", self._start_date,
                              self._end_date)
        else:
            agg.addRangeMatch("info.event.time",
                              datetime.utcnow() + relativedelta(months=-6))

        #     agg.addProject( { "_id" : 0,
        #                       "name" : "$info.attendee.member.name",
        #                       "urlname" : "$info.event.group.urlname",
        #                       "event_name" : "$info.event.name" })

        agg.addProject({
            "_id": 0,
            "name": "$info.attendee.member.name",
            "event": "$info.event.name",
            "date": "$info.event.time"
        })
        print(agg)
        agg.addGroup({"_id": "$name", "count": {"$sum": 1}})

        if self._sorter:
            agg.addSort(self._sorter)

        if self._view:
            agg.create_view(self._mdb.database(), "active_users_view")

        formatter = CursorFormatter(agg, filename, self._format)
        formatter.output(fieldNames=["_id", "count"], limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 2
0
    def get_events(self, urls, when="past", filename=None):
        '''
        Get events when=past means past events. when="upcoming" means future events.
        '''

        agg = None

        if when == "past":
            agg = Agg(self._mdb.pastEventsCollection())

        elif when == "upcoming":
            agg = Agg(self._mdb.upcomingEventsCollection())

        agg.addMatch({
            "batchID": self._batchID,
            "event.status": when,
            "event.group.urlname": {
                "$in": urls
            }
        })

        if self._start_date or self._end_date:
            agg.addRangeMatch("event.time", self._start_date, self._end_date)

        agg.addProject({
            "_id": 0,
            "group": u"$event.group.urlname",
            "name": u"$event.name",
            "country": "$event.venue.country",
            "rsvp_count": "$event.yes_rsvp_count",
            "date": "$event.time"
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if self._view:
            agg.create_view(self._mdb.database(), "events_view")

        formatter = CursorFormatter(agg, filename, self._format)
        formatter.output(fieldNames=["group", "name", "rsvp_count", "date"],
                         datemap=["date"],
                         limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 3
0
    def get_RSVP_history(self, urls, filename=None):
        '''
        Get the list of past events for groups (urls) and report on the date of the event
        and the number of RSVPs.
        '''

        agg = Agg(self._mdb.pastEventsCollection())

        agg.addMatch({"event.group.urlname": {"$in": urls}})

        if self._start_date or self._end_date:
            agg.addRangeMatch("event.time", self._start_date, self._end_date)

        agg.addProject({
            "timestamp": "$event.time",
            "event": "$event.name",
            "country": "$event.venue.country",
            "rsvp_count": "$event.yes_rsvp_count"
        })

        agg.addMatch({"timestamp": {"$type": "date"}})
        agg.addGroup({
            "_id": "$timestamp",
            #"event" : { "$addToSet" : { "event" : "$event", "country" : "$country" }},
            "rsvp_count": {
                "$sum": "$rsvp_count"
            }
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if filename:
            self._filename = filename

        if self._view:
            agg.create_view(self._mdb.database(), "rsvps_view")

        formatter = CursorFormatter(agg.aggregate(), self._filename,
                                    self._format)
        filename = formatter.output(fieldNames=["_id", "rsvp_count"],
                                    datemap=["_id"],
                                    limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 4
0
    def get_organisers(self, urls, filename=None):
        '''
        Get all the members of all the groups (urls). Range is join_time.
        '''

        agg = Agg(self._mdb.proGroupsCollection())

        agg.addMatch({"batchID": self._batchID})
        agg.addMatch({"group.urlname": {"$in": urls}})

        if self._start_date or self._end_date:
            agg.addRangeMatch("group.created", self._start_date,
                              self._end_date)

        agg.addUnwind("$group.organizers")

        agg.addGroup({
            "_id": "$group.organizers",
            "groups": {
                "$addToSet": "$group.urlname"
            }
        })

        agg.addProject({
            "_id": 0,
            "name": "$_id.name",
            "role": "$_id.permission",
            "groups": "$groups"
            ""
        })

        print(agg)
        if self._sorter:
            agg.addSort(self._sorter)

        if self._view:
            agg.create_view(self._mdb.database(), "new_members_view")

        formatter = CursorFormatter(agg, filename, self._format)
        formatter.output(fieldNames=["name", "role", "groups"],
                         limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 5
0
    def get_events(self, urls, filename=None):

        agg = Agg(self._mdb.pastEventsCollection())

        agg.addMatch({
            "batchID": self._batchID,
            "event.status": "past",
            "event.group.urlname": {
                "$in": urls
            }
        })

        if self._start_date or self._end_date:
            agg.addRangeMatch("event.time", self._start_date, self._end_date)

        agg.addProject({
            "_id": 0,
            "group": u"$event.group.urlname",
            "name": u"$event.name",
            "rsvp_count": "$event.yes_rsvp_count",
            "date": "$event.time"
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if filename:
            self._filename = filename

        if self._view:
            agg.create_view(self._mdb.database(), "events_view")

        formatter = CursorFormatter(agg, self._filename, self._format)
        filename = formatter.output(
            fieldNames=["group", "name", "rsvp_count", "date"],
            datemap=["date"],
            limit=self._limit)

        if self._filename != "-":
            self._files.append(self._filename)
Esempio n. 6
0
    def get_job_groups( self, filename=None ):
        '''
        Find all the jobs in SFCD
        '''
    
        agg = Agg( self._collection )

        # [ { "$group" : { "_id" : "$Job Function", "count" : { "$sum" : 1  }}} ]
        agg.addGroup( { "_id"   : "$Job Function",
                        "count" : { "$sum" : 1 }})
        
        if self._sorter:
            agg.addSort( self._sorter )

        print( agg )
        for i in agg() :
            print( i ) 
        formatter = CursorFormatter( agg, filename, self._format )
        formatter.output( fieldNames= [ "_id", "count" ], limit=self._limit )
        
        if filename != "-":
            self._files.append( filename )
Esempio n. 7
0
    def get_rsvps( self, urls, filename=None):   
        '''
        Lookup RSVPs by user. So for each user collect how many events they RSVPed to.
        '''
        agg = Agg( self._mdb.attendeesCollection())
        
        agg.addMatch({ "batchID"            : self._batchID,
                       "info.event.group.urlname" : { "$in" : urls }} )
        
        if self._start_date or self._end_date :
            agg.addRangeMatch( "info.event.time", self._start_date, self._end_date )
        
        agg.addProject( { "_id"        : 0,
                          "attendee"   : "$info.attendee.member.name", 
                          "group"      : "$info.event.group.urlname",
                          "date"       : "$info.event.time",
                          "event_name" : "$info.event.name" })
        
#         agg.addGroup( { "_id" : {  "attendee": "$attendee", "group": "$group" },
#                         "event_count" : { "$sum" : 1 }})
#                         
#         agg.addProject( { "_id" : 0,
#                           "attendee" : "$_id.attendee",
#                           "group" : "$_id.group",
#                           "date"  : "$event.time",
#                           "event_count" : 1 } )
        
        if self._sorter :
            agg.addSort( self._sorter)

        if self._view :
            agg.create_view( self._mdb.database(), "rsvps_view" )
            
        formatter = CursorFormatter( agg, filename, self._format )
        formatter.output( fieldNames= [ "attendee", "group", "date", "event_name" ], datemap=[ "date" ],  limit=self._limit )

        if filename != "-":
            self._files.append( filename )
Esempio n. 8
0
    def getMembers(self, urls, filename=None):
        '''
        Get a count of the members for each group in "urls"
        Range doens't make sense here so its not used. If supplied it is ignored.
        '''

        agg = Agg(self._mdb.groupsCollection())

        agg.addMatch({
            "batchID": {
                "$in": [self._batchID]
            },
            "group.urlname": {
                "$in": urls
            }
        })

        agg.addProject({
            "_id": 0,
            "urlname": "$group.urlname",
            "country": "$group.country",
            "batchID": 1,
            "member_count": "$group.member_count"
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if filename:
            self._filename = filename

        if self._view:
            agg.create_view(self._mdb.database(), "members_view")

        formatter = CursorFormatter(agg.aggregate(), self._filename,
                                    self._format)
        formatter.output(
            fieldNames=["urlname", "country", "batchID", "member_count"])
Esempio n. 9
0
    def get_rsvp_by_event(self, urls, filename="rsvp_events"):

        agg = Agg(self._mdb.pastEventsCollection())

        agg.addMatch({
            "batchID": self._batchID,
            "event.group.urlname": {
                "$in": urls
            }
        })

        if self._start_date or self._end_date:
            agg.addRangeMatch("event.time", self._start_date, self._end_date)

        agg.addGroup({
            "_id": "$event.group.urlname",
            "rsvp_count": {
                "$sum": "$event.yes_rsvp_count"
            }
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if filename:
            self._filename = filename

        if self._view:
            agg.create_view(self._mdb.database(), "rsvps_by_event_view")

        formatter = CursorFormatter(agg.aggregate(), self._filename,
                                    self._format)
        filename = formatter.output(fieldNames=["_id", "rsvp_count"],
                                    limit=self._limit)

        if self._filename != "-":
            self._files.append(self._filename)
Esempio n. 10
0
    def get_new_members(self, urls, filename=None):
        '''
        Get all the members of all the groups (urls). Range is join_time.
        '''

        agg = Agg(self._mdb.proMembersCollection())

        agg.addMatch({"batchID": self._batchID})

        if self._start_date or self._end_date:
            agg.addRangeMatch("member.join_time", self._start_date,
                              self._end_date)

        agg.addUnwind("$member.chapters")
        agg.addMatch({"member.chapters.urlname": {"$in": urls}})

        agg.addProject({
            "_id": 0,
            "group": "$member.chapters.urlname",
            "name": "$member.member_name",
            "join_date": "$member.join_time"
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if self._view:
            agg.create_view(self._mdb.database(), "new_members_view")

        formatter = CursorFormatter(agg, filename, self._format)
        formatter.output(fieldNames=["group", "name", "join_date"],
                         datemap=['join_date'],
                         limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 11
0
    def get_total_events(self, urls, when="past", filename=None):

        agg = None

        if when == "past":
            agg = Agg(self._mdb.pastEventsCollection())

        elif when == "upcoming":
            agg = Agg(self._mdb.upcomingEventsCollection())

        if self._start_date or self._end_date:
            agg.addRangeMatch("event.time", self._start_date, self._end_date)

        agg.addMatch({
            "batchID": self._batchID,
            "event.status": when,
            "event.group.urlname": {
                "$in": urls
            }
        })

        agg.addProject({
            "_id": 0,
            "group": "$event.group.urlname",
            "rsvp": "$event.group.yes_rsvp_count",
            "month": {
                "$month": "$event.time"
            },
            "year": {
                "$year": "$event.time"
            }
        })

        agg.addGroup({
            "_id": {
                "month": "$month",
                "year": "$year"
            },
            "count": {
                "$sum": 1
            }
        })

        agg.addProject({
            "month": "$_id.month",
            "year": "$_id.year",
            "count": "$count"
        })

        if self._sorter:
            agg.addSort(self._sorter)

        if self._view:
            agg.create_view(self._mdb.database(), "total_events")

        if filename:
            self._filename = filename

        formatter = CursorFormatter(agg, self._filename, self._format)
        filename = formatter.output(fieldNames=["month", "year", "count"],
                                    limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 12
0
    def get_groups(self, urls, filename=None):
        '''
        Get all the groups listed by urls and their start dates
        '''

        if self._pro_account:
            agg = Agg(self._mdb.proGroupsCollection())
            agg.addMatch({
                "batchID": self._batchID,
                "group.urlname": {
                    "$in": urls
                }
            })
            if self._start_date or self._end_date:
                agg.addRangeMatch("group.founded_date", self._start_date,
                                  self._end_date)
            agg.addProject({
                "_id": 0,
                "urlname": "$group.urlname",
                "url": {
                    "$concat": ["http://www.meetup.com/", "$group.urlname"]
                },
                "members": "$group.member_count",
                "city": "$group.city",
                "country": "$group.country",
                "last_event": "$group.last_event",
                "founded": "$group.founded_date"
            })
            # print( "Using pro search : %i" % self._batchID )
            # agg.echo()
        else:
            agg = Agg(self._mdb.groupsCollection())
            agg.addMatch({
                "batchID": self._batchID,
                "group.urlname": {
                    "$in": urls
                }
            })
            if self._start_date or self._end_date:
                agg.addRangeMatch("group.created", self._start_date,
                                  self._end_date)
            agg.addProject({
                "_id": 0,
                "urlname": "$group.urlname",
                "url": {
                    "$concat": ["http://www.meetup.com/", "$group.urlname"]
                },
                "members": "$group.members",
                "city": "$group.city",
                "country": "$group.country",
                "last_event": "not present",
                "founded": "$group.created"
            })
            print("Using nopro search")
        if self._sorter:
            agg.addSort(self._sorter)

        if self._view:
            agg.create_view(self._mdb.database(), "groups_view")

        formatter = CursorFormatter(agg, filename, self._format)
        formatter.output(fieldNames=[
            "urlname", "url", "last_event", "members", "city", "country",
            "founded"
        ],
                         datemap=["last_event", "founded"],
                         limit=self._limit)

        if filename != "-":
            self._files.append(filename)

        return filename
Esempio n. 13
0
    def get_member_history(self, urls, filename=None):
        '''
        Got into every batch and see what the member count was for each group (URL) this uses all 
        the batches to get a history of a group.
        Range is used to select batches in this case via the "timestamp" field.
        '''
        audit = Audit(self._mdb)

        validBatches = list(audit.getCurrentValidBatchIDs())

        agg = Agg(self._mdb.groupsCollection())

        if self._start_date or self._end_date:
            agg.addRangeMatch("timestamp", self._start_date, self._end_date)

        agg.addMatch({
            "batchID": {
                "$in": validBatches
            },
            "group.urlname": {
                "$in": urls
            }
        })

        agg.addProject({
            "_id":
            0,
            "timestamp":
            1,
            "batchID":
            1,
            "urlname":
            "$group.urlname",
            #"count" : "$group.members" } )
            "count":
            Agg.ifNull("$group.member_count", "$group.members")
        })

        agg.addGroup({
            "_id": {
                "ts": "$timestamp",
                "batchID": "$batchID"
            },
            "groups": {
                "$addToSet": "$urlname"
            },
            "count": {
                "$sum": "$count"
            }
        })

        #CursorFormatter( agg.aggregate()).output()

        if self._sorter:
            agg.addSort(self._sorter)

        #CursorFormatter( agg.aggregate()).output()
        if filename:
            self._filename = filename

        if self._view:
            agg.create_view(self._mdb.database(), "groups_view")

        formatter = CursorFormatter(agg.aggregate(), self._filename,
                                    self._format)
        formatter.output(fieldNames=["_id", "groups", "count"],
                         datemap=["_id.ts"],
                         limit=self._limit)

        if filename != "-":
            self._files.append(filename)
Esempio n. 14
0
class Members(MUGData):
    '''
    classdocs
    '''
    def __init__(self, mdb, collection_name=None):
        '''
        Constructor
        '''

        if collection_name is None:
            collection_name = "members"

        super(Members, self).__init__(mdb, collection_name)
        self._membersAgg = Agg(self._collection)
        self._membersAgg.addMatch({"member.name": {"$exists": 1}})
        self._membersAgg.addProject({"_id": 0, "name": "$member.name"})
        self._membersAgg.addGroup({"_id": "$name", "occurences": {"$sum": 1}})
        self._membersAgg.addSort(
            Sorter(occurences=pymongo.DESCENDING))  # largest first
        self._memberCount = 0

    def get_all_members(self, query=None):
        '''
        Query meetup API for multiple groups.
        '''
        return self.find(query)

    def get_by_name(self, name):
        member = self.find_one({"member.member_name": name})

        if member is None:
            return None
        else:
            return member["member"]

    def get_by_ID(self, member_id):
        val = self.find_one({"member.member_id": member_id})

        if val is None:
            return val
        else:
            return val["member"]

    def get_by_join_date(self, start, end):

        return self.find({"member.join_time": {"$gte": start, "$lte": end}})

    def joined_by_year(self):

        agg_pipe = Agg(self._collection)
        agg_pipe.addMatch({"batchID": self._audit.get_last_valid_batch_id()})
        agg_pipe.addProject({
            "_id": 0,
            "member_id": "$member.member_id",
            "member_name": "$member.member_name",
            "year": {
                "$year": "$member.join_time"
            },
        })
        agg_pipe.addGroup({"_id": "$year", "total_registered": {"$sum": 1}})

        return agg_pipe.aggregate()

    def distinct_members(self):
        return self._collection.distinct("member.name")

    def get_members(self):
        return self._membersAgg.aggregate()

    def summary(self, doc):
        return "name: %s, id: %s, country: %s" % (doc["member"]["member_name"],
                                                  doc["member"]["member_id"],
                                                  doc["member"]["country"])

    def one_line(self, doc):
        return "name : %s, id: %s" % (doc["member"]["member_name"],
                                      doc["member"]["member_id"])