def get_member_history(self, urls, filename=None): ''' Got into every batch and see what the member count was for each group (URL) this uses all the batches to get a history of a group. Range is used to select batches in this case via the "timestamp" field. ''' audit = Audit(self._mdb) validBatches = list(audit.get_valid_batch_ids()) agg = Agg(self._mdb.groupsCollection()) if self._start_date or self._end_date: agg.addRangeMatch("timestamp", self._start_date, self._end_date) agg.addMatch({ "batchID": { "$in": validBatches }, "group.urlname": { "$in": urls } }) agg.addProject({ "_id": 0, "timestamp": 1, "batchID": 1, "urlname": "$group.urlname", "count": "$group.members" }) # agg.addGroup( { "_id" : { "ts": "$timestamp", "batchID" : "$batchID" }, # "groups" : { "$addToSet" : "$urlname" }, # "count" : { "$sum" : "$count"}}) # # CursorFormatter( agg.aggregate()).output() if self._sorter: agg.addSort(self._sorter) if self._view: agg.create_view(self._mdb.database(), "groups_view") formatter = CursorFormatter(agg.aggregate(), filename, self._format) formatter.output( fieldNames=["timestamp", "batchID", "urlname", "count"], datemap=["timestamp"], limit=self._limit) if filename != "-": self._files.append(filename)
class Test_audit(unittest.TestCase): def setUp(self): self._mdb = MUGAlyserMongoDB( uri="mongodb://localhost/TEST_AUDIT" ) self._audit = Audit( self._mdb ) def tearDown(self): self._mdb.client().drop_database( "TEST_AUDIT" ) #@unittest.skip def test_get_current_batch_id(self): self.assertFalse( self._audit.in_batch()) batch_id = self._audit.start_batch( doc = { "test" : "doc"}) self.assertTrue( self._audit.in_batch()) self._audit.end_batch( batch_id ) self.assertTrue( self._audit.get_batch( batch_id )) self.assertFalse( self._audit.in_batch()) self.assertEqual( batch_id, self._audit.get_last_valid_batch_id()) def test_get_valid_batches(self): id1 = self._audit.start_batch( doc = { "test" : "doc"}) id2 = self._audit.start_batch( doc = { "test" : "doc"}) self.assertTrue( self._audit.in_batch()) self._audit.end_batch( id2 ) self.assertTrue( self._audit.in_batch()) self._audit.end_batch( id1 ) batch = self._audit.get_batch_end( id1 ) self.assertGreaterEqual( batch[ 'end'], parse( "1-Jun-2017", ) ) self.assertFalse( self._audit.in_batch()) idlist = list( self._audit.get_valid_batch_ids()) self.assertTrue( id1 in idlist ) self.assertTrue( id2 in idlist ) def test_get_last_batch_id(self): id1 = self._audit.start_batch( doc = { "test" : "doc"}) id2 = self._audit.start_batch( doc = { "test" : "doc"}) self.assertEqual( 101, self._audit.get_last_batch_id()) self._audit.end_batch( id2 ) self.assertEqual( 101, self._audit.get_last_batch_id()) self._audit.end_batch( id1 ) id1 = self._audit.start_batch( doc = { "test" : "doc"}) self.assertEqual( 102, self._audit.get_last_batch_id()) self._audit.end_batch( id1 ) def test_pro_batch_id(self):