Esempio n. 1
0
    def setUp(self):
        super().setUp()
        self.req = ParsedRequest()
        with self.app.test_request_context(URL_PREFIX):
            self.f = PublishFilterService(datasource='publish_filters', backend=superdesk.get_backend())
            self.s = SubscribersService(datasource='subscribers', backend=superdesk.get_backend())

            self.articles = [{'_id': '1', 'urgency': 1, 'headline': 'story', 'state': 'fetched'},
                             {'_id': '2', 'headline': 'prtorque', 'state': 'fetched'},
                             {'_id': '3', 'urgency': 3, 'headline': 'creator', 'state': 'fetched'},
                             {'_id': '4', 'urgency': 4, 'state': 'fetched'},
                             {'_id': '5', 'urgency': 2, 'state': 'fetched'},
                             {'_id': '6', 'state': 'fetched'}]
            self.app.data.insert('archive', self.articles)

            self.app.data.insert('filter_conditions',
                                 [{'_id': 1,
                                   'field': 'headline',
                                   'operator': 'like',
                                   'value': 'tor',
                                   'name': 'test-1'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 2,
                                   'field': 'urgency',
                                   'operator': 'in',
                                   'value': '2',
                                   'name': 'test-2'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 3,
                                   'field': 'headline',
                                   'operator': 'endswith',
                                   'value': 'tor',
                                   'name': 'test-3'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 4,
                                   'field': 'urgency',
                                   'operator': 'in',
                                   'value': '2,3,4',
                                   'name': 'test-4'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 5,
                                   'field': 'headline',
                                   'operator': 'startswith',
                                   'value': 'sto',
                                   'name': 'test-5'}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 1,
                                   "publish_filter": [{"expression": {"fc": [1]}}],
                                   "name": "soccer-only"}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 2,
                                   "publish_filter": [{"expression": {"fc": [4, 3]}}],
                                   "name": "soccer-only2"}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 3,
                                   "publish_filter": [{"expression": {"pf": [1], "fc": [2]}}],
                                   "name": "soccer-only3"}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 4,
                                   "publish_filter": [{"expression": {"fc": [3]}}, {"expression": {"fc": [5]}}],
                                   "name": "soccer-only4"}])

            self.app.data.insert('subscribers',
                                 [{"_id": 1,
                                   "publish_filter": {"filter_id": 3, "filter_type": "blocking"},
                                   "name": "sub1"}])
            self.app.data.insert('subscribers',
                                 [{"_id": 2,
                                   "publish_filter": {"filter_id": 1, "filter_type": "blocking"},
                                   "name": "sub1"}])
Esempio n. 2
0
class PublishFilterTests(TestCase):

    def setUp(self):
        super().setUp()
        self.req = ParsedRequest()
        with self.app.test_request_context(URL_PREFIX):
            self.f = PublishFilterService(datasource='publish_filters', backend=superdesk.get_backend())
            self.s = SubscribersService(datasource='subscribers', backend=superdesk.get_backend())

            self.articles = [{'_id': '1', 'urgency': 1, 'headline': 'story', 'state': 'fetched'},
                             {'_id': '2', 'headline': 'prtorque', 'state': 'fetched'},
                             {'_id': '3', 'urgency': 3, 'headline': 'creator', 'state': 'fetched'},
                             {'_id': '4', 'urgency': 4, 'state': 'fetched'},
                             {'_id': '5', 'urgency': 2, 'state': 'fetched'},
                             {'_id': '6', 'state': 'fetched'}]
            self.app.data.insert('archive', self.articles)

            self.app.data.insert('filter_conditions',
                                 [{'_id': 1,
                                   'field': 'headline',
                                   'operator': 'like',
                                   'value': 'tor',
                                   'name': 'test-1'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 2,
                                   'field': 'urgency',
                                   'operator': 'in',
                                   'value': '2',
                                   'name': 'test-2'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 3,
                                   'field': 'headline',
                                   'operator': 'endswith',
                                   'value': 'tor',
                                   'name': 'test-3'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 4,
                                   'field': 'urgency',
                                   'operator': 'in',
                                   'value': '2,3,4',
                                   'name': 'test-4'}])
            self.app.data.insert('filter_conditions',
                                 [{'_id': 5,
                                   'field': 'headline',
                                   'operator': 'startswith',
                                   'value': 'sto',
                                   'name': 'test-5'}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 1,
                                   "publish_filter": [{"expression": {"fc": [1]}}],
                                   "name": "soccer-only"}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 2,
                                   "publish_filter": [{"expression": {"fc": [4, 3]}}],
                                   "name": "soccer-only2"}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 3,
                                   "publish_filter": [{"expression": {"pf": [1], "fc": [2]}}],
                                   "name": "soccer-only3"}])

            self.app.data.insert('publish_filters',
                                 [{"_id": 4,
                                   "publish_filter": [{"expression": {"fc": [3]}}, {"expression": {"fc": [5]}}],
                                   "name": "soccer-only4"}])

            self.app.data.insert('subscribers',
                                 [{"_id": 1,
                                   "publish_filter": {"filter_id": 3, "filter_type": "blocking"},
                                   "name": "sub1"}])
            self.app.data.insert('subscribers',
                                 [{"_id": 2,
                                   "publish_filter": {"filter_id": 1, "filter_type": "blocking"},
                                   "name": "sub1"}])

    def test_build_mongo_query_using_like_filter_single_fc(self):
        doc = {'publish_filter': [{"expression": {"fc": [1]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(3, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('3' in doc_ids)

    def test_build_mongo_query_using_like_filter_single_pf(self):
        doc = {'publish_filter': [{"expression": {"pf": [1]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(3, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('3' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_filter_condition(self):
        doc = {'publish_filter': [{"expression": {"fc": [1]}}, {"expression": {"fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(4, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('5' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_pf(self):
        doc = {'publish_filter': [{"expression": {"pf": [1]}}, {"expression": {"fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(4, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('5' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_filter_condition2(self):
        doc = {'publish_filter': [{"expression": {"fc": [3, 4]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_pf2(self):
        doc = {'publish_filter': [{"expression": {"pf": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_condition3(self):
        doc = {'publish_filter': [{"expression": {"fc": [3, 4]}}, {"expression": {"fc": [1, 2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_pf3(self):
        doc = {'publish_filter': [{"expression": {"pf": [2]}}, {"expression": {"pf": [1], "fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = self.f.build_mongo_query(doc)
            docs = superdesk.get_resource_service('archive').\
                get_from_mongo(req=self.req, lookup=query)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_single_filter_condition(self):
        doc = {'publish_filter': [{"expression": {"fc": [1]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(3, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_single_publish_filter(self):
        doc = {'publish_filter': [{"expression": {"pf": [1]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(3, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_multi_filter_condition(self):
        doc = {'publish_filter': [{"expression": {"fc": [1]}}, {"expression": {"fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(4, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('3' in doc_ids)
            self.assertTrue('5' in doc_ids)

    def test_build_mongo_query_using_like_filter_multi_publish_filter(self):
        doc = {'publish_filter': [{"expression": {"pf": [1]}}, {"expression": {"fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(4, docs.count())
            self.assertTrue('1' in doc_ids)
            self.assertTrue('2' in doc_ids)
            self.assertTrue('3' in doc_ids)
            self.assertTrue('5' in doc_ids)

    def test_build_elastic_query_using_like_filter_multi_filter_condition2(self):
        doc = {'publish_filter': [{"expression": {"fc": [3, 4]}}, {"expression": {"fc": [1, 2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_multi_publish_filter2(self):
        doc = {'publish_filter': [{"expression": {"fc": [4, 3]}},
                                  {"expression": {"pf": [1], "fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_multi_publish_filter3(self):
        doc = {'publish_filter': [{"expression": {"pf": [2]}}, {"expression": {"pf": [1], "fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_multi_publish_filter4(self):
        doc = {'publish_filter': [{"expression": {"pf": [2]}}, {"expression": {"pf": [3]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_build_elastic_query_using_like_filter_multi_publish_filter4(self):
        doc = {'publish_filter': [{"expression": {"pf": [4], "fc": [4]}}], 'name': 'pf-1'}
        with self.app.app_context():
            query = {'query': {'filtered': {'query': self.f._get_elastic_query(doc)}}}
            self.req.args = {'source': json.dumps(query)}
            docs = superdesk.get_resource_service('archive').get(req=self.req, lookup=None)
            doc_ids = [d['_id'] for d in docs]
            self.assertEqual(1, docs.count())
            self.assertTrue('3' in doc_ids)

    def test_does_match_using_like_filter_single_fc(self):
        doc = {'publish_filter': [{"expression": {"fc": [1]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertTrue(self.f.does_match(doc, self.articles[0]))
            self.assertTrue(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertFalse(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_single_pf(self):
        doc = {'publish_filter': [{"expression": {"pf": [1]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertTrue(self.f.does_match(doc, self.articles[0]))
            self.assertTrue(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertFalse(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_multi_fc(self):
        doc = {'publish_filter': [{"expression": {"fc": [1]}}, {"expression": {"fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertTrue(self.f.does_match(doc, self.articles[0]))
            self.assertTrue(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertTrue(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_multi_pf(self):
        doc = {'publish_filter': [{"expression": {"pf": [1]}}, {"expression": {"fc": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertTrue(self.f.does_match(doc, self.articles[0]))
            self.assertTrue(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertTrue(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_multi_fc2(self):
        doc = {'publish_filter': [{"expression": {"fc": [3, 4]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertFalse(self.f.does_match(doc, self.articles[0]))
            self.assertFalse(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertFalse(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_multi_pf2(self):
        doc = {'publish_filter': [{"expression": {"pf": [2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertFalse(self.f.does_match(doc, self.articles[0]))
            self.assertFalse(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertFalse(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_multi_fc3(self):
        doc = {'publish_filter': [{"expression": {"fc": [3, 4]}}, {"expression": {"fc": [1, 2]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertFalse(self.f.does_match(doc, self.articles[0]))
            self.assertFalse(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertFalse(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_does_match_using_like_filter_multi_pf3(self):
        doc = {'publish_filter': [{"expression": {"pf": [4], "fc": [4]}}], 'name': 'pf-1'}
        with self.app.app_context():
            self.assertFalse(self.f.does_match(doc, self.articles[0]))
            self.assertFalse(self.f.does_match(doc, self.articles[1]))
            self.assertTrue(self.f.does_match(doc, self.articles[2]))
            self.assertFalse(self.f.does_match(doc, self.articles[3]))
            self.assertFalse(self.f.does_match(doc, self.articles[4]))
            self.assertFalse(self.f.does_match(doc, self.articles[5]))

    def test_if_pf_is_used(self):
        with self.app.app_context():
            self.assertTrue(self.f._get_publish_filters_by_publish_filter(1).count() == 1)
            self.assertTrue(self.f._get_publish_filters_by_publish_filter(4).count() == 0)

    def test_if_fc_is_used(self):
        with self.app.app_context():
            self.assertTrue(len(self.f._get_publish_filters_by_filter_condition(1)) == 2)
            self.assertTrue(len(self.f._get_publish_filters_by_filter_condition(3)) == 2)
            self.assertTrue(len(self.f._get_publish_filters_by_filter_condition(2)) == 1)

    def test_get_subscribers_by_filter_condition(self):
        filter_condition1 = {'field': 'urgency', 'operator': 'in', 'value': '2'}
        filter_condition2 = {'field': 'urgency', 'operator': 'in', 'value': '1'}
        filter_condition3 = {'field': 'headline', 'operator': 'like', 'value': 'tor'}
        filter_condition4 = {'field': 'urgency', 'operator': 'nin', 'value': '3'}

        with self.app.app_context():
            cmd = VocabulariesPopulateCommand()
            filename = os.path.join(os.path.abspath(
                os.path.dirname("apps/prepopulate/data_initialization/vocabularies.json")), "vocabularies.json")
            cmd.run(filename)
            self.assertTrue(len(self.s._get_subscribers_by_filter_condition(filter_condition1)) == 1)
            self.assertTrue(len(self.s._get_subscribers_by_filter_condition(filter_condition2)) == 0)
            self.assertTrue(len(self.s._get_subscribers_by_filter_condition(filter_condition3)) == 2)
            self.assertTrue(len(self.s._get_subscribers_by_filter_condition(filter_condition4)) == 1)