def test_mongo_using_embargo_filter_with_eq_comp(self): f = FilterCondition('embargo', 'eq', 'true') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count())
def test_mongo_using_desk_filter_in_list(self): f = FilterCondition('desk', 'in', '1,2') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(2, docs.count())
def test_mongo_using_in_filter(self): f = FilterCondition("urgency", "in", "3,4") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(2, docs.count()) self.assertEqual("3", docs[0]["_id"]) self.assertEqual("4", docs[1]["_id"])
def test_mongo_using_notlike_filter(self): f = FilterCondition("headline", "notlike", "Que") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(11, docs.count()) doc_ids = [d["_id"] for d in docs] self.assertTrue("2" not in doc_ids)
def test_does_match_with_in_filter(self): f = FilterCondition('urgency', 'in', '3,4') self.assertFalse(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertTrue(f.does_match(self.articles[2])) self.assertTrue(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_mongo_using_endswith_filter(self): f = FilterCondition('headline', 'endswith', 'Que') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('2', docs[0]['_id'])
def test_elastic_using_endswith_filter(self): f = FilterCondition("headline", "endswith", "Que") query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, "keyword") docs = get_resource_service("archive").get(req=self.req, lookup=None) self.assertEqual(1, docs.count()) self.assertEqual("2", docs[0]["_id"])
def test_mongo_using_priority_compare_lte_filter(self): f = FilterCondition('priority', 'lte', '3') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('5', docs[0]['_id'])
def test_mongo_using_subject_filter_complete_string_eq(self): f = FilterCondition('subject', 'eq', '05005003') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('8', docs[0]['_id'])
def test_mongo_using_ingest_provider_filter_eq(self): f = FilterCondition('ingest_provider', 'eq', '1') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('4', docs[0]['_id'])
def test_does_match_with_eq_string(self): f = FilterCondition('headline', 'eq', 'Story') self.assertTrue(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_does_match_with_in_filter(self): f = FilterCondition("urgency", "in", "3,4") self.assertFalse(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertTrue(f.does_match(self.articles[2])) self.assertTrue(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_mongo_using_genre_filter_complete_string(self): f = FilterCondition('genre', 'in', 'Sidebar') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('7', docs[0]['_id'])
def test_mongo_using_endswith_filter(self): f = FilterCondition("headline", "endswith", "Que") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("2", docs[0]["_id"])
def test_does_match_with_like_partial(self): f = FilterCondition("headline", "like", "tor") self.assertTrue(f.does_match(self.articles[0])) self.assertTrue(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_mongo_using_subject_filter_complete_string_eq(self): f = FilterCondition("subject", "eq", "05005003") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("8", docs[0]["_id"])
def test_mongo_using_like_filter_complete_string_eq(self): f = FilterCondition("headline", "like", "story") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("1", docs[0]["_id"])
def test_mongo_using_category_filter_complete_string(self): f = FilterCondition("anpa_category", "in", "a,i") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("9", docs[0]["_id"])
def test_mongo_using_desk_filter_eq(self): f = FilterCondition("desk", "eq", "1") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("4", docs[0]["_id"])
def test_mongo_using_priority_compare_lte_filter(self): f = FilterCondition("priority", "lte", "3") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("5", docs[0]["_id"])
def test_mongo_using_desk_filter_eq(self): f = FilterCondition('desk', 'eq', '1') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('4', docs[0]['_id'])
def test_does_match_with_like_partial(self): f = FilterCondition('headline', 'like', 'tor') self.assertTrue(f.does_match(self.articles[0])) self.assertTrue(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_elastic_using_endswith_filter(self): f = FilterCondition('headline', 'endswith', 'Que') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, 'keyword') docs = get_resource_service('archive').get(req=self.req, lookup=None) self.assertEqual(1, docs.count()) self.assertEqual('2', docs[0]['_id'])
def test_mongo_using_place_filter_complete_string(self): f = FilterCondition('place', 'in', 'NSW') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('11', docs[0]['_id'])
def test_does_match_with_startswith_filter(self): f = FilterCondition('headline', 'startswith', 'Sto') self.assertTrue(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_does_match_with_notlike_filter(self): f = FilterCondition('headline', 'notlike', 'Que') self.assertTrue(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertTrue(f.does_match(self.articles[2])) self.assertTrue(f.does_match(self.articles[3])) self.assertTrue(f.does_match(self.articles[4])) self.assertTrue(f.does_match(self.articles[5]))
def test_mongo_using_like_filter_complete_string_eq(self): f = FilterCondition('headline', 'like', 'story') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual('1', docs[0]['_id'])
def test_does_match_with_eq(self): f = FilterCondition('urgency', 'eq', '1') self.assertTrue(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5]))
def test_does_match_with_nin_filter(self): f = FilterCondition('urgency', 'nin', '2,3,4') self.assertTrue(f.does_match(self.articles[0])) self.assertTrue(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertTrue(f.does_match(self.articles[5]))
def test_mongo_featuremedia_exists(self): f = FilterCondition("featuremedia", "exists", "true") query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service("archive").get_from_mongo(req=self.req, lookup=query) self.assertEqual(1, docs.count()) self.assertEqual("4", docs[0]["_id"])
def test_elastic_using_anpa_category_filter_complete_string(self): f = FilterCondition('anpa_category', 'in', 'a,i') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query) docs = get_resource_service('archive').get(req=self.req, lookup=None) doc_ids = [d['_id'] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue('9' in doc_ids)
def test_mongo_using_in_filter(self): f = FilterCondition('urgency', 'in', '3,4') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(2, docs.count()) self.assertEqual('3', docs[0]['_id']) self.assertEqual('4', docs[1]['_id'])
def test_mongo_using_notlike_filter(self): f = FilterCondition('headline', 'notlike', 'Que') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(10, docs.count()) doc_ids = [d['_id'] for d in docs] self.assertTrue('2' not in doc_ids)
def test_elastic_using_subject_filter_complete_string(self): f = FilterCondition("subject", "in", "05005003") query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query) docs = get_resource_service("archive").get(req=self.req, lookup=None) doc_ids = [d["_id"] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue("8" in doc_ids)
def test_elastic_using_notlike_filter(self): f = FilterCondition("headline", "notlike", "que") query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, "not") docs = get_resource_service("archive").get(req=self.req, lookup=None) self.assertEqual(9, docs.count()) doc_ids = [d["_id"] for d in docs] self.assertTrue("2" not in doc_ids)
def test_elastic_using_notlike_filter(self): f = FilterCondition('headline', 'notlike', 'que') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, 'not') docs = get_resource_service('archive').get(req=self.req, lookup=None) self.assertEqual(9, docs.count()) doc_ids = [d['_id'] for d in docs] self.assertTrue('2' not in doc_ids)
def test_elastic_using_ingest_provider_filter_eq(self): f = FilterCondition("ingest_provider", "eq", "1") query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query) docs = get_resource_service("archive").get(req=self.req, lookup=None) doc_ids = [d["_id"] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue("4" in doc_ids)
def test_elastic_using_sms_filter(self): f = FilterCondition('sms', 'in', 'true') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query) docs = get_resource_service('archive').get(req=self.req, lookup=None) doc_ids = [d['_id'] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue('3' in doc_ids)
def test_elastic_using_notlike_filter(self): f = FilterCondition('headline', 'notlike', 'que') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, 'not') docs = get_resource_service('archive').get(req=self.req, lookup=None) self.assertEqual(8, docs.count()) doc_ids = [d['_id'] for d in docs] self.assertTrue('2' not in doc_ids)
def test_elastic_using_featuremedia_exists(self): f = FilterCondition("featuremedia", "exists", "true") query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, search_type="exists") docs = get_resource_service("archive").get(req=self.req, lookup=None) doc_ids = [d["_id"] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue("4" in doc_ids)
def test_elastic_using_place_filter_complete_string(self): f = FilterCondition("place", "match", "NSW") query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, "match") docs = get_resource_service("archive").get(req=self.req, lookup=None) doc_ids = [d["_id"] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue("11" in doc_ids)
def test_elastic_using_place_filter_complete_string(self): f = FilterCondition('place', 'match', 'NSW') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, 'match') docs = get_resource_service('archive').get(req=self.req, lookup=None) doc_ids = [d['_id'] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue('11' in doc_ids)
def test_elastic_using_ingest_provider_filter_eq(self): f = FilterCondition('ingest_provider', 'eq', '1') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query) docs = get_resource_service('archive').get(req=self.req, lookup=None) doc_ids = [d['_id'] for d in docs] self.assertEqual(1, docs.count()) self.assertTrue('4' in doc_ids)
def test_mongo_using_like_filter_partial_string(self): f = FilterCondition('headline', 'like', 'tor') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) doc_ids = [d['_id'] for d in docs] self.assertEqual(2, docs.count()) self.assertTrue('1' in doc_ids) self.assertTrue('2' in doc_ids)
def test_mongo_using_notin_filter(self): f = FilterCondition('urgency', 'nin', '2,3,4') query = f.get_mongo_query() with self.app.app_context(): docs = get_resource_service('archive'). \ get_from_mongo(req=self.req, lookup=query) self.assertEqual(8, docs.count()) doc_ids = [d['_id'] for d in docs] self.assertTrue('1' in doc_ids) self.assertTrue('2' in doc_ids)
def test_elastic_using_nin_filter(self): f = FilterCondition('urgency', 'nin', '3,4') query = f.get_elastic_query() with self.app.app_context(): self._setup_elastic_args(query, 'not') docs = get_resource_service('archive').get(req=self.req, lookup=None) self.assertEqual(8, docs.count()) doc_ids = [d['_id'] for d in docs] self.assertTrue('6' in doc_ids) self.assertTrue('5' in doc_ids)
def test_does_match_with_genre_filter(self): f = FilterCondition('genre', 'in', 'Sidebar') self.assertFalse(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertFalse(f.does_match(self.articles[5])) self.assertTrue(f.does_match(self.articles[6])) self.assertFalse(f.does_match(self.articles[7])) self.assertFalse(f.does_match({'genre': None})) self.assertTrue(f.does_match({'genre': [{'name': 'Sidebar'}]})) self.assertFalse(f.does_match({'genre': [{'name': 'Article'}]})) self.assertTrue(f.does_match({'genre': [{'name': 'Sidebar'}, {'name': 'Article'}]}))
def build_mongo_query(self, doc): filter_condition_service = get_resource_service('filter_conditions') expressions = [] for expression in doc.get('content_filter', []): filter_conditions = [] if 'fc' in expression.get('expression', {}): for f in expression['expression']['fc']: current_filter = FilterCondition.parse(filter_condition_service.find_one(req=None, _id=f)) mongo_query = current_filter.get_mongo_query() filter_conditions.append(mongo_query) if 'pf' in expression.get('expression', {}): for f in expression['expression']['pf']: current_filter = super().find_one(req=None, _id=f) mongo_query = self.build_mongo_query(current_filter) filter_conditions.append(mongo_query) if len(filter_conditions) > 1: expressions.append({'$and': filter_conditions}) else: expressions.extend(filter_conditions) if len(expressions) > 1: return {'$or': expressions} else: return expressions[0]
def _get_elastic_query(self, doc, matching=True): expressions_list = [] if matching: expressions = {'should': expressions_list} else: expressions = {'must_not': expressions_list} filter_condition_service = get_resource_service('filter_conditions') for expression in doc.get('content_filter', []): filter_conditions = {'must': [], 'must_not': [{"term": {"state": "spiked"}}]} if 'fc' in expression.get('expression', {}): for f in expression['expression']['fc']: current_filter = FilterCondition.parse(filter_condition_service.find_one(req=None, _id=f)) elastic_query = current_filter.get_elastic_query() if current_filter.contains_not(): filter_conditions['must_not'].append(elastic_query) else: filter_conditions['must'].append(elastic_query) if 'pf' in expression.get('expression', {}): for f in expression['expression']['pf']: current_filter = super().find_one(req=None, _id=f) elastic_query = self._get_elastic_query(current_filter) filter_conditions['must'].append(elastic_query) expressions_list.append({'bool': filter_conditions}) return {'bool': expressions}
def build_mongo_query(self, doc): filter_condition_service = get_resource_service("filter_conditions") expressions = [] for expression in doc.get("content_filter", []): filter_conditions = [] if "fc" in expression.get("expression", {}): for f in expression["expression"]["fc"]: current_filter = FilterCondition.parse(filter_condition_service.find_one(req=None, _id=f)) mongo_query = current_filter.get_mongo_query() filter_conditions.append(mongo_query) if "pf" in expression.get("expression", {}): for f in expression["expression"]["pf"]: current_filter = super().find_one(req=None, _id=f) mongo_query = self.build_mongo_query(current_filter) filter_conditions.append(mongo_query) if len(filter_conditions) > 1: expressions.append({"$and": filter_conditions}) else: expressions.extend(filter_conditions) if len(expressions) > 1: return {"$or": expressions} else: return expressions[0]
def test_does_match_with_in_filter_case_insensitive(self): f = FilterCondition('source', 'in', 'aap,reuters') self.assertTrue(f.does_match({'source': 'AAP'})) self.assertTrue(f.does_match({'source': 'aap'})) self.assertTrue(f.does_match({'source': 'REUTERS'})) f = FilterCondition('source', 'in', 'AAP') self.assertTrue(f.does_match({'source': 'AAP'})) self.assertTrue(f.does_match({'source': 'aap'})) self.assertFalse(f.does_match({'source': 'REUTERS'}))
def test_does_match_with_embargo_filter_with_true(self): f = FilterCondition('embargo', 'eq', 'true') self.assertFalse(f.does_match(self.articles[0])) self.assertFalse(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertFalse(f.does_match(self.articles[3])) self.assertFalse(f.does_match(self.articles[4])) self.assertTrue(f.does_match(self.articles[5])) self.assertFalse(f.does_match(self.articles[6])) self.assertFalse(f.does_match(self.articles[7]))
def test_does_match_with_sms_filter(self): f = FilterCondition('sms', 'nin', 'true') self.assertTrue(f.does_match(self.articles[0])) self.assertTrue(f.does_match(self.articles[1])) self.assertFalse(f.does_match(self.articles[2])) self.assertTrue(f.does_match(self.articles[3])) self.assertTrue(f.does_match(self.articles[4])) self.assertTrue(f.does_match(self.articles[5])) self.assertTrue(f.does_match(self.articles[6])) self.assertTrue(f.does_match(self.articles[7]))