def test_aggs_with_request(self):
params = {
'params': {
'dates': {
'filter': 'yesterday'
}
},
'aggs': {
'group': {
'field': 'anpa_category.qcode'
},
'subgroup': {
'field': 'urgency'
}
},
'repo': 'published',
'return_type': 'text/csv'
}
expected_args = {
'params': params['params'],
'aggs': params['aggs'],
'repo': params['repo'],
'return_type': params['return_type']
}
request = ParsedRequest()
# Request object with source as a json in string format
request.args = {
'params': json.dumps(params['params']),
'repo': params['repo'],
'return_type': params['return_type'],
'aggs': json.dumps(params['aggs'])
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with args an ImmutableMultiDict
request.args = ImmutableMultiDict({
'params': params['params'],
'repo': params['repo'],
'return_type': params['return_type'],
'aggs': params['aggs']
})
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with source as a normal dict
request.args = {
'params': params['params'],
'repo': params['repo'],
'return_type': params['return_type'],
'aggs': params['aggs']
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
def get_history_items(self, last_id, gte, item_id, chunk_size=0):
history_service = get_resource_service('archive_history')
last_processed_id = last_id
while True:
req = ParsedRequest()
req.sort = '[("_id", 1), ("version", 1)]'
query = {'$and': []}
if gte:
query['$and'].append({'_created': {'$gte': date_to_str(gte)}})
if item_id:
query['$and'].append({'item_id': str(item_id)})
if last_processed_id:
query['$and'].append({'_id': {'$gt': str(last_processed_id)}})
req.where = json.dumps(query)
if chunk_size > 0:
req.max_results = int(chunk_size)
items = list(history_service.get(req=req, lookup=None))
if len(items) < 1:
break
last_processed_id = items[-1][config.ID_FIELD]
yield items
def test_bulletin_builder_formatter(self):
article = {
'source': 'AAP',
'anpa_category': [{
'qcode': 'a'
}],
'headline': 'This is a test headline',
'byline': 'joe',
'slugline': 'slugline',
'subject': [{
'qcode': '02011001'
}],
'anpa_take_key': 'take_key',
'unique_id': '1',
'type': 'preformatted',
'body_html': 'The story body',
'word_count': '1',
'priority': '1',
'firstcreated': utcnow(),
'versioncreated': utcnow(),
'lock_user': ObjectId()
}
subscriber = self.app.data.find('subscribers', None, None)[0]
seq, item = self._formatter.format(article, subscriber)[0]
self.assertGreater(int(seq), 0)
self.assertEqual(
json.dumps(article, default=json_serialize_datetime_objectId),
item)
def TestBulletinBuilderFormatter(self):
article = {
'source': 'AAP',
'anpa-category': {'qcode': 'a'},
'headline': 'This is a test headline',
'byline': 'joe',
'slugline': 'slugline',
'subject': [{'qcode': '02011001'}],
'anpa_take_key': 'take_key',
'unique_id': '1',
'type': 'preformatted',
'body_html': 'The story body',
'word_count': '1',
'priority': '1',
'firstcreated': utcnow(),
'versioncreated': utcnow(),
'lock_user': ObjectId()
}
with self.app.app_context():
subscriber = self.app.data.find('subscribers', None, None)[0]
f = AAPBulletinBuilderFormatter()
seq, item = f.format(article, subscriber)[0]
self.assertGreater(int(seq), 0)
self.assertEqual(json.dumps(article, default=json_serialize_datetime_objectId), item)
def test_bulletin_builder_formatter(self):
article = {
config.ID_FIELD: '123',
config.VERSION: 2,
'source': 'AAP',
'anpa_category': [{'qcode': 'a'}],
'headline': 'This is a test headline',
'byline': 'joe',
'slugline': 'slugline',
'subject': [{'qcode': '02011001'}],
'anpa_take_key': 'take_key',
'unique_id': '1',
'type': 'preformatted',
'body_html': 'The story body',
'word_count': '1',
'priority': '1',
'firstcreated': utcnow(),
'versioncreated': utcnow(),
'lock_user': ObjectId()
}
subscriber = self.app.data.find('subscribers', None, None)[0]
seq, item = self._formatter.format(article, subscriber)[0]
item = json.loads(item)
self.assertGreater(int(seq), 0)
self.assertEqual(article[config.ID_FIELD], item.get('id'))
self.assertEqual(article[config.VERSION], item.get('version'))
self.assertEqual(article[ITEM_TYPE], item.get(ITEM_TYPE))
self.assertEqual(article.get(PACKAGE_TYPE, ''), item.get(PACKAGE_TYPE))
self.assertEqual(article['headline'], item.get('headline'))
self.assertEqual(article['slugline'], item.get('slugline'))
self.assertEqual(json.dumps(article, default=json_serialize_datetime_objectId),
item.get('data'))
def test_aggs_with_lookup(self):
lookup = {
'params': {
'dates': {
'filter': 'yesterday'
}
},
'aggs': {
'group': {
'field': 'anpa_category.qcode'
},
'subgroup': {
'field': 'urgency'
}
},
'repo': 'published',
'return_type': 'text/csv'
}
expected_args = {
'params': lookup['params'],
'aggs': lookup['aggs'],
'repo': lookup['repo'],
'return_type': lookup['return_type']
}
# Lookup with aggs as a json in string format
args = self.service._get_request_or_lookup(
req=None,
params=json.dumps(lookup['params']),
repo=lookup['repo'],
return_type=lookup['return_type'],
aggs=json.dumps(lookup['aggs']))
self.assertEqual(args, expected_args)
# Lookup with aggs as a normal dict
args = self.service._get_request_or_lookup(
req=None,
params=lookup['params'],
repo=lookup['repo'],
return_type=lookup['return_type'],
aggs=lookup['aggs'])
self.assertEqual(args, expected_args)
def test_bulletin_builder_formatter(self):
article = {
config.ID_FIELD: '123',
config.VERSION: 2,
'source': 'AAP',
'anpa_category': [{
'qcode': 'a'
}],
'headline': 'This is a test headline',
'byline': 'joe',
'slugline': 'slugline',
'subject': [{
'qcode': '02011001'
}],
'anpa_take_key': 'take_key',
'unique_id': '1',
'type': 'preformatted',
'body_html': 'The story body',
'word_count': '1',
'priority': '1',
'firstcreated': utcnow(),
'versioncreated': utcnow(),
'lock_user': ObjectId()
}
subscriber = self.app.data.find('subscribers', None, None)[0]
seq, item = self._formatter.format(article, subscriber)[0]
item = json.loads(item)
self.assertGreater(int(seq), 0)
self.assertEqual(article[config.ID_FIELD], item.get('id'))
self.assertEqual(article[config.VERSION], item.get('version'))
self.assertEqual(article[ITEM_TYPE], item.get(ITEM_TYPE))
self.assertEqual(article.get(PACKAGE_TYPE, ''), item.get(PACKAGE_TYPE))
self.assertEqual(article['headline'], item.get('headline'))
self.assertEqual(article['slugline'], item.get('slugline'))
self.assertEqual(
json.dumps(article, default=json_serialize_datetime_objectId),
item.get('data'))
def get(self, req, lookup):
"""
Overriding to pass user as search parameter
"""
session_user = str(get_user_id(required=True))
if not req:
req = ParsedRequest()
where = json.loads(req.where) if req.where else {}
if lookup:
where.update(lookup)
if '$or' not in where:
where['$or'] = []
# Restrict the saved reports to either global or owned by current user
where['$or'].extend([{'is_global': True}, {'user': session_user}])
req.where = json.dumps(where)
return super().get(req, lookup=None)
def test_get_with_lookup(self):
lookup = {
'source': {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [],
'must_not': []
}
}
}
}
},
'params': {
'dates': {'filter': 'yesterday'}
},
'repo': 'published',
'return_type': 'text/csv'
}
expected_args = {
'source': lookup['source'],
'repo': lookup['repo'],
'return_type': lookup['return_type']
}
# Lookup with source as a json in string format
args = self.service._get_request_or_lookup(
req=None,
source=json.dumps(lookup['source']),
repo=lookup['repo'],
return_type=lookup['return_type']
)
self.assertEqual(args, expected_args)
# Lookup with source as a normal dict
args = self.service._get_request_or_lookup(
req=None,
source=lookup['source'],
repo=lookup['repo'],
return_type=lookup['return_type']
)
self.assertEqual(args, expected_args)
# return_type default
args = self.service._get_request_or_lookup(
req=None,
source=lookup['source'],
repo=lookup['repo']
)
self.assertEqual(args.get('return_type'), 'aggregations')
# Lookup with params as json in string format
expected_args = {
'params': lookup['params'],
'repo': lookup['repo'],
'return_type': lookup['return_type']
}
args = self.service._get_request_or_lookup(
req=None,
params=json.dumps(lookup['params']),
repo=lookup['repo'],
return_type=lookup['return_type']
)
self.assertEqual(args, expected_args)
# Lookup with params as normal dict
args = self.service._get_request_or_lookup(
req=None,
params=lookup['params'],
repo=lookup['repo'],
return_type=lookup['return_type']
)
self.assertEqual(args, expected_args)
def test_get_with_request(self):
params = {
'source': {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [],
'must_not': []
}
}
}
}
},
'params': {
'dates': {'filter': 'yesterday'}
},
'repo': 'published',
'return_type': 'text/csv'
}
expected_args = {
'source': params['source'],
'repo': params['repo'],
'return_type': params['return_type']
}
request = ParsedRequest()
# Request object with source as a json in string format
request.args = {
'source': json.dumps(params['source']),
'repo': params['repo'],
'return_type': params['return_type'],
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with args an ImmutableMultiDict
request.args = ImmutableMultiDict({
'source': json.dumps(params['source']),
'repo': params['repo'],
'return_type': params['return_type'],
})
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with source as a normal dict
request.args = {
'source': params['source'],
'repo': params['repo'],
'return_type': params['return_type'],
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# return_type default
request.args = {
'source': params['source'],
'repo': params['repo'],
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args.get('return_type'), 'aggregations')
# Request object with params as json in string format
expected_args = {
'params': params['params'],
'repo': params['repo'],
'return_type': params['return_type']
}
request.args = {
'params': json.dumps(params['params']),
'repo': params['repo'],
'return_type': params['return_type'],
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with params as a normal dict
request.args = {
'params': params['params'],
'repo': params['repo'],
'return_type': params['return_type'],
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)