def test_expiring_with_content(self):
provider_name = 'reuters'
guid = 'tag_reuters.com_2014_newsml_KBN0FL0NM:10'
provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
provider_service = self._get_provider_service(provider)
provider_service.provider = provider
provider_service.URL = provider.get('config', {}).get('url')
items = provider_service.fetch_ingest(guid)
now = utcnow()
for i, item in enumerate(items):
item['ingest_provider'] = provider['_id']
expiry_time = now - timedelta(hours=11)
if i > 4:
expiry_time = now + timedelta(minutes=11)
item['expiry'] = item['versioncreated'] = expiry_time
service = get_resource_service('ingest')
service.post(items)
# ingest the items and expire them
before = service.get(req=None, lookup={})
self.assertEqual(6, before.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# only one left in ingest
after = service.get(req=None, lookup={})
self.assertEqual(1, after.count())
def test_expiring_content_with_files(self):
provider_name = 'reuters'
guid = 'tag_reuters.com_2014_newsml_KBN0FL0NM:10'
provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
provider_service = self._get_provider_service(provider)
provider_service.provider = provider
provider_service.URL = provider.get('config', {}).get('url')
items = provider_service.fetch_ingest(guid)
for item in items:
item['ingest_provider'] = provider['_id']
now = utcnow()
items[0]['expiry'] = now - timedelta(hours=11)
items[1]['expiry'] = now - timedelta(hours=11)
items[2]['expiry'] = now + timedelta(hours=11)
items[5]['versioncreated'] = now + timedelta(minutes=11)
# ingest the items and expire them
self.ingest_items(items, provider, provider_service)
# four files in grid fs
current_files = self.app.media.fs('upload').find()
self.assertEqual(4, current_files.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# all gone
current_files = self.app.media.fs('upload').find()
self.assertEqual(0, current_files.count())
def test_expiring_with_content(self):
provider, provider_service = self.setup_reuters_provider()
items = provider_service.fetch_ingest(reuters_guid)
now = utcnow()
for i, item in enumerate(items):
item['ingest_provider'] = provider['_id']
expiry_time = now - timedelta(hours=11)
if i > 4:
expiry_time = now + timedelta(minutes=11)
item['expiry'] = item['versioncreated'] = expiry_time
service = get_resource_service('ingest')
service.post(items)
# ingest the items and expire them
before = service.get(req=None, lookup={})
self.assertEqual(6, before.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# only one left in ingest
after = service.get(req=None, lookup={})
self.assertEqual(1, after.count())
req = ParsedRequest()
self.assertEqual(1, self.app.data.elastic.find('ingest', req, {}).count())
self.assertEqual(1, self.app.data.mongo.find('ingest', req, {}).count())
def test_expiring_content_with_files(self):
provider, provider_service = self.setup_reuters_provider()
items = provider_service.fetch_ingest(reuters_guid)
for item in items:
item['ingest_provider'] = provider['_id']
now = utcnow()
items[0]['expiry'] = now - timedelta(hours=11)
items[1]['expiry'] = now - timedelta(hours=11)
items[2]['expiry'] = now + timedelta(hours=11)
items[5]['versioncreated'] = now + timedelta(minutes=11)
# ingest the items and expire them
self.ingest_items(items, provider, provider_service)
# four files in grid fs
current_files = self.app.media.fs('upload').find()
self.assertEqual(4, current_files.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# all gone
current_files = self.app.media.fs('upload').find()
self.assertEqual(0, current_files.count())
def test_expiring_with_content(self):
provider_name = 'reuters'
guid = 'tag_reuters.com_2014_newsml_KBN0FL0NM:10'
provider = get_resource_service('ingest_providers').find_one(
name=provider_name, req=None)
provider_service = self._get_provider_service(provider)
provider_service.provider = provider
provider_service.URL = provider.get('config', {}).get('url')
items = provider_service.fetch_ingest(guid)
now = utcnow()
for i, item in enumerate(items):
item['ingest_provider'] = provider['_id']
expiry_time = now - timedelta(hours=11)
if i > 4:
expiry_time = now + timedelta(minutes=11)
item['expiry'] = item['versioncreated'] = expiry_time
service = get_resource_service('ingest')
service.post(items)
# ingest the items and expire them
before = service.get(req=None, lookup={})
self.assertEqual(6, before.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# only one left in ingest
after = service.get(req=None, lookup={})
self.assertEqual(1, after.count())
def test_expiring_content_with_files(self):
provider, provider_service = self.setup_reuters_provider()
items = provider_service.fetch_ingest(reuters_guid)
for item in items:
item['ingest_provider'] = provider['_id']
now = utcnow()
items[0]['expiry'] = now - timedelta(hours=11)
items[1]['expiry'] = now - timedelta(hours=11)
items[2]['expiry'] = now + timedelta(hours=11)
items[5]['versioncreated'] = now + timedelta(minutes=11)
# ingest the items and expire them
self.ingest_items(items, provider, provider_service)
# four files in grid fs
current_files = self.app.media.fs('upload').find()
self.assertEqual(4, current_files.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# all gone
current_files = self.app.media.fs('upload').find()
self.assertEqual(0, current_files.count())
def test_expiring_with_content(self):
provider, provider_service = self.setup_reuters_provider()
items = provider_service.fetch_ingest(reuters_guid)
now = utcnow()
for i, item in enumerate(items):
item['ingest_provider'] = provider['_id']
expiry_time = now - timedelta(hours=11)
if i > 4:
expiry_time = now + timedelta(minutes=11)
item['expiry'] = item['versioncreated'] = expiry_time
service = get_resource_service('ingest')
service.post(items)
# ingest the items and expire them
before = service.get(req=None, lookup={})
self.assertEqual(6, before.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# only one left in ingest
after = service.get(req=None, lookup={})
self.assertEqual(1, after.count())
req = ParsedRequest()
self.assertEqual(1, self.app.data.elastic.find('ingest', req, {}).count())
self.assertEqual(1, self.app.data.mongo.find('ingest', req, {}).count())
def test_expiring_content_with_files(self):
provider_name = 'reuters'
guid = 'tag_reuters.com_2014_newsml_KBN0FL0NM:10'
provider = get_resource_service('ingest_providers').find_one(name=provider_name, req=None)
provider_service = self._get_provider_service(provider)
provider_service.provider = provider
provider_service.URL = provider.get('config', {}).get('url')
items = provider_service.fetch_ingest(guid)
for item in items:
item['ingest_provider'] = provider['_id']
now = utcnow()
items[0]['expiry'] = now - timedelta(hours=11)
items[1]['expiry'] = now - timedelta(hours=11)
items[2]['expiry'] = now + timedelta(hours=11)
items[5]['versioncreated'] = now + timedelta(minutes=11)
# ingest the items and expire them
self.ingest_items(items, provider, provider_service)
# four files in grid fs
current_files = self.app.media.fs('upload').find()
self.assertEqual(4, current_files.count())
remove = RemoveExpiredContent()
remove.run(provider.get('type'))
# all gone
current_files = self.app.media.fs('upload').find()
self.assertEqual(0, current_files.count())
def test_removing_expired_items_from_elastic_only(self):
now = utcnow()
self.app.data.elastic.insert('ingest', [
{'_id': 'foo', 'expiry': now - timedelta(minutes=30)},
{'_id': 'bar', 'expiry': now + timedelta(minutes=30)},
])
RemoveExpiredContent().run()
self.assertEqual(1, self.app.data.elastic.find('ingest', ParsedRequest(), {}).count())
def test_removing_expired_items_from_elastic_only(self):
now = utcnow()
self.app.data.elastic.insert(
"ingest",
[
{
"_id": "foo",
"expiry": now - timedelta(minutes=30)
},
{
"_id": "bar",
"expiry": now + timedelta(minutes=30)
},
],
)
RemoveExpiredContent().run()
self.assertEqual(
1,
self.app.data.elastic.find("ingest", ParsedRequest(), {}).count())
def gc_ingest():
RemoveExpiredContent().run()
