def filter_expired_items(provider, items):
"""
Filters out the item from the list of articles to be ingested
if they are expired and item['type'] not in provider['content_types'].
:param provider: Ingest Provider Details.
:type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
:param items: list of items received from the provider
:type items: list
:return: list of items which can be saved into ingest collection
:rtype: list
"""
def is_not_expired(item):
if item.get('expiry') or item.get('versioncreated'):
expiry = item.get('expiry', item['versioncreated'] + delta)
if expiry.tzinfo:
return expiry > utcnow()
else:
return expiry > datetime.now()
return False
try:
delta = timedelta(minutes=provider.get('content_expiry', app.config['INGEST_EXPIRY_MINUTES']))
filtered_items = [item for item in items if is_not_expired(item) and
item.get(ITEM_TYPE, 'text') in provider.get('content_types', [])]
if len(items) != len(filtered_items):
logger.debug('Received {0} articles from provider {1}, but only {2} are eligible to be saved in ingest'
.format(len(items), provider['name'], len(filtered_items)))
return filtered_items
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items):
"""Filter out expired items from the list of articles to be ingested.
Filte both expired and `item['type'] not in provider['content_types']`.
:param provider: Ingest Provider Details.
:type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
:param items: list of items received from the provider
:type items: list
:return: list of items which can be saved into ingest collection
:rtype: list
"""
def is_not_expired(item):
if item.get('expiry') or item.get('versioncreated'):
expiry = item.get('expiry', item['versioncreated'] + delta)
if expiry.tzinfo:
return expiry > utcnow()
else:
return expiry > datetime.now()
return False
try:
delta = timedelta(minutes=provider.get('content_expiry', app.config['INGEST_EXPIRY_MINUTES']))
filtered_items = [item for item in items if is_not_expired(item) and
item.get(ITEM_TYPE, 'text') in provider.get('content_types', [])]
if len(items) != len(filtered_items):
logger.debug('Received {0} articles from provider {1}, but only {2} are eligible to be saved in ingest'
.format(len(items), provider['name'], len(filtered_items)))
return filtered_items
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items):
try:
days_to_keep_content = provider.get('days_to_keep', DAYS_TO_KEEP)
expiration_date = utcnow() - timedelta(days=days_to_keep_content)
return [item for item in items if item.get('versioncreated', utcnow()) > expiration_date]
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items):
try:
days_to_keep_content = provider.get('days_to_keep', DAYS_TO_KEEP)
expiration_date = utcnow() - timedelta(days=days_to_keep_content)
return [
item for item in items
if item.get('versioncreated', utcnow()) > expiration_date
]
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items):
def is_not_expired(item):
if item.get('expiry') or item.get('versioncreated'):
expiry = item.get('expiry', item['versioncreated'] + delta)
if expiry.tzinfo:
return expiry > utcnow()
return False
try:
delta = timedelta(minutes=provider.get('content_expiry', app.config['INGEST_EXPIRY_MINUTES']))
return [item for item in items if is_not_expired(item)]
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items):
def is_not_expired(item):
if item.get("expiry") or item.get("versioncreated"):
expiry = item.get("expiry", item["versioncreated"] + delta)
if expiry.tzinfo:
return expiry > utcnow()
return False
try:
delta = timedelta(minutes=provider.get("content_expiry", INGEST_EXPIRY_MINUTES))
return [item for item in items if is_not_expired(item)]
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items):
def is_not_expired(item):
if item.get('expiry') or item.get('versioncreated'):
expiry = item.get('expiry', item['versioncreated'] + delta)
if expiry.tzinfo:
return expiry > utcnow()
return False
try:
delta = timedelta(minutes=provider.get(
'content_expiry', app.config['INGEST_EXPIRY_MINUTES']))
return [item for item in items if is_not_expired(item)]
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
def test_raise_providerFilterExpiredContentError(self):
with assert_raises(ProviderError) as error_context:
try:
ex = Exception("Testing providerFilterExpiredContentError")
raise ex
except Exception:
raise ProviderError.providerFilterExpiredContentError(ex, self.provider)
exception = error_context.exception
self.assertTrue(exception.code == 2006)
self.assertTrue(exception.message == "Expired content could not be filtered")
self.assertIsNotNone(exception.system_exception)
self.assertEqual(exception.system_exception.args[0], "Testing providerFilterExpiredContentError")
self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
self.assertEqual(self.mock_logger_handler.messages['error'][0],
"ProviderError Error 2006 - Expired content could not be filtered: "
"Testing providerFilterExpiredContentError on channel TestProvider")
def test_raise_providerFilterExpiredContentError(self):
with assert_raises(ProviderError) as error_context:
try:
ex = Exception("Testing providerFilterExpiredContentError")
raise ex
except Exception:
raise ProviderError.providerFilterExpiredContentError(ex, self.provider)
exception = error_context.exception
self.assertTrue(exception.code == 2006)
self.assertTrue(exception.message == "Expired content could not be filtered")
self.assertIsNotNone(exception.system_exception)
self.assertEqual(exception.system_exception.args[0], "Testing providerFilterExpiredContentError")
self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
self.assertEqual(self.mock_logger_handler.messages['error'][0],
"ProviderError Error 2006 - Expired content could not be filtered: "
"Testing providerFilterExpiredContentError on channel TestProvider")
def filter_expired_items(provider, items):
"""Filter out expired items from the list of articles to be ingested.
Filte both expired and `item['type'] not in provider['content_types']`.
:param provider: Ingest Provider Details.
:type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource`
:param items: list of items received from the provider
:type items: list
:return: list of items which can be saved into ingest collection
:rtype: list
"""
try:
try:
content_expiry = int(provider['content_expiry'])
except ValueError:
logger.warning(
'invalid content_expiry: content_expiry={value}'.format(
value=provider['content_expiry']))
del provider['content_expiry']
content_expiry = None
except (TypeError, KeyError):
content_expiry = None
else:
if content_expiry < 0:
del provider['content_expiry']
content_expiry = None
delta = timedelta(
minutes=content_expiry or app.config['INGEST_EXPIRY_MINUTES'])
filtered_items = [
item for item in items if is_not_expired(item, delta) and item.get(
ITEM_TYPE, 'text') in provider.get('content_types', [])
]
if len(items) != len(filtered_items):
logger.warning(
'Received {0} articles from provider {1}, but only {2} are eligible to be saved in ingest'
.format(len(items), provider['name'], len(filtered_items)))
return filtered_items
except Exception as ex:
raise ProviderError.providerFilterExpiredContentError(ex, provider)
