def filter_expired_items(provider, items): """ Filters out the item from the list of articles to be ingested if they are expired and item['type'] not in provider['content_types']. :param provider: Ingest Provider Details. :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource` :param items: list of items received from the provider :type items: list :return: list of items which can be saved into ingest collection :rtype: list """ def is_not_expired(item): if item.get('expiry') or item.get('versioncreated'): expiry = item.get('expiry', item['versioncreated'] + delta) if expiry.tzinfo: return expiry > utcnow() else: return expiry > datetime.now() return False try: delta = timedelta(minutes=provider.get('content_expiry', app.config['INGEST_EXPIRY_MINUTES'])) filtered_items = [item for item in items if is_not_expired(item) and item.get(ITEM_TYPE, 'text') in provider.get('content_types', [])] if len(items) != len(filtered_items): logger.debug('Received {0} articles from provider {1}, but only {2} are eligible to be saved in ingest' .format(len(items), provider['name'], len(filtered_items))) return filtered_items except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items): """Filter out expired items from the list of articles to be ingested. Filte both expired and `item['type'] not in provider['content_types']`. :param provider: Ingest Provider Details. :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource` :param items: list of items received from the provider :type items: list :return: list of items which can be saved into ingest collection :rtype: list """ def is_not_expired(item): if item.get('expiry') or item.get('versioncreated'): expiry = item.get('expiry', item['versioncreated'] + delta) if expiry.tzinfo: return expiry > utcnow() else: return expiry > datetime.now() return False try: delta = timedelta(minutes=provider.get('content_expiry', app.config['INGEST_EXPIRY_MINUTES'])) filtered_items = [item for item in items if is_not_expired(item) and item.get(ITEM_TYPE, 'text') in provider.get('content_types', [])] if len(items) != len(filtered_items): logger.debug('Received {0} articles from provider {1}, but only {2} are eligible to be saved in ingest' .format(len(items), provider['name'], len(filtered_items))) return filtered_items except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items): try: days_to_keep_content = provider.get('days_to_keep', DAYS_TO_KEEP) expiration_date = utcnow() - timedelta(days=days_to_keep_content) return [item for item in items if item.get('versioncreated', utcnow()) > expiration_date] except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items): try: days_to_keep_content = provider.get('days_to_keep', DAYS_TO_KEEP) expiration_date = utcnow() - timedelta(days=days_to_keep_content) return [ item for item in items if item.get('versioncreated', utcnow()) > expiration_date ] except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items): def is_not_expired(item): if item.get('expiry') or item.get('versioncreated'): expiry = item.get('expiry', item['versioncreated'] + delta) if expiry.tzinfo: return expiry > utcnow() return False try: delta = timedelta(minutes=provider.get('content_expiry', app.config['INGEST_EXPIRY_MINUTES'])) return [item for item in items if is_not_expired(item)] except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items): def is_not_expired(item): if item.get("expiry") or item.get("versioncreated"): expiry = item.get("expiry", item["versioncreated"] + delta) if expiry.tzinfo: return expiry > utcnow() return False try: delta = timedelta(minutes=provider.get("content_expiry", INGEST_EXPIRY_MINUTES)) return [item for item in items if is_not_expired(item)] except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def filter_expired_items(provider, items): def is_not_expired(item): if item.get('expiry') or item.get('versioncreated'): expiry = item.get('expiry', item['versioncreated'] + delta) if expiry.tzinfo: return expiry > utcnow() return False try: delta = timedelta(minutes=provider.get( 'content_expiry', app.config['INGEST_EXPIRY_MINUTES'])) return [item for item in items if is_not_expired(item)] except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)
def test_raise_providerFilterExpiredContentError(self): with assert_raises(ProviderError) as error_context: try: ex = Exception("Testing providerFilterExpiredContentError") raise ex except Exception: raise ProviderError.providerFilterExpiredContentError(ex, self.provider) exception = error_context.exception self.assertTrue(exception.code == 2006) self.assertTrue(exception.message == "Expired content could not be filtered") self.assertIsNotNone(exception.system_exception) self.assertEqual(exception.system_exception.args[0], "Testing providerFilterExpiredContentError") self.assertEqual(len(self.mock_logger_handler.messages['error']), 1) self.assertEqual(self.mock_logger_handler.messages['error'][0], "ProviderError Error 2006 - Expired content could not be filtered: " "Testing providerFilterExpiredContentError on channel TestProvider")
def test_raise_providerFilterExpiredContentError(self): with assert_raises(ProviderError) as error_context: try: ex = Exception("Testing providerFilterExpiredContentError") raise ex except Exception: raise ProviderError.providerFilterExpiredContentError(ex, self.provider) exception = error_context.exception self.assertTrue(exception.code == 2006) self.assertTrue(exception.message == "Expired content could not be filtered") self.assertIsNotNone(exception.system_exception) self.assertEqual(exception.system_exception.args[0], "Testing providerFilterExpiredContentError") self.assertEqual(len(self.mock_logger_handler.messages['error']), 1) self.assertEqual(self.mock_logger_handler.messages['error'][0], "ProviderError Error 2006 - Expired content could not be filtered: " "Testing providerFilterExpiredContentError on channel TestProvider")
def filter_expired_items(provider, items): """Filter out expired items from the list of articles to be ingested. Filte both expired and `item['type'] not in provider['content_types']`. :param provider: Ingest Provider Details. :type provider: dict :py:class: `superdesk.io.ingest_provider_model.IngestProviderResource` :param items: list of items received from the provider :type items: list :return: list of items which can be saved into ingest collection :rtype: list """ try: try: content_expiry = int(provider['content_expiry']) except ValueError: logger.warning( 'invalid content_expiry: content_expiry={value}'.format( value=provider['content_expiry'])) del provider['content_expiry'] content_expiry = None except (TypeError, KeyError): content_expiry = None else: if content_expiry < 0: del provider['content_expiry'] content_expiry = None delta = timedelta( minutes=content_expiry or app.config['INGEST_EXPIRY_MINUTES']) filtered_items = [ item for item in items if is_not_expired(item, delta) and item.get( ITEM_TYPE, 'text') in provider.get('content_types', []) ] if len(items) != len(filtered_items): logger.warning( 'Received {0} articles from provider {1}, but only {2} are eligible to be saved in ingest' .format(len(items), provider['name'], len(filtered_items))) return filtered_items except Exception as ex: raise ProviderError.providerFilterExpiredContentError(ex, provider)