def process_data(self, token, trigger_id, date_triggered):
# call the model
from th_rss.models import Rss
# call the cache
from django.core.cache import get_cache
# get the URL from the trigger id
rss = Rss.objects.get(trigger_id=trigger_id)
self.name = rss.name
logger.debug("RSS Feeds from %s : url %s", self.name, rss.url)
# get the cache settings if any
parms = self._cache_settings()
# check if the cache is set otherwise no cache support at all
if 'rss' in parms:
# cache rss backend + parms
cache = get_cache('rss', **parms)
# datas from the cache
self.data = cache.get(self.name)
# data not in cache or expiried
if self.data is None or len(self.data) == 0:
# retreive the data
feeds = Feeds(**{'url_to_parse': rss.url}).datas()
# put in cache
if 'rss' in parms:
cache.set(self.name, feeds, parms['rss']['TIMEOUT'])
# get the cache
self.data = cache.get(self.name)
else:
self.data = feeds
# return the datas
return self.data
def read_data(self, token, trigger_id, date_triggered):
"""
get the data from the service
:param trigger_id: trigger ID to process
:param date_triggered: the date of the last trigger
:type trigger_id: int
:type date_triggered: datetime
:return: list of data found from the date_triggered filter
:rtype: list
"""
# get the URL from the trigger id
rss = super(ServiceRss, self).read_data('Rss', trigger_id)
logger.debug("RSS Feeds from %s : url %s", rss.name, rss.url)
now = arrow.utcnow().to(settings.TIME_ZONE)
published = ''
my_feeds = []
# retrieve the data
feeds = Feeds(**{'url_to_parse': rss.url}).datas()
for entry in feeds.entries:
if hasattr(entry, 'published_parsed'):
published = datetime.datetime.utcfromtimestamp(
time.mktime(entry.published_parsed))
elif hasattr(entry, 'created_parsed'):
published = datetime.datetime.utcfromtimestamp(
time.mktime(entry.created_parsed))
elif hasattr(entry, 'updated_parsed'):
published = datetime.datetime.utcfromtimestamp(
time.mktime(entry.updated_parsed))
if published == '':
published = now
else:
published = arrow.get(str(published)).to(settings.TIME_ZONE)
date_triggered = arrow.get(str(date_triggered)).to(
settings.TIME_ZONE)
if date_triggered is not None and\
published is not None and\
now >= published and\
published >= date_triggered:
my_feeds.append(entry)
cache.set('th_rss_' + str(trigger_id), my_feeds)
cache.set('th_rss_uuid_{}'.format(rss.uuid), my_feeds)
# return the data
return my_feeds
def read_data(self, **kwargs):
"""
get the data from the service
:param kwargs: contain keyword args : trigger_id and model name
:type kwargs: dict
:rtype: dict
"""
date_triggered = kwargs.get('date_triggered')
trigger_id = kwargs.get('trigger_id')
kwargs['model_name'] = 'Rss'
# get the URL from the trigger id
rss = super(ServiceRss, self).read_data(**kwargs)
logger.debug("RSS Feeds from %s : url %s", rss.name, rss.url)
now = arrow.utcnow().to(settings.TIME_ZONE)
published = ''
my_feeds = []
# retrieve the data
feeds = Feeds(**{'url_to_parse': rss.url}).datas()
for entry in feeds.entries:
if hasattr(entry, 'published_parsed'):
published = datetime.datetime.utcfromtimestamp(
time.mktime(entry.published_parsed))
elif hasattr(entry, 'created_parsed'):
published = datetime.datetime.utcfromtimestamp(
time.mktime(entry.created_parsed))
elif hasattr(entry, 'updated_parsed'):
published = datetime.datetime.utcfromtimestamp(
time.mktime(entry.updated_parsed))
if published == '':
published = now
else:
published = arrow.get(str(published)).to(settings.TIME_ZONE)
date_triggered = arrow.get(
str(date_triggered)).to(settings.TIME_ZONE)
if date_triggered is not None and\
published is not None and\
now >= published >= date_triggered:
my_feeds.append(entry)
cache.set('th_rss_' + str(trigger_id), my_feeds)
cache.set('th_rss_uuid_{}'.format(rss.uuid), my_feeds)
# return the data
return my_feeds
def read_data(self, **kwargs):
"""
get the data from the service
:param kwargs: contain keyword args : trigger_id and model name
:type kwargs: dict
:rtype: dict
"""
date_triggered = kwargs.get('date_triggered')
trigger_id = kwargs.get('trigger_id')
kwargs['model_name'] = 'Rss'
kwargs['app_label'] = 'django_th'
# get the URL from the trigger id
rss = super(ServiceRss, self).read_data(**kwargs)
logger.debug("RSS Feeds from %s : url %s", rss.name, rss.url)
now = arrow.utcnow().to(settings.TIME_ZONE)
my_feeds = []
# retrieve the data
feeds = Feeds(**{'url_to_parse': rss.url}).datas()
for entry in feeds.entries:
# entry.*_parsed may be None when the date in a RSS Feed is invalid
# so will have the "now" date as default
published = self._get_published(entry)
if published == '':
published = now
else:
published = arrow.get(str(published)).to(settings.TIME_ZONE)
date_triggered = arrow.get(str(date_triggered)).to(
settings.TIME_ZONE)
if date_triggered is not None and\
published is not None and\
now >= published >= date_triggered:
my_feeds.append(entry)
# digester
self.send_digest_event(trigger_id, entry.title, entry.link)
cache.set('th_rss_' + str(trigger_id), my_feeds)
cache.set('th_rss_uuid_{}'.format(rss.uuid), my_feeds)
# return the data
return my_feeds