result, data = imap_conn.uid('fetch',
','.join(emails_ids_chunk),
'(RFC822)')
if result == 'OK':
for raw_data in data:
if len(raw_data) == 1:
continue
yield raw_data[1]
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(MailAccount,
MailAccount.test_connection,
globals(),
queue=u'swarm',
expire=3600)
register_task_method(MailAccount,
MailAccount.update_mailboxes,
globals(),
queue=u'refresh',
expire=3600)
# They are registered just above.
MailAccount.usable_start_task = staticmethod(
mailaccount_update_mailboxes_task) # NOQA
MailAccount.reset_unusable_task = staticmethod(
mailaccount_test_connection_task) # NOQA
content=self.name,
url=self.url,
media=self.origin_to_json,
date_published=self.date_published
or getattr(related_to, 'date_published', None),
authors=[(a.name or a.origin_name) for a in self.authors.all()],
date_updated=None,
language=self.language.dj_code if self.language else None,
text_direction=self.text_direction,
tags=[t.name for t in self.tags.all()],
)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Tweet, Tweet.post_create_task, globals(), queue=u'create')
register_task_method(Tweet, Tweet.fetch_entities, globals(), queue=u'fetch')
# register_task_method(Tweet, Tweet.find_image,
# globals(), queue=u'fetch', default_retry_delay=3600)
@task(queue='background')
def mark_tweet_deleted(tweet_id):
try:
tweet = Tweet.objects.get(tweet_id=tweet_id)
except:
LOGGER.warning(u'Unknown tweet to delete: %s', tweet_id)
# Search/tweets doesn't use locations (bounding boxes),
# but geocodes (bounding circles)… We need to convert,
# and results won't be exactly the same.
# Geocode: 37.781157,-122.398720,1mi
self.__consume_items('search/tweets',
parameters, backfilling=True)
finally:
self.check_new_good_period(period_start_item, self.oldest_id)
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(TwitterFeed, TwitterFeed.backfill,
globals(), queue=u'permanent')
register_task_method(TwitterFeed, TwitterFeed.consume,
globals(), queue=u'permanent')
# ————————————————————————————————————————————————————————————————————— Signals
#
# HEADS UP: see subscription.py for other signals.
#
def twitterfeed_pre_save(instance, **kwargs):
""" Update owner's subscription name when twitterfeed name changes. """
twitterfeed = instance
elif proposal.action == PROPOSAL_ACTION_TYPES.CHANGE:
# Change proposal.target in self.
# Note: proposal.target can be self.
retval = True
else:
LOGGER.warning(u'Unhandled action %s for %s on %s',
proposal.action, proposal.target, self)
return retval
# ——————————————————————————————————————————————————————————————— Tasks methods
register_task_method(Folder, Folder.purge, globals(), queue=u'background')
# ————————————————————————————————————————————————————————————————————— Signals
def folder_pre_save(instance, **kwargs):
if not instance.slug:
instance.slug = slugify(instance.name)
if instance.is_shared and 'is_shared' in instance.changed_fields:
if instance.is_root:
# For now, root folder cannot be
# shared. This seems a bad idea.
instance.is_shared = False
else:
LOGGER.info(
u'URL of %s (#%s) successfully absolutized '
u'from %s to %s.', self._meta.model.__name__, self.id,
old_url, final_url)
else:
# Don't do the job twice.
if self.url_error:
statsd.gauge('articles.counts.url_errors', -1, delta=True)
statsd.gauge('articles.counts.absolutes', 1, delta=True)
self.url_absolute = True
self.url_error = None
# Don't waste a version just for that.
self.save_without_historical_record()
return True
# ———————————————————————————————————————————————————————————————— Celery Tasks
# HEADS UP: we need to register against BaseItem, because UrlItem is abstract
# and cannot run .objects.get() in register_task_method().
register_task_method(BaseItem,
UrlItem.absolutize_url,
globals(),
queue=u'swarm',
default_retry_delay=3600)
folder.compute_cached_descriptors(all=True, unread=True)
LOGGER.info(
u'Checked subscription #%s. '
u'%s/%s non-existing/re-checked, '
u'%s/%s read/unread and %s not created.', self.id,
counters.missing, counters.rechecked, counters.reads,
counters.unreads, counters.failed)
return counters
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(Subscription,
Subscription.mark_all_read_in_database,
globals(),
queue=u'background')
register_task_method(Subscription,
Subscription.check_reads,
globals(),
queue=u'check')
# ————————————————————————————————————————————————————————————————————— Signals
def subscription_pre_save(instance, **kwargs):
""" Subscribe the mailfeed's owner if feed is beiing created. """
subscription = instance
if not subscription.pk:
baseitem_create_reads_task.apply((self.id, ))
baseitem_process_task.apply((self.id, ))
else:
post_create_reads_chain = tasks_chain(
baseitem_process_task.si(self.id), )
baseitem_create_reads_task.apply_async(
args=(self.id, ),
kwargs={'stop_chain_on_false': True},
link=post_create_reads_chain)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Email, Email.post_create_task, globals(), queue=u'create')
# ————————————————————————————————————————————————————————————————————— Signals
def email_pre_save(instance, **kwargs):
""" Make a slug if none. """
email = instance
if not email.slug:
email.slug = slugify(email.name)
def email_post_save(instance, **kwargs):
else:
post_create_reads_chain = tasks_chain(
baseitem_process_task.si(self.id),
)
baseitem_create_reads_task.apply_async(
args=(self.id, ),
kwargs={'stop_chain_on_false': True},
link=post_create_reads_chain
)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Email, Email.post_create_task,
globals(), queue=u'create')
# ————————————————————————————————————————————————————————————————————— Signals
def email_pre_save(instance, **kwargs):
""" Make a slug if none. """
email = instance
if not email.slug:
email.slug = slugify(email.name)
def email_post_save(instance, **kwargs):
# Update the last connection datetime.
self.mark_usable(verbose=False)
if as_text:
return json.dumps({
'owned': lists.owned,
'subscribed': lists.subscribed,
})
return lists
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(TwitterAccount, TwitterAccount.check_feeds,
globals(), queue=u'background')
register_task_method(TwitterAccount, TwitterAccount.check_lists,
globals(), queue=u'background')
register_task_method(TwitterAccount, TwitterAccount.test_connection,
globals(), queue=u'swarm', expire=3600)
register_task_method(TwitterAccount, TwitterAccount.update_lists,
globals(), queue=u'refresh', expire=3600)
@task
def check_social_user(social_user_id, **kwargs):
social_user = UserSocialAuth.objects.get(id=social_user_id)
if not social_user.provider == 'twitter':
return
if new_interval != self.fetch_interval:
LOGGER.info(
u'Fetch interval changed from %s to %s '
u'for feed %s (%s new article(s), %s '
u'duplicate(s)).', self.fetch_interval, new_interval, self,
new_items, duplicates)
self.fetch_interval = new_interval
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(
BaseFeed,
BaseFeed.refresh,
# No `expire` argument here, it's computed
# in the refresh_all_feeds() task dynamically.
globals(),
queue=u'refresh')
register_task_method(BaseFeed,
BaseFeed.update_all_items_count,
globals(),
queue=u'background')
register_task_method(BaseFeed,
BaseFeed.update_subscriptions_count,
globals(),
queue=u'background')
register_task_method(BaseFeed,
BaseFeed.update_recent_items_count,
globals(),
date_published=self.date_published
or getattr(related_to, 'date_published', None),
authors=[(a.name or a.origin_name)
for a in self.authors.all()],
date_updated=None,
language=self.language.dj_code
if self.language else None,
text_direction=self.text_direction,
tags=[t.name for t in self.tags.all()],
)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Tweet, Tweet.post_create_task,
globals(), queue=u'create')
register_task_method(Tweet, Tweet.fetch_entities,
globals(), queue=u'fetch')
# register_task_method(Tweet, Tweet.find_image,
# globals(), queue=u'fetch', default_retry_delay=3600)
@task(queue='background')
def mark_tweet_deleted(tweet_id):
try:
tweet = Tweet.objects.get(tweet_id=tweet_id)
except:
LOGGER.warning(u'Unknown tweet to delete: %s', tweet_id)
u', '.join(unicode(r) for r in recipients))
def resend(self, recipients):
""" Re-send an existing poke to new recipients.
This will add the recipients to the original poke without creating
a new one.
"""
self.send_to_recipients(recipients)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Poke, Poke.post_create_task,
globals(), queue=u'create')
# ————————————————————————————————————————————————————————————————————— Signals
def poke_pre_save(instance, **kwargs):
""" Make a slug if none. """
poke = instance
if not poke.slug:
poke.slug = slugify(poke.name)
def poke_post_save(instance, **kwargs):
date_published=self.date_published
or getattr(related_to, 'date_published', None),
authors=[(a.name or a.origin_name)
for a in self.authors.all()],
date_updated=None,
language=self.language.dj_code
if self.language else None,
text_direction=self.text_direction,
tags=[t.name for t in self.tags.all()],
)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Article, Article.post_create_task,
globals(), queue=u'create')
# register_task_method(Article, Article.find_image,
# globals(), queue=u'fetch', default_retry_delay=3600)
# ————————————————————————————————————————————————————————————————————— Signals
def article_pre_save(instance, **kwargs):
""" Make a slug if none. """
article = instance
if not article.slug:
article.slug = slugify(article.name)
u'email with UID %s (%s)',
self, msg_uid, data)
yield message
else:
# Without the cache, we fetch in
# chunks to lower network overhead.
result, data = imap_conn.uid('fetch',
','.join(emails_ids_chunk),
'(RFC822)')
if result == 'OK':
for raw_data in data:
if len(raw_data) == 1:
continue
yield raw_data[1]
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(MailAccount, MailAccount.test_connection,
globals(), queue=u'swarm', expire=3600)
register_task_method(MailAccount, MailAccount.update_mailboxes,
globals(), queue=u'refresh', expire=3600)
# They are registered just above.
MailAccount.usable_start_task = staticmethod(mailaccount_update_mailboxes_task) # NOQA
MailAccount.reset_unusable_task = staticmethod(mailaccount_test_connection_task) # NOQA
elif proposal.action == PROPOSAL_ACTION_TYPES.CHANGE:
# Change proposal.target in self.
# Note: proposal.target can be self.
retval = True
else:
LOGGER.warning(u'Unhandled action %s for %s on %s',
proposal.action, proposal.target, self)
return retval
# ——————————————————————————————————————————————————————————————— Tasks methods
register_task_method(Folder, Folder.purge, globals(), queue=u'background')
# ————————————————————————————————————————————————————————————————————— Signals
def folder_pre_save(instance, **kwargs):
if not instance.slug:
instance.slug = slugify(instance.name)
if instance.is_shared and 'is_shared' in instance.changed_fields:
if instance.is_root:
# For now, root folder cannot be
# shared. This seems a bad idea.
instance.is_shared = False
self.tags.add(*tags)
else:
LOGGER.warning(u'%s %s: %s original data is empty!',
self_name, self_id, self.origin)
self.original_data.twitter_processed = True
self.original_data.save()
BaseItem.add_original_data = \
BaseItem_add_original_data_method
BaseItem.remove_original_data = \
BaseItem_remove_original_data_method
BaseItem.postprocess_original_data = \
BaseItem_postprocess_original_data_method
BaseItem.postprocess_guess_original_data = \
BaseItem_postprocess_guess_original_data_method
BaseItem.postprocess_feedparser_data = \
BaseItem_postprocess_feedparser_data_method
BaseItem.postprocess_google_reader_data = \
BaseItem_postprocess_google_reader_data_method
BaseItem.postprocess_twitter_data = \
BaseItem_postprocess_twitter_data_method
# HEADS UP: we need to register against BaseItem, because OriginalData
# cannot .objects.get() an Article in register_task_method().
register_task_method(BaseItem, BaseItem.postprocess_original_data,
globals(), queue=u'background')
original.register_duplicate(self)
return False
# Any other exception will raise. This is intentional.
else:
LOGGER.info(u'URL of %s (#%s) successfully absolutized '
u'from %s to %s.', self._meta.model.__name__,
self.id, old_url, final_url)
else:
# Don't do the job twice.
if self.url_error:
statsd.gauge('articles.counts.url_errors', -1, delta=True)
statsd.gauge('articles.counts.absolutes', 1, delta=True)
self.url_absolute = True
self.url_error = None
# Don't waste a version just for that.
self.save_without_historical_record()
return True
# ———————————————————————————————————————————————————————————————— Celery Tasks
# HEADS UP: we need to register against BaseItem, because UrlItem is abstract
# and cannot run .objects.get() in register_task_method().
register_task_method(BaseItem, UrlItem.absolutize_url,
globals(), queue=u'swarm', default_retry_delay=3600)
'failed': self._import_failed_,
}
if self._import_created_['articles'] or self._import_created_['feeds']:
self.status = IMPORT_STATUS.FINISHED
elif self._import_failed_:
self.status = IMPORT_STATUS.FAILED
self.date_finished = now()
self.save()
# ———————————————————————————————————————————————————————————— Methods as tasks
register_task_method(UserImport, UserImport.run,
globals(), queue=u'background')
# ————————————————————————————————————————————————————————————————————— Signals
def userimport_post_save(instance, **kwargs):
if kwargs.get('created', False):
# HEADS UP: this task is declared by
# the register_task_method call below.
userimport_run_task.delay(instance.id) # NOQA
elif instance.status == IMPORT_STATUS.NEW:
# relaunch the importer task.
# HEADS UP: this task is declared by
# later in the global check.
LOGGER.exception(u'%s %s: read %s activation failed.',
self._meta.verbose_name, self.id, read.id)
else:
if verbose:
LOGGER.warning(u'%s %s: currently BAD, aborting reads '
u'activation.', self._meta.verbose_name,
self.id)
def to_json(self, related_to=None):
""" This method is abstract, subclasses must implement it.
But having it defined allows to send sentry messages, to
eventually remember to implement it in subclasses.
"""
LOGGER.error(u'%s.to_json() is not yet implemented.',
self._meta.model.__name__)
return None
# ——————————————————————————————————————————————————————————————————————— Tasks
register_task_method(BaseItem, BaseItem.process,
globals(), queue=u'fetch', default_retry_delay=3600)
register_task_method(BaseItem, BaseItem.create_reads,
globals(), queue=u'create')
# Update the last connection datetime.
self.mark_usable(verbose=False)
if as_text:
return json.dumps({
'owned': lists.owned,
'subscribed': lists.subscribed,
})
return lists
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(TwitterAccount,
TwitterAccount.check_feeds,
globals(),
queue=u'background')
register_task_method(TwitterAccount,
TwitterAccount.check_lists,
globals(),
queue=u'background')
register_task_method(TwitterAccount,
TwitterAccount.test_connection,
globals(),
queue=u'swarm',
expire=3600)
register_task_method(TwitterAccount,
TwitterAccount.update_lists,
globals(),
queue=u'refresh',
expire=3600)
if verbose:
LOGGER.warning(
u'%s %s: currently BAD, aborting reads '
u'activation.', self._meta.verbose_name, self.id)
def to_json(self, related_to=None):
""" This method is abstract, subclasses must implement it.
But having it defined allows to send sentry messages, to
eventually remember to implement it in subclasses.
"""
LOGGER.error(u'%s.to_json() is not yet implemented.',
self._meta.model.__name__)
return None
# ——————————————————————————————————————————————————————————————————————— Tasks
register_task_method(BaseItem,
BaseItem.process,
globals(),
queue=u'fetch',
default_retry_delay=3600)
register_task_method(BaseItem,
BaseItem.create_reads,
globals(),
queue=u'create')
LOGGER.info(u'Sent poke %s to %s.', self.id,
u', '.join(unicode(r) for r in recipients))
def resend(self, recipients):
""" Re-send an existing poke to new recipients.
This will add the recipients to the original poke without creating
a new one.
"""
self.send_to_recipients(recipients)
# ———————————————————————————————————————————————————————————————— Celery Tasks
register_task_method(Poke, Poke.post_create_task, globals(), queue=u'create')
# ————————————————————————————————————————————————————————————————————— Signals
def poke_pre_save(instance, **kwargs):
""" Make a slug if none. """
poke = instance
if not poke.slug:
poke.slug = slugify(poke.name)
def poke_post_save(instance, **kwargs):
# but geocodes (bounding circles)… We need to convert,
# and results won't be exactly the same.
# Geocode: 37.781157,-122.398720,1mi
self.__consume_items('search/tweets',
parameters,
backfilling=True)
finally:
self.check_new_good_period(period_start_item, self.oldest_id)
# ———————————————————————————————————————————————————————————————— Celery tasks
register_task_method(TwitterFeed,
TwitterFeed.backfill,
globals(),
queue=u'permanent')
register_task_method(TwitterFeed,
TwitterFeed.consume,
globals(),
queue=u'permanent')
# ————————————————————————————————————————————————————————————————————— Signals
#
# HEADS UP: see subscription.py for other signals.
#
def twitterfeed_pre_save(instance, **kwargs):
""" Update owner's subscription name when twitterfeed name changes. """
