result, data = imap_conn.uid('fetch', ','.join(emails_ids_chunk), '(RFC822)') if result == 'OK': for raw_data in data: if len(raw_data) == 1: continue yield raw_data[1] # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(MailAccount, MailAccount.test_connection, globals(), queue=u'swarm', expire=3600) register_task_method(MailAccount, MailAccount.update_mailboxes, globals(), queue=u'refresh', expire=3600) # They are registered just above. MailAccount.usable_start_task = staticmethod( mailaccount_update_mailboxes_task) # NOQA MailAccount.reset_unusable_task = staticmethod( mailaccount_test_connection_task) # NOQA
content=self.name, url=self.url, media=self.origin_to_json, date_published=self.date_published or getattr(related_to, 'date_published', None), authors=[(a.name or a.origin_name) for a in self.authors.all()], date_updated=None, language=self.language.dj_code if self.language else None, text_direction=self.text_direction, tags=[t.name for t in self.tags.all()], ) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Tweet, Tweet.post_create_task, globals(), queue=u'create') register_task_method(Tweet, Tweet.fetch_entities, globals(), queue=u'fetch') # register_task_method(Tweet, Tweet.find_image, # globals(), queue=u'fetch', default_retry_delay=3600) @task(queue='background') def mark_tweet_deleted(tweet_id): try: tweet = Tweet.objects.get(tweet_id=tweet_id) except: LOGGER.warning(u'Unknown tweet to delete: %s', tweet_id)
# Search/tweets doesn't use locations (bounding boxes), # but geocodes (bounding circles)… We need to convert, # and results won't be exactly the same. # Geocode: 37.781157,-122.398720,1mi self.__consume_items('search/tweets', parameters, backfilling=True) finally: self.check_new_good_period(period_start_item, self.oldest_id) # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(TwitterFeed, TwitterFeed.backfill, globals(), queue=u'permanent') register_task_method(TwitterFeed, TwitterFeed.consume, globals(), queue=u'permanent') # ————————————————————————————————————————————————————————————————————— Signals # # HEADS UP: see subscription.py for other signals. # def twitterfeed_pre_save(instance, **kwargs): """ Update owner's subscription name when twitterfeed name changes. """ twitterfeed = instance
elif proposal.action == PROPOSAL_ACTION_TYPES.CHANGE: # Change proposal.target in self. # Note: proposal.target can be self. retval = True else: LOGGER.warning(u'Unhandled action %s for %s on %s', proposal.action, proposal.target, self) return retval # ——————————————————————————————————————————————————————————————— Tasks methods register_task_method(Folder, Folder.purge, globals(), queue=u'background') # ————————————————————————————————————————————————————————————————————— Signals def folder_pre_save(instance, **kwargs): if not instance.slug: instance.slug = slugify(instance.name) if instance.is_shared and 'is_shared' in instance.changed_fields: if instance.is_root: # For now, root folder cannot be # shared. This seems a bad idea. instance.is_shared = False
else: LOGGER.info( u'URL of %s (#%s) successfully absolutized ' u'from %s to %s.', self._meta.model.__name__, self.id, old_url, final_url) else: # Don't do the job twice. if self.url_error: statsd.gauge('articles.counts.url_errors', -1, delta=True) statsd.gauge('articles.counts.absolutes', 1, delta=True) self.url_absolute = True self.url_error = None # Don't waste a version just for that. self.save_without_historical_record() return True # ———————————————————————————————————————————————————————————————— Celery Tasks # HEADS UP: we need to register against BaseItem, because UrlItem is abstract # and cannot run .objects.get() in register_task_method(). register_task_method(BaseItem, UrlItem.absolutize_url, globals(), queue=u'swarm', default_retry_delay=3600)
folder.compute_cached_descriptors(all=True, unread=True) LOGGER.info( u'Checked subscription #%s. ' u'%s/%s non-existing/re-checked, ' u'%s/%s read/unread and %s not created.', self.id, counters.missing, counters.rechecked, counters.reads, counters.unreads, counters.failed) return counters # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(Subscription, Subscription.mark_all_read_in_database, globals(), queue=u'background') register_task_method(Subscription, Subscription.check_reads, globals(), queue=u'check') # ————————————————————————————————————————————————————————————————————— Signals def subscription_pre_save(instance, **kwargs): """ Subscribe the mailfeed's owner if feed is beiing created. """ subscription = instance if not subscription.pk:
baseitem_create_reads_task.apply((self.id, )) baseitem_process_task.apply((self.id, )) else: post_create_reads_chain = tasks_chain( baseitem_process_task.si(self.id), ) baseitem_create_reads_task.apply_async( args=(self.id, ), kwargs={'stop_chain_on_false': True}, link=post_create_reads_chain) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Email, Email.post_create_task, globals(), queue=u'create') # ————————————————————————————————————————————————————————————————————— Signals def email_pre_save(instance, **kwargs): """ Make a slug if none. """ email = instance if not email.slug: email.slug = slugify(email.name) def email_post_save(instance, **kwargs):
else: post_create_reads_chain = tasks_chain( baseitem_process_task.si(self.id), ) baseitem_create_reads_task.apply_async( args=(self.id, ), kwargs={'stop_chain_on_false': True}, link=post_create_reads_chain ) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Email, Email.post_create_task, globals(), queue=u'create') # ————————————————————————————————————————————————————————————————————— Signals def email_pre_save(instance, **kwargs): """ Make a slug if none. """ email = instance if not email.slug: email.slug = slugify(email.name) def email_post_save(instance, **kwargs):
# Update the last connection datetime. self.mark_usable(verbose=False) if as_text: return json.dumps({ 'owned': lists.owned, 'subscribed': lists.subscribed, }) return lists # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(TwitterAccount, TwitterAccount.check_feeds, globals(), queue=u'background') register_task_method(TwitterAccount, TwitterAccount.check_lists, globals(), queue=u'background') register_task_method(TwitterAccount, TwitterAccount.test_connection, globals(), queue=u'swarm', expire=3600) register_task_method(TwitterAccount, TwitterAccount.update_lists, globals(), queue=u'refresh', expire=3600) @task def check_social_user(social_user_id, **kwargs): social_user = UserSocialAuth.objects.get(id=social_user_id) if not social_user.provider == 'twitter': return
if new_interval != self.fetch_interval: LOGGER.info( u'Fetch interval changed from %s to %s ' u'for feed %s (%s new article(s), %s ' u'duplicate(s)).', self.fetch_interval, new_interval, self, new_items, duplicates) self.fetch_interval = new_interval # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method( BaseFeed, BaseFeed.refresh, # No `expire` argument here, it's computed # in the refresh_all_feeds() task dynamically. globals(), queue=u'refresh') register_task_method(BaseFeed, BaseFeed.update_all_items_count, globals(), queue=u'background') register_task_method(BaseFeed, BaseFeed.update_subscriptions_count, globals(), queue=u'background') register_task_method(BaseFeed, BaseFeed.update_recent_items_count, globals(),
date_published=self.date_published or getattr(related_to, 'date_published', None), authors=[(a.name or a.origin_name) for a in self.authors.all()], date_updated=None, language=self.language.dj_code if self.language else None, text_direction=self.text_direction, tags=[t.name for t in self.tags.all()], ) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Tweet, Tweet.post_create_task, globals(), queue=u'create') register_task_method(Tweet, Tweet.fetch_entities, globals(), queue=u'fetch') # register_task_method(Tweet, Tweet.find_image, # globals(), queue=u'fetch', default_retry_delay=3600) @task(queue='background') def mark_tweet_deleted(tweet_id): try: tweet = Tweet.objects.get(tweet_id=tweet_id) except: LOGGER.warning(u'Unknown tweet to delete: %s', tweet_id)
u', '.join(unicode(r) for r in recipients)) def resend(self, recipients): """ Re-send an existing poke to new recipients. This will add the recipients to the original poke without creating a new one. """ self.send_to_recipients(recipients) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Poke, Poke.post_create_task, globals(), queue=u'create') # ————————————————————————————————————————————————————————————————————— Signals def poke_pre_save(instance, **kwargs): """ Make a slug if none. """ poke = instance if not poke.slug: poke.slug = slugify(poke.name) def poke_post_save(instance, **kwargs):
date_published=self.date_published or getattr(related_to, 'date_published', None), authors=[(a.name or a.origin_name) for a in self.authors.all()], date_updated=None, language=self.language.dj_code if self.language else None, text_direction=self.text_direction, tags=[t.name for t in self.tags.all()], ) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Article, Article.post_create_task, globals(), queue=u'create') # register_task_method(Article, Article.find_image, # globals(), queue=u'fetch', default_retry_delay=3600) # ————————————————————————————————————————————————————————————————————— Signals def article_pre_save(instance, **kwargs): """ Make a slug if none. """ article = instance if not article.slug: article.slug = slugify(article.name)
u'email with UID %s (%s)', self, msg_uid, data) yield message else: # Without the cache, we fetch in # chunks to lower network overhead. result, data = imap_conn.uid('fetch', ','.join(emails_ids_chunk), '(RFC822)') if result == 'OK': for raw_data in data: if len(raw_data) == 1: continue yield raw_data[1] # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(MailAccount, MailAccount.test_connection, globals(), queue=u'swarm', expire=3600) register_task_method(MailAccount, MailAccount.update_mailboxes, globals(), queue=u'refresh', expire=3600) # They are registered just above. MailAccount.usable_start_task = staticmethod(mailaccount_update_mailboxes_task) # NOQA MailAccount.reset_unusable_task = staticmethod(mailaccount_test_connection_task) # NOQA
elif proposal.action == PROPOSAL_ACTION_TYPES.CHANGE: # Change proposal.target in self. # Note: proposal.target can be self. retval = True else: LOGGER.warning(u'Unhandled action %s for %s on %s', proposal.action, proposal.target, self) return retval # ——————————————————————————————————————————————————————————————— Tasks methods register_task_method(Folder, Folder.purge, globals(), queue=u'background') # ————————————————————————————————————————————————————————————————————— Signals def folder_pre_save(instance, **kwargs): if not instance.slug: instance.slug = slugify(instance.name) if instance.is_shared and 'is_shared' in instance.changed_fields: if instance.is_root: # For now, root folder cannot be # shared. This seems a bad idea. instance.is_shared = False
self.tags.add(*tags) else: LOGGER.warning(u'%s %s: %s original data is empty!', self_name, self_id, self.origin) self.original_data.twitter_processed = True self.original_data.save() BaseItem.add_original_data = \ BaseItem_add_original_data_method BaseItem.remove_original_data = \ BaseItem_remove_original_data_method BaseItem.postprocess_original_data = \ BaseItem_postprocess_original_data_method BaseItem.postprocess_guess_original_data = \ BaseItem_postprocess_guess_original_data_method BaseItem.postprocess_feedparser_data = \ BaseItem_postprocess_feedparser_data_method BaseItem.postprocess_google_reader_data = \ BaseItem_postprocess_google_reader_data_method BaseItem.postprocess_twitter_data = \ BaseItem_postprocess_twitter_data_method # HEADS UP: we need to register against BaseItem, because OriginalData # cannot .objects.get() an Article in register_task_method(). register_task_method(BaseItem, BaseItem.postprocess_original_data, globals(), queue=u'background')
original.register_duplicate(self) return False # Any other exception will raise. This is intentional. else: LOGGER.info(u'URL of %s (#%s) successfully absolutized ' u'from %s to %s.', self._meta.model.__name__, self.id, old_url, final_url) else: # Don't do the job twice. if self.url_error: statsd.gauge('articles.counts.url_errors', -1, delta=True) statsd.gauge('articles.counts.absolutes', 1, delta=True) self.url_absolute = True self.url_error = None # Don't waste a version just for that. self.save_without_historical_record() return True # ———————————————————————————————————————————————————————————————— Celery Tasks # HEADS UP: we need to register against BaseItem, because UrlItem is abstract # and cannot run .objects.get() in register_task_method(). register_task_method(BaseItem, UrlItem.absolutize_url, globals(), queue=u'swarm', default_retry_delay=3600)
'failed': self._import_failed_, } if self._import_created_['articles'] or self._import_created_['feeds']: self.status = IMPORT_STATUS.FINISHED elif self._import_failed_: self.status = IMPORT_STATUS.FAILED self.date_finished = now() self.save() # ———————————————————————————————————————————————————————————— Methods as tasks register_task_method(UserImport, UserImport.run, globals(), queue=u'background') # ————————————————————————————————————————————————————————————————————— Signals def userimport_post_save(instance, **kwargs): if kwargs.get('created', False): # HEADS UP: this task is declared by # the register_task_method call below. userimport_run_task.delay(instance.id) # NOQA elif instance.status == IMPORT_STATUS.NEW: # relaunch the importer task. # HEADS UP: this task is declared by
# later in the global check. LOGGER.exception(u'%s %s: read %s activation failed.', self._meta.verbose_name, self.id, read.id) else: if verbose: LOGGER.warning(u'%s %s: currently BAD, aborting reads ' u'activation.', self._meta.verbose_name, self.id) def to_json(self, related_to=None): """ This method is abstract, subclasses must implement it. But having it defined allows to send sentry messages, to eventually remember to implement it in subclasses. """ LOGGER.error(u'%s.to_json() is not yet implemented.', self._meta.model.__name__) return None # ——————————————————————————————————————————————————————————————————————— Tasks register_task_method(BaseItem, BaseItem.process, globals(), queue=u'fetch', default_retry_delay=3600) register_task_method(BaseItem, BaseItem.create_reads, globals(), queue=u'create')
# Update the last connection datetime. self.mark_usable(verbose=False) if as_text: return json.dumps({ 'owned': lists.owned, 'subscribed': lists.subscribed, }) return lists # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(TwitterAccount, TwitterAccount.check_feeds, globals(), queue=u'background') register_task_method(TwitterAccount, TwitterAccount.check_lists, globals(), queue=u'background') register_task_method(TwitterAccount, TwitterAccount.test_connection, globals(), queue=u'swarm', expire=3600) register_task_method(TwitterAccount, TwitterAccount.update_lists, globals(), queue=u'refresh', expire=3600)
if verbose: LOGGER.warning( u'%s %s: currently BAD, aborting reads ' u'activation.', self._meta.verbose_name, self.id) def to_json(self, related_to=None): """ This method is abstract, subclasses must implement it. But having it defined allows to send sentry messages, to eventually remember to implement it in subclasses. """ LOGGER.error(u'%s.to_json() is not yet implemented.', self._meta.model.__name__) return None # ——————————————————————————————————————————————————————————————————————— Tasks register_task_method(BaseItem, BaseItem.process, globals(), queue=u'fetch', default_retry_delay=3600) register_task_method(BaseItem, BaseItem.create_reads, globals(), queue=u'create')
LOGGER.info(u'Sent poke %s to %s.', self.id, u', '.join(unicode(r) for r in recipients)) def resend(self, recipients): """ Re-send an existing poke to new recipients. This will add the recipients to the original poke without creating a new one. """ self.send_to_recipients(recipients) # ———————————————————————————————————————————————————————————————— Celery Tasks register_task_method(Poke, Poke.post_create_task, globals(), queue=u'create') # ————————————————————————————————————————————————————————————————————— Signals def poke_pre_save(instance, **kwargs): """ Make a slug if none. """ poke = instance if not poke.slug: poke.slug = slugify(poke.name) def poke_post_save(instance, **kwargs):
# but geocodes (bounding circles)… We need to convert, # and results won't be exactly the same. # Geocode: 37.781157,-122.398720,1mi self.__consume_items('search/tweets', parameters, backfilling=True) finally: self.check_new_good_period(period_start_item, self.oldest_id) # ———————————————————————————————————————————————————————————————— Celery tasks register_task_method(TwitterFeed, TwitterFeed.backfill, globals(), queue=u'permanent') register_task_method(TwitterFeed, TwitterFeed.consume, globals(), queue=u'permanent') # ————————————————————————————————————————————————————————————————————— Signals # # HEADS UP: see subscription.py for other signals. # def twitterfeed_pre_save(instance, **kwargs): """ Update owner's subscription name when twitterfeed name changes. """