def start(self): while True: try: # dequeue a processed resource message = self._mq_client.get_message(self._processed_resource_queue) resource = self._mq_codec.decode(message.body, Resource) except EmptyQueueException: self._logger.info('Empty queue. Sleeping...') time.sleep(self.EMPTY_QUEUE_TIMEOUT) continue resource._id = makeIdFromURI(resource.uri) story = self._story_provider.provide_for_resource(resource) resource.story_id = story._id try: # save and index the resource self._resource_collection.insert_model(resource) self._index_helper.index_resource(resource, self._index) self._logger.debug('Resource %s: Inserted and indexed.' % resource._id) # queue update update = StoryUpdate() update.story_id = story._id update.type = STORY_UPDATE_RESOURCE_INSERTED update.value = resource._id self._update_collection.insert_model(update) self._su_client.update(story._id) except DuplicateKeyError: self._logger.warning('Resource %s: Already inserted.' % resource._id) # delete the queue message self._mq_client.delete_message(self._processed_resource_queue, message.id)
def start(self): self._resource_collection.ensure_index('published') while True: # find current time sort = [('published', -1)] last_resource = self._resource_collection.find_one_model(sort=sort) if last_resource is None: logging.debug('No resources found.') time.sleep(self._search_timeout) continue self._current_time = last_resource.published # check all open stories if they need to be closed query = {'status': STORY_STATUS_OPEN} count = 0 for story in self._story_collection.find_models(query): if self._check_closing_condition(story): # create an update update = StoryUpdate() update.story_id = story._id update.type = STORY_UPDATE_STORY_CLOSED update.value = self._current_time # queue the update self._update_collection.insert_model(update) # apply the update self._su_client.update(story._id) logging.info('Closing story %s after %d seconds.' % (story, self._current_time - story.created)) count += 1 logging.debug('Checked %d stories.' % count) time.sleep(self._search_timeout)