def start(self):
while True:
try:
# dequeue a processed resource
message = self._mq_client.get_message(self._processed_resource_queue)
resource = self._mq_codec.decode(message.body, Resource)
except EmptyQueueException:
self._logger.info('Empty queue. Sleeping...')
time.sleep(self.EMPTY_QUEUE_TIMEOUT)
continue
resource._id = makeIdFromURI(resource.uri)
story = self._story_provider.provide_for_resource(resource)
resource.story_id = story._id
try:
# save and index the resource
self._resource_collection.insert_model(resource)
self._index_helper.index_resource(resource, self._index)
self._logger.debug('Resource %s: Inserted and indexed.' % resource._id)
# queue update
update = StoryUpdate()
update.story_id = story._id
update.type = STORY_UPDATE_RESOURCE_INSERTED
update.value = resource._id
self._update_collection.insert_model(update)
self._su_client.update(story._id)
except DuplicateKeyError:
self._logger.warning('Resource %s: Already inserted.' % resource._id)
# delete the queue message
self._mq_client.delete_message(self._processed_resource_queue, message.id)
def start(self):
self._resource_collection.ensure_index('published')
while True:
# find current time
sort = [('published', -1)]
last_resource = self._resource_collection.find_one_model(sort=sort)
if last_resource is None:
logging.debug('No resources found.')
time.sleep(self._search_timeout)
continue
self._current_time = last_resource.published
# check all open stories if they need to be closed
query = {'status': STORY_STATUS_OPEN}
count = 0
for story in self._story_collection.find_models(query):
if self._check_closing_condition(story):
# create an update
update = StoryUpdate()
update.story_id = story._id
update.type = STORY_UPDATE_STORY_CLOSED
update.value = self._current_time
# queue the update
self._update_collection.insert_model(update)
# apply the update
self._su_client.update(story._id)
logging.info('Closing story %s after %d seconds.' %
(story, self._current_time - story.created))
count += 1
logging.debug('Checked %d stories.' % count)
time.sleep(self._search_timeout)
