result['article_title'], result['publication_date_raw'],
result['article_body'], lang, result['doc_id'])
except Exception as e:
mongo_error.append(e)
if mongo_error:
logger.warning("There were error(s) in the Mongo loading {0}".format(mongo_error))
#except Exception as e:
# logger.warning("Error downloading {0}: {1}".format(source, e))
# Set up connection to queue.
# The queue contains source-days left to download.
credentials = pika.PlainCredentials('guest', 'guest')
parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials)
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue='cloacina_process2', durable=True)
channel.confirm_delivery()
# Handle the rate limit
global doc_count
doc_count = 0
print doc_count
def download(ch, method, properties, body):
source = json.loads(body)
print(" [x] Received %r" % source)
total = get_source_day_total(source["source_name"], source["date"])
total = int(total)
print total
