result['article_title'], result['publication_date_raw'], result['article_body'], lang, result['doc_id']) except Exception as e: mongo_error.append(e) if mongo_error: logger.warning("There were error(s) in the Mongo loading {0}".format(mongo_error)) #except Exception as e: # logger.warning("Error downloading {0}: {1}".format(source, e)) # Set up connection to queue. # The queue contains source-days left to download. credentials = pika.PlainCredentials('guest', 'guest') parameters = pika.ConnectionParameters('localhost', 5672, '/', credentials) connection = pika.BlockingConnection(parameters) channel = connection.channel() channel.queue_declare(queue='cloacina_process2', durable=True) channel.confirm_delivery() # Handle the rate limit global doc_count doc_count = 0 print doc_count def download(ch, method, properties, body): source = json.loads(body) print(" [x] Received %r" % source) total = get_source_day_total(source["source_name"], source["date"]) total = int(total) print total