def main(): start = time.time() if len(sys.argv) < 2: print 'Please, specify a number of records to process and regular expression (optional)!' sys.exit(1) configs = ConfSpyPy.load('spypy.cfg') dataspypy = DataSpyPy(configs['host'], configs['port']) procspypy = ProcSpyPy(dataspypy, configs['google_analytics'], configs['google_adsense']) queue = Queue.Queue() for i in range(configs['threads']): dp = DocumentProcessor(queue, procspypy) dp.setDaemon(True) dp.start() records = dataspypy.get_unprocessed_records( int(sys.argv[1]), sys.argv[2] if len(sys.argv) == 3 else None) for record in records: queue.put(record) queue.join() print "Elapsed Time: %s" % (time.time() - start)
def main(): start = time.time() if len(sys.argv) < 2: print 'Please, specify a number of records to process and regular expression (optional)!' sys.exit(1) configs = ConfSpyPy.load('spypy.cfg') dataspypy = DataSpyPy(configs['host'], configs['port']) procspypy = ProcSpyPy(dataspypy, configs['google_analytics'], configs['google_adsense']) queue = Queue.Queue() for i in range(configs['threads']): dp = DocumentProcessor(queue, procspypy) dp.setDaemon(True) dp.start() records = dataspypy.get_unprocessed_records(int(sys.argv[1]), sys.argv[2] if len(sys.argv) == 3 else None) for record in records: queue.put(record) queue.join() print "Elapsed Time: %s" % (time.time() - start)
def main(): if len(sys.argv) < 2: print 'Please, specify a number of records to enqueue and regular expression (optional)!' sys.exit(1) configs = ConfSpyPy.load('spypy.cfg') connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) channel = connection.channel() channel.queue_declare(queue=configs['queue'], durable=False) dataspypy = DataSpyPy(configs['host'], configs['port']) records = dataspypy.get_unprocessed_records(int(sys.argv[1]), sys.argv[2] if len(sys.argv) == 3 else None) for record in records: print 'Enqueuing %s' % record['domain'] channel.basic_publish(exchange='', routing_key=configs['queue'], body=dumps(record)) connection.close()
def main(): if len(sys.argv) < 2: print 'Please, specify a number of records to enqueue and regular expression (optional)!' sys.exit(1) configs = ConfSpyPy.load('spypy.cfg') connection = pika.BlockingConnection( pika.ConnectionParameters(host='localhost')) channel = connection.channel() channel.queue_declare(queue=configs['queue'], durable=False) dataspypy = DataSpyPy(configs['host'], configs['port']) records = dataspypy.get_unprocessed_records( int(sys.argv[1]), sys.argv[2] if len(sys.argv) == 3 else None) for record in records: print 'Enqueuing %s' % record['domain'] channel.basic_publish(exchange='', routing_key=configs['queue'], body=dumps(record)) connection.close()