コード例 #1
0
ファイル: clearqueue.py プロジェクト: vinodsharma/Crawler
#!/usr/bin/env python
import pika
import os, signal, time
import sys
from controller import RabbitMQ
import json
quit = "quit"
visit = "visit"

wid = sys.argv[1]
rabbitMQServer = sys.argv[2]
rabbitMQQueue = sys.argv[3]
msg = RabbitMQ(rabbitMQServer,rabbitMQQueue)
#msg = RabbitMQ("localhost","newhello")

print " [worker",wid,"] Started"
#print sys.argv[1]
def callback(ch, method, properties, body):
	msg.ack(ch,method)
	#ch.basic_ack(delivery_tag = method.delivery_tag)
	#print " [x] Received %r" % (body,)
	print " [worker",wid,"] Received %r" % (body,)
	#message = body
	message = json.loads(body)
	if message.get("command") == quit:
		time.sleep(2);
		msg.stoprecv()
	elif message.get("command") == visit:
		print " [worker",wid,"] visiting: ", message.get("url")
	else:
		print " [worker",wid,"] Invalid command"
コード例 #2
0
ファイル: browser.py プロジェクト: vinodsharma/Crawler
    quit = "quit"
    visit = "visit"

    #wid = sys.argv[1]
    #rabbitMQServer = sys.argv[2]
    #rabbitMQQueue = sys.argv[3]
    logfile = open(logFileName+"_worker"+str(wid)+".log","w")
    logfile.write(" [worker"+str(wid)+"] Started\n")
    #print " [worker",wid,"] Started"
    
    browser = Browser(fastcrawl,int(maxurl))
    
    cdb = couchdb.Server(couchDbServer)
    try:
        db = cdb[couchDbName]
    except:
        db = cdb.create(couchDbName)
    msg = None
    msg = RabbitMQ(rabbitMQServer,rabbitMQQueue)
    #if fastcrawl == "no":
    #       msg = RabbitMQ(rabbitMQServer,rabbitMQQueue)
    #       msg.recv(callback)
    if fastcrawl == "yes":
        msg1 = RabbitMQ(rabbitMQServer,"crawl"+rabbitMQQueue)
        msg1.recv(fastcrawlHandler)

    msg.recv(callback)
#browser.visit('http://www.google.com')
#gtk.main()