class Executor(object):
def __init__(self, master_address, name, task):
self.master_address = master_address
self.name = name
self.master_ip, self.master_port = master_address.split(":")
self.rpc_proxy = SlaveRPC(self.master_ip, self.master_port)
self.task_info = task
def run(self):
raise NotImplementedError
def close(self):
self.rpc_proxy.close()
class Executor(object):
def __init__(self, master_address, name, task):
self.master_address = master_address
self.name = name
self.master_ip, self.master_port = master_address.split(":")
self.rpc_proxy = SlaveRPC(self.master_ip, self.master_port)
self.task_info = task
def run(self):
raise NotImplementedError
def close(self):
self.rpc_proxy.close()
def __init__(self, master_address, name, task):
self.master_address = master_address
self.name = name
self.master_ip, self.master_port = master_address.split(":")
self.rpc_proxy = SlaveRPC(self.master_ip, self.master_port)
self.task_info = task
def __init__(self, master_address, name, task):
self.master_address = master_address
self.name = name
self.master_ip, self.master_port = master_address.split(":")
self.rpc_proxy = SlaveRPC(self.master_ip, self.master_port)
self.task_info = task
if not options.master:
error("Wrong options: Master address required.")
else:
pattern = re.compile(
r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}:[0-9]{2,5}')
if not pattern.match(options.master):
error("Wrong address, your address must be such format: <ip:port>")
if not options.name:
error("Wrong options: Slave name required.")
if options.type not in SLAVE_TYPE:
error("Wrong type: your slave's type must be spider or parser.")
(master_host, master_port) = options.master.split(':')
""" Register to master """
slave = {"name": options.name, "type": options.type}
rpc_proxy = SlaveRPC(master_host, master_port)
if rpc_proxy.register(slave):
""" Monitor heartbeat """
# TODO debug monitor
server = Subscribe("task", "127.0.0.1")
try:
# while True:
# rpc_proxy.heartbeat(slave, interval=5)
for msg in server.ps.listen():
if msg['type'] == 'message':
print "Receive task: ", msg['data']
task = eval(msg['data'])
if (options.type == "spider"):
executor = SpiderExecutor(options.master, options.name,
task)
