class Executor(object): def __init__(self, master_address, name, task): self.master_address = master_address self.name = name self.master_ip, self.master_port = master_address.split(":") self.rpc_proxy = SlaveRPC(self.master_ip, self.master_port) self.task_info = task def run(self): raise NotImplementedError def close(self): self.rpc_proxy.close()
def __init__(self, master_address, name, task): self.master_address = master_address self.name = name self.master_ip, self.master_port = master_address.split(":") self.rpc_proxy = SlaveRPC(self.master_ip, self.master_port) self.task_info = task
if not options.master: error("Wrong options: Master address required.") else: pattern = re.compile( r'[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}:[0-9]{2,5}') if not pattern.match(options.master): error("Wrong address, your address must be such format: <ip:port>") if not options.name: error("Wrong options: Slave name required.") if options.type not in SLAVE_TYPE: error("Wrong type: your slave's type must be spider or parser.") (master_host, master_port) = options.master.split(':') """ Register to master """ slave = {"name": options.name, "type": options.type} rpc_proxy = SlaveRPC(master_host, master_port) if rpc_proxy.register(slave): """ Monitor heartbeat """ # TODO debug monitor server = Subscribe("task", "127.0.0.1") try: # while True: # rpc_proxy.heartbeat(slave, interval=5) for msg in server.ps.listen(): if msg['type'] == 'message': print "Receive task: ", msg['data'] task = eval(msg['data']) if (options.type == "spider"): executor = SpiderExecutor(options.master, options.name, task)