def gotResult(self, data, task, ttype): ''' 获取数据。任务分2种。 1. 商铺信息,需要抓取商铺的商品列表 2. 商品信息,需要抓取商品的基本信息 ''' # TODO refactor this if data: if ttype == 'extract': total_page, hrefs = json.loads(data) total_page = int(total_page) hrefs = json.loads(hrefs) tids = check_duplicate(self.redis, hrefs) #save_tasks(self.redis, tids) for h in hrefs: tmp_tid = self.new_task_id() log.info(h) tmp_tbody = {'task': h} tmp_task = BaseTask(tmp_tid, tmp_tbody) self.redis.push_list_data('task_queue', cPickle.dumps(tmp_task)) task = cPickle.loads(task) page = task.tbody.get('page', 1) if page == 1 and page < total_page: tmp_tbody = task.tbody for p in xrange(page, total_page): tmp_tid = self.new_task_id() tmp_tbody['page'] = p+1 tmp_task = BaseTask(tmp_tid, tmp_tbody) self.redis.push_list_data('extract_queue', cPickle.dumps(tmp_task)) else: save_items(json.loads(data)) else: log.debug('Got an invalid task: %s when taking task: %s' % (task, ttype))
def register( self, servicename, version_major, version_minor, nodename, client, ): ''' register the client to controller ''' clientid = self.newClientId() # 如果给定的servicename不在versions中,表示该节点是无效节点 if servicename not in self.versions: log.info("Added client: %s %s Failed. No such servicename" % (str(clientid), servicename)) return (servicename + ' is not in a known service', 0) version = self.versions[servicename] client_version = (version_major, version_minor) if client_version < version[0]: return ( 'version %s is below %s please update the client' % ( repr(client_version), repr(version[0]), ), 0, ) if client_version > version[1]: return ( 'version %s is above %s please update the controller' % ( repr(client_version), repr(version[1]), ), 0, ) self.clients[clientid] = { 'client': client, 'id': clientid, 'servicename': servicename, 'name': nodename, 'users': [], 'processing': {}, 'ip': client.broker.transport.getPeer().host, 'last_call': time.time(), } client.notifyOnDisconnect(lambda c: self.unregister(clientid)) log.info("Added client: %s %s" % (str(clientid), servicename)) return ('succeed', clientid)
def register( self, servicename, version_major, version_minor, nodename, client, ): ''' register the client to controller ''' clientid = self.newClientId() # 如果给定的servicename不在versions中,表示该节点是无效节点 if servicename not in self.versions: log.info("Added client: %s %s Failed. No such servicename" % (str(clientid), servicename)) return (servicename+' is not in a known service', 0) version = self.versions[servicename] client_version = (version_major, version_minor) if client_version < version[0]: return ( 'version %s is below %s please update the client' % ( repr(client_version), repr(version[0]), ), 0, ) if client_version > version[1]: return ( 'version %s is above %s please update the controller' % ( repr(client_version), repr(version[1]), ), 0, ) self.clients[clientid] = { 'client': client, 'id': clientid, 'servicename': servicename, 'name': nodename, 'users': [], 'processing': {}, 'ip': client.broker.transport.getPeer().host, 'last_call': time.time(), } client.notifyOnDisconnect(lambda c: self.unregister(clientid)) log.info("Added client: %s %s" % (str(clientid), servicename)) return ('succeed', clientid)
def clientPull(self, clientid): ''' ''' client = self.clients[clientid] push_queue = self.push_queue[name] # if there are no push_queue if not push_queue: # generate a new pull record and add into pull_queue defer = Deferred() pullid = self.newRequestId() pull = { 'id': pullid, 'servicename': name, 'defer': defer, 'clientid': clientid, } client['pulling'].add(pullid) self.pull_requests[pullid] = pull self.pull_queue[name].append(pullid) request = yield defer del self.pull_requests[pullid] requestid = request['id'] client['processing'].add(requestid) else: # get a request from push_queue and add into processing queue requestid = push_queue.popleft() client['processing'].add(requestid) request = self.push_requests[requestid] self.processing_timeout[requestid] = reactor.callLater( self.client_request_timeout, self.clientProcTimeout, requestid, clientid, ) log.info("Sent To: clientid %s, requestid %s." % ( clientid, request['id'], )) # return requestid, method, args, kwargs to client and # client run it. returnValue(( request['id'], request['method'], request['args'], request['kwargs'], ))
def clientPull(self, clientid): ''' ''' client = self.clients[clientid] push_queue = self.push_queue[name] # if there are no push_queue if not push_queue: # generate a new pull record and add into pull_queue defer = Deferred() pullid = self.newRequestId() pull = { 'id': pullid, 'servicename': name, 'defer': defer, 'clientid': clientid, } client['pulling'].add(pullid) self.pull_requests[pullid] = pull self.pull_queue[name].append(pullid) request = yield defer del self.pull_requests[pullid] requestid = request['id'] client['processing'].add(requestid) else: # get a request from push_queue and add into processing queue requestid = push_queue.popleft() client['processing'].add(requestid) request = self.push_requests[requestid] self.processing_timeout[requestid] = reactor.callLater( self.client_request_timeout, self.clientProcTimeout, requestid, clientid, ) log.info("Sent To: clientid %s, requestid %s." % ( clientid, request['id'], )) # return requestid, method, args, kwargs to client and # client run it. returnValue(( request['id'], request['method'], request['args'], request['kwargs'], ))
def clientReturn(self, clientid, requestid, result): ''' ''' log.info("Returned: clientid: %s, requestid: %s" % ( clientid, requestid, )) # remove this request from processing deque client = self.clients[clientid] client['processing'].discard(requestid) # try to cancel the processing request. # if occured an exception, that means the request # was already finishd. try: self.processing_timeout[requestid].cancel() del self.processing_timeout[requestid] except KeyError: # 已经处理完成 pass if requestid in self.push_requests: push = self.push_requests[requestid] if 'error' not in result: push['defer'].callback(result['result']) else: error = result['error'] push['defer'].errback( failure.Failure( pb.RemoteError( error['type'], error['value'], error['traceback'], ))) servicename = push['servicename'] # remove this request from push_queue try: self.push_queue[servicename].remove(requestid) except: pass if push['clientid'] is not None: try: self.clients[push['clientid']]['pushing'].discard( requestid) except: pass
def clientReturn(self, clientid, requestid, result): ''' ''' log.info("Returned: clientid: %s, requestid: %s" % ( clientid, requestid, )) # remove this request from processing deque client = self.clients[clientid] client['processing'].discard(requestid) # try to cancel the processing request. # if occured an exception, that means the request # was already finishd. try: self.processing_timeout[requestid].cancel() del self.processing_timeout[requestid] except KeyError: # 已经处理完成 pass if requestid in self.push_requests: push = self.push_requests[requestid] if 'error' not in result: push['defer'].callback(result['result']) else: error = result['error'] push['defer'].errback(failure.Failure( pb.RemoteError( error['type'], error['value'], error['traceback'], ))) servicename = push['servicename'] # remove this request from push_queue try: self.push_queue[servicename].remove(requestid) except: pass if push['clientid'] is not None: try: self.clients[push['clientid']]['pushing'].discard(requestid) except: pass
def gotResult(self, data, task, ttype): ''' 获取数据。任务分2种。 1. 商铺信息,需要抓取商铺的商品列表 2. 商品信息,需要抓取商品的基本信息 ''' # TODO refactor this if data: if ttype == 'extract': total_page, hrefs = json.loads(data) total_page = int(total_page) hrefs = json.loads(hrefs) tids = check_duplicate(self.redis, hrefs) #save_tasks(self.redis, tids) for h in hrefs: tmp_tid = self.new_task_id() log.info(h) tmp_tbody = {'task': h} tmp_task = BaseTask(tmp_tid, tmp_tbody) self.redis.push_list_data('task_queue', cPickle.dumps(tmp_task)) task = cPickle.loads(task) page = task.tbody.get('page', 1) if page == 1 and page < total_page: tmp_tbody = task.tbody for p in xrange(page, total_page): tmp_tid = self.new_task_id() tmp_tbody['page'] = p + 1 tmp_task = BaseTask(tmp_tid, tmp_tbody) self.redis.push_list_data('extract_queue', cPickle.dumps(tmp_task)) else: save_items(json.loads(data)) else: log.debug('Got an invalid task: %s when taking task: %s' % (task, ttype))
def unregister(self, clientid): ''' unregister the given clientid ''' if clientid not in self.clients: return False #client = self.clients[clientid] # 判断client中在处理的消息 # pulling, processing, pushing # for pull_id in client['pulling']: # servicename = self.pull_requests[pull_id]['servicename'] # try: # self.pull_queue[servicename].remove(pull_id) # except ValueError: # pass # del self.pull_requests[pull_id] # # processing set # if client['processing']: # for requestid in client['processing']: # servicename = self.push_requests[requestid]['servicename'] # self.addRequest(servicename, requestid) # # pushing set # for push_id in client['pushing']: # servicename = self.push_requests[push_id]['servicename'] # try: # self.push_queue[servicename].remove(push_id) # except ValueError: # pass # del self.push_requests[push_id] del self.clients[clientid] log.info("Removed client: " + str(clientid)) return True
def unregister(self, clientid): ''' unregister the given clientid ''' if clientid not in self.clients: return False #client = self.clients[clientid] # 判断client中在处理的消息 # pulling, processing, pushing # for pull_id in client['pulling']: # servicename = self.pull_requests[pull_id]['servicename'] # try: # self.pull_queue[servicename].remove(pull_id) # except ValueError: # pass # del self.pull_requests[pull_id] # # processing set # if client['processing']: # for requestid in client['processing']: # servicename = self.push_requests[requestid]['servicename'] # self.addRequest(servicename, requestid) # # pushing set # for push_id in client['pushing']: # servicename = self.push_requests[push_id]['servicename'] # try: # self.push_queue[servicename].remove(push_id) # except ValueError: # pass # del self.push_requests[push_id] del self.clients[clientid] log.info("Removed client: " + str(clientid)) return True
def searchLoop(self, agent): ''' ''' needbreak = False while 1: result = None if agent.remove: self.agent_pool.removeAgent(agent) break reqid, task = yield self.callController('nextRequest', 'extract') log.info(repr(task)) try: result = yield self.search(agent, task) log.debug('Got data %s' % repr(result)) except InfiniteLoginError: log.exception() yield self.callController("fail", task=task) needbreak = True except: log.exception() self.callController('sendResult', reqid, task, json.dumps(result)) if needbreak: break
def searchLoop(self, agent): ''' ''' needbreak = False while 1: result = None if agent.remove: self.agent_pool.removeAgent(agent) break reqid, task = yield self.callController('nextRequest', 'data') log.info('Got Task %s with reqid: %s' % (repr(task), reqid)) try: result = yield self.search(agent, task) log.debug('Got data %s' % repr(result)) except InfiniteLoginError: log.exception() yield self.callController("fail", task=task) needbreak = True except: log.exception() self.callController('sendResult', reqid, task, json.dumps(result)) if needbreak: break
def clientFail(self, *args, **kwargs): ''' called when client failed ''' clientid = kwargs.get('clientid') log.info("%s Client Failed, reason: %s" % (clientid, kwargs.get('reason', '')))
def clientFail(self, *args, **kwargs): ''' called when client failed ''' clientid = kwargs.get('clientid') log.info("%s Client Failed, reason: %s" % (clientid, kwargs.get('reason', '')))