def __op_maps(self, data_loaded, connection): tool.send_msg(connection, pickle.dumps({ 'rule': self.__ie.map(), 'action': self.__ie.action_map() }, -1)) pass
def extract_modal_rule(request): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(config.socket_addr_extractor) except socket_error as serr: return render( request, 'app/error.html', { "page": { "title": "Error" }, "code": serr.errno, "content": str(serr.strerror) }) data_string = pickle.dumps({"operation": config.socket_CMD_extractor_maps}, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) maps = pickle.loads(data_string) return render( request, 'app/extract_modal_rule.html', { "page": { "title": "Extract" }, 'rule_maps': maps['rule'], 'action': maps['action'] })
def __op_rejudge_done(self, data_loaded, connection): item_id = int(data_loaded['id']) decision = int(data_loaded['decision']) item = UrlItem.load(id=item_id) del self.__ext_queue[item_id] self.__send_back_to_judge(item, decision) tool.send_msg(connection, "0") pass
def __op_maps(self, data_loaded, connection): tool.send_msg( connection, pickle.dumps( { 'rule': self.__ie.map(), 'action': self.__ie.action_map() }, -1)) pass
def re_judge(post,sock): data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_rejudge_done, "id": post.get("id"), "decision": post.get("decision") }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) return data_string
def __op_add_extractor(self, data_loaded, connection): item_id = int(data_loaded['id']) extractor = data_loaded['extractor'] item = UrlItem.load_with_content(item_id, file_path=config.path_extractor_inbox) self.__extract(item, extractor) del self.__ext_queue[item['id']] tool.send_msg(connection, "0") pass
def judge(post, sock): data_string = pickle.dumps( { "operation": config.socket_CMD_judge_done, "id": post.get("id"), "decision": post.get("decision") }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) return data_string
def __op_test_rule(self, data_loaded, connection): item_id = int(data_loaded['id']) rule = data_loaded['rule'] attrid = int(data_loaded['attrid']) item = UrlItem.load_with_content(id=item_id, file_path=config.path_extractor_inbox) tool.send_msg( connection, self.__ie.extract_attr(item, rule_id_or_dict=rule, attr_id=attrid)) pass
def __op_test_rule(self, data_loaded, connection): item_id = int(data_loaded['id']) rule = data_loaded['rule'] attrid = int(data_loaded['attrid']) item = UrlItem.load_with_content( id=item_id, file_path=config.path_extractor_inbox) tool.send_msg( connection, self.__ie.extract_attr(item, rule_id_or_dict=rule, attr_id=attrid) ) pass
def refreshextractor(request): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(config.socket_addr_extractor) except socket_error as serr: return '{"status": "error", "code": serr.errno, "msg": str(serr.strerror)}' req = { "operation": config.socket_CMD_extractor_refresh, } tool.send_msg(sock, pickle.dumps(req,-1)) return HttpResponse("Success")
def refreshextractor(request): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(config.socket_addr_extractor) except socket_error as serr: return '{"status": "error", "code": serr.errno, "msg": str(serr.strerror)}' req = { "operation": config.socket_CMD_extractor_refresh, } tool.send_msg(sock, pickle.dumps(req, -1)) return HttpResponse("Success")
def __op_add_rule(self, data_loaded, connection): rule = data_loaded['rule'] attrid = int(data_loaded['attrid']) new_id = self.__ie.add_rule(attrid, rule) tool.send_msg( connection, pickle.dumps({ 'rule': self.__ie.map(), 'action': self.__ie.action_map() }, -1) ) pass
def __op_add_rule(self, data_loaded, connection): rule = data_loaded['rule'] attrid = int(data_loaded['attrid']) new_id = self.__ie.add_rule(attrid, rule) tool.send_msg( connection, pickle.dumps( { 'rule': self.__ie.map(), 'action': self.__ie.action_map() }, -1)) pass
def extract(post,sock): selected = post.getlist('selected[]') selected = [int(x) for x in selected] data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_add_extract, "id": post.get("id"), "extractor": selected }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) return data_string
def extract(post, sock): selected = post.getlist('selected[]') selected = [int(x) for x in selected] data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_add_extract, "id": post.get("id"), "extractor": selected }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) return data_string
def extract(request): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(config.socket_addr_extractor) except socket_error as serr: return render(request, 'app/error.html', {"page": {"title": "Error"}, "code": serr.errno, "content": str(serr.strerror)}) data_string = pickle.dumps({"operation": config.socket_CMD_extractor_list}, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) ll = pickle.loads(data_string) return render(request, 'app/extract.html', {"page": {"title": "Extract"}, 'ext_list': ll})
def __op_add_extractor(self, data_loaded, connection): item_id = int(data_loaded['id']) extractor = data_loaded['extractor'] item = UrlItem.load_with_content( item_id, file_path=config.path_extractor_inbox) self.__extract(item,extractor) del self.__ext_queue[item['id']] tool.send_msg( connection, "0" ) pass
def preview(post,sock): extractor = [int(x) for x in post.getlist('extractor[]')] data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_preview, "id": post.get("id"), "extractor": extractor }, -1) log.info(data_string) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) preview = pickle.loads(data_string) return render(request, 'app/extract_modal_preview.html', {'preview': preview})
def preview(post, sock): extractor = [int(x) for x in post.getlist('extractor[]')] data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_preview, "id": post.get("id"), "extractor": extractor }, -1) log.info(data_string) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) preview = pickle.loads(data_string) return render(request, 'app/extract_modal_preview.html', {'preview': preview})
def rule_add(post,sock): d = rule_post2dict(post) selected = post.getlist('selected[]') selected = [int(x) for x in selected] data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_add_rule, "attrid": post.get("attrid"), "rule": d }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) maps = pickle.loads(data_string) return render(request, 'app/extract_modal_rule.html', {"page": {"title": "Extract"}, 'rule_maps': maps['rule'], 'action': maps['action'],'selected':selected})
def __op_preview(self, data_loaded, connection): log.info(data_loaded['extractor']) if data_loaded['extractor'] == config.const_RULE_UNKNOW: result = {x: "" for x in xrange(1, self.__ie.num_attr() + 1)} else: item_id = int(data_loaded['id']) item = UrlItem.load_with_content( item_id, file_path=config.path_extractor_inbox) extractor = data_loaded['extractor'] result = self.__ie.extract(item, extractor) preview = list() for att, str in result.iteritems(): preview.insert(att, dict(name=self.__ie.name(att), value=str)) log.info(preview) tool.send_msg(connection, pickle.dumps(preview, -1)) pass
def rule_test(post, sock): d = rule_post2dict(post) data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_test_rule, "id": post.get("id"), "attrid": post.get("attrid"), "rule": d }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) if data_string: return data_string else: return '#ERROR'
def rule_test(post,sock): d = rule_post2dict(post) data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_test_rule, "id": post.get("id"), "attrid": post.get("attrid"), "rule": d }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) if data_string: return data_string else: return '#ERROR'
def __op_done(self, data_loaded, connection): item_id = int(data_loaded['id']) decision = int(data_loaded['decision']) item = UrlItem.load(id=item_id) item['is_target'] = decision item.save() if int(item['is_target']) in [config.const_IS_TARGET_MULTIPLE, config.const_IS_TARGET_SIGNLE]: self.__send_to_extractor(item) else: os.remove(config.path_judge_inbox + "/%s" % item.filename()) self.__relearn_clf(self.__judge_queue[item_id]['feature'],decision) del self.__judge_queue[item_id] tool.send_msg(connection, "0") pass
def __op_done(self, data_loaded, connection): item_id = int(data_loaded['id']) decision = int(data_loaded['decision']) item = UrlItem.load(id=item_id) item['is_target'] = decision item.save() if int(item['is_target']) in [ config.const_IS_TARGET_MULTIPLE, config.const_IS_TARGET_SIGNLE ]: # item is target self.__send_to_extractor(item) else: # item is not target os.remove(config.path_judge_inbox + "/%s" % item.filename()) self.__relearn_clf(self.__judge_queue[item_id]['feature'], decision) del self.__judge_queue[item_id] tool.send_msg(connection, "0") pass
def rule_add(post, sock): d = rule_post2dict(post) selected = post.getlist('selected[]') selected = [int(x) for x in selected] data_string = pickle.dumps( { "operation": config.socket_CMD_extractor_add_rule, "attrid": post.get("attrid"), "rule": d }, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) maps = pickle.loads(data_string) return render( request, 'app/extract_modal_rule.html', { "page": { "title": "Extract" }, 'rule_maps': maps['rule'], 'action': maps['action'], 'selected': selected })
def judge(request): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(config.socket_addr_judge) except socket_error as serr: return render( request, 'app/error.html', { "page": { "title": "Error" }, "code": serr.errno, "content": str(serr.strerror) }) data_string = pickle.dumps({"operation": config.socket_CMD_judge_list}, -1) tool.send_msg(sock, data_string) data_string = tool.recv_msg(sock) ll = pickle.loads(data_string) return render(request, 'app/judge.html', { "page": { "title": "Judge" }, 'judge_list': ll })
def __op_list(self, data_loaded, connection): tool.send_msg(connection, pickle.dumps(self.__ext_queue, -1)) pass
def __op_list(self, data_loaded, connection): tool.send_msg(connection, pickle.dumps(self.__judge_queue, -1)) pass