def __op_maps(self, data_loaded, connection):
tool.send_msg(connection,
pickle.dumps({
'rule': self.__ie.map(),
'action': self.__ie.action_map()
}, -1))
pass
def extract_modal_rule(request):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(config.socket_addr_extractor)
except socket_error as serr:
return render(
request, 'app/error.html', {
"page": {
"title": "Error"
},
"code": serr.errno,
"content": str(serr.strerror)
})
data_string = pickle.dumps({"operation": config.socket_CMD_extractor_maps},
-1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
maps = pickle.loads(data_string)
return render(
request, 'app/extract_modal_rule.html', {
"page": {
"title": "Extract"
},
'rule_maps': maps['rule'],
'action': maps['action']
})
def __op_rejudge_done(self, data_loaded, connection):
item_id = int(data_loaded['id'])
decision = int(data_loaded['decision'])
item = UrlItem.load(id=item_id)
del self.__ext_queue[item_id]
self.__send_back_to_judge(item, decision)
tool.send_msg(connection, "0")
pass
def __op_rejudge_done(self, data_loaded, connection):
item_id = int(data_loaded['id'])
decision = int(data_loaded['decision'])
item = UrlItem.load(id=item_id)
del self.__ext_queue[item_id]
self.__send_back_to_judge(item, decision)
tool.send_msg(connection, "0")
pass
def __op_maps(self, data_loaded, connection):
tool.send_msg(
connection,
pickle.dumps(
{
'rule': self.__ie.map(),
'action': self.__ie.action_map()
}, -1))
pass
def re_judge(post,sock):
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_rejudge_done,
"id": post.get("id"),
"decision": post.get("decision")
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
return data_string
def __op_add_extractor(self, data_loaded, connection):
item_id = int(data_loaded['id'])
extractor = data_loaded['extractor']
item = UrlItem.load_with_content(item_id,
file_path=config.path_extractor_inbox)
self.__extract(item, extractor)
del self.__ext_queue[item['id']]
tool.send_msg(connection, "0")
pass
def judge(post, sock):
data_string = pickle.dumps(
{
"operation": config.socket_CMD_judge_done,
"id": post.get("id"),
"decision": post.get("decision")
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
return data_string
def __op_test_rule(self, data_loaded, connection):
item_id = int(data_loaded['id'])
rule = data_loaded['rule']
attrid = int(data_loaded['attrid'])
item = UrlItem.load_with_content(id=item_id,
file_path=config.path_extractor_inbox)
tool.send_msg(
connection,
self.__ie.extract_attr(item, rule_id_or_dict=rule, attr_id=attrid))
pass
def __op_test_rule(self, data_loaded, connection):
item_id = int(data_loaded['id'])
rule = data_loaded['rule']
attrid = int(data_loaded['attrid'])
item = UrlItem.load_with_content(
id=item_id, file_path=config.path_extractor_inbox)
tool.send_msg(
connection,
self.__ie.extract_attr(item, rule_id_or_dict=rule, attr_id=attrid)
)
pass
def refreshextractor(request):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(config.socket_addr_extractor)
except socket_error as serr:
return '{"status": "error", "code": serr.errno, "msg": str(serr.strerror)}'
req = {
"operation": config.socket_CMD_extractor_refresh,
}
tool.send_msg(sock, pickle.dumps(req,-1))
return HttpResponse("Success")
def refreshextractor(request):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(config.socket_addr_extractor)
except socket_error as serr:
return '{"status": "error", "code": serr.errno, "msg": str(serr.strerror)}'
req = {
"operation": config.socket_CMD_extractor_refresh,
}
tool.send_msg(sock, pickle.dumps(req, -1))
return HttpResponse("Success")
def __op_add_rule(self, data_loaded, connection):
rule = data_loaded['rule']
attrid = int(data_loaded['attrid'])
new_id = self.__ie.add_rule(attrid, rule)
tool.send_msg(
connection,
pickle.dumps({
'rule': self.__ie.map(),
'action': self.__ie.action_map()
}, -1)
)
pass
def __op_add_rule(self, data_loaded, connection):
rule = data_loaded['rule']
attrid = int(data_loaded['attrid'])
new_id = self.__ie.add_rule(attrid, rule)
tool.send_msg(
connection,
pickle.dumps(
{
'rule': self.__ie.map(),
'action': self.__ie.action_map()
}, -1))
pass
def extract(post,sock):
selected = post.getlist('selected[]')
selected = [int(x) for x in selected]
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_add_extract,
"id": post.get("id"),
"extractor": selected
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
return data_string
def extract(post, sock):
selected = post.getlist('selected[]')
selected = [int(x) for x in selected]
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_add_extract,
"id": post.get("id"),
"extractor": selected
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
return data_string
def extract(request):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(config.socket_addr_extractor)
except socket_error as serr:
return render(request, 'app/error.html',
{"page": {"title": "Error"}, "code": serr.errno, "content": str(serr.strerror)})
data_string = pickle.dumps({"operation": config.socket_CMD_extractor_list}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
ll = pickle.loads(data_string)
return render(request, 'app/extract.html', {"page": {"title": "Extract"}, 'ext_list': ll})
def __op_add_extractor(self, data_loaded, connection):
item_id = int(data_loaded['id'])
extractor = data_loaded['extractor']
item = UrlItem.load_with_content(
item_id, file_path=config.path_extractor_inbox)
self.__extract(item,extractor)
del self.__ext_queue[item['id']]
tool.send_msg(
connection,
"0"
)
pass
def preview(post,sock):
extractor = [int(x) for x in post.getlist('extractor[]')]
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_preview,
"id": post.get("id"),
"extractor": extractor
}, -1)
log.info(data_string)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
preview = pickle.loads(data_string)
return render(request, 'app/extract_modal_preview.html', {'preview': preview})
def preview(post, sock):
extractor = [int(x) for x in post.getlist('extractor[]')]
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_preview,
"id": post.get("id"),
"extractor": extractor
}, -1)
log.info(data_string)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
preview = pickle.loads(data_string)
return render(request, 'app/extract_modal_preview.html',
{'preview': preview})
def rule_add(post,sock):
d = rule_post2dict(post)
selected = post.getlist('selected[]')
selected = [int(x) for x in selected]
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_add_rule,
"attrid": post.get("attrid"),
"rule": d
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
maps = pickle.loads(data_string)
return render(request, 'app/extract_modal_rule.html',
{"page": {"title": "Extract"}, 'rule_maps': maps['rule'], 'action': maps['action'],'selected':selected})
def __op_preview(self, data_loaded, connection):
log.info(data_loaded['extractor'])
if data_loaded['extractor'] == config.const_RULE_UNKNOW:
result = {x: "" for x in xrange(1, self.__ie.num_attr() + 1)}
else:
item_id = int(data_loaded['id'])
item = UrlItem.load_with_content(
item_id, file_path=config.path_extractor_inbox)
extractor = data_loaded['extractor']
result = self.__ie.extract(item, extractor)
preview = list()
for att, str in result.iteritems():
preview.insert(att, dict(name=self.__ie.name(att), value=str))
log.info(preview)
tool.send_msg(connection, pickle.dumps(preview, -1))
pass
def rule_test(post, sock):
d = rule_post2dict(post)
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_test_rule,
"id": post.get("id"),
"attrid": post.get("attrid"),
"rule": d
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
if data_string:
return data_string
else:
return '#ERROR'
def rule_test(post,sock):
d = rule_post2dict(post)
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_test_rule,
"id": post.get("id"),
"attrid": post.get("attrid"),
"rule": d
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
if data_string:
return data_string
else:
return '#ERROR'
def __op_preview(self, data_loaded, connection):
log.info(data_loaded['extractor'])
if data_loaded['extractor'] == config.const_RULE_UNKNOW:
result = {x: "" for x in xrange(1, self.__ie.num_attr() + 1)}
else:
item_id = int(data_loaded['id'])
item = UrlItem.load_with_content(
item_id, file_path=config.path_extractor_inbox)
extractor = data_loaded['extractor']
result = self.__ie.extract(item, extractor)
preview = list()
for att, str in result.iteritems():
preview.insert(att, dict(name=self.__ie.name(att), value=str))
log.info(preview)
tool.send_msg(connection, pickle.dumps(preview, -1))
pass
def __op_done(self, data_loaded, connection):
item_id = int(data_loaded['id'])
decision = int(data_loaded['decision'])
item = UrlItem.load(id=item_id)
item['is_target'] = decision
item.save()
if int(item['is_target']) in [config.const_IS_TARGET_MULTIPLE, config.const_IS_TARGET_SIGNLE]:
self.__send_to_extractor(item)
else:
os.remove(config.path_judge_inbox + "/%s" % item.filename())
self.__relearn_clf(self.__judge_queue[item_id]['feature'],decision)
del self.__judge_queue[item_id]
tool.send_msg(connection, "0")
pass
def __op_done(self, data_loaded, connection):
item_id = int(data_loaded['id'])
decision = int(data_loaded['decision'])
item = UrlItem.load(id=item_id)
item['is_target'] = decision
item.save()
if int(item['is_target']) in [
config.const_IS_TARGET_MULTIPLE, config.const_IS_TARGET_SIGNLE
]:
# item is target
self.__send_to_extractor(item)
else:
# item is not target
os.remove(config.path_judge_inbox + "/%s" % item.filename())
self.__relearn_clf(self.__judge_queue[item_id]['feature'], decision)
del self.__judge_queue[item_id]
tool.send_msg(connection, "0")
pass
def rule_add(post, sock):
d = rule_post2dict(post)
selected = post.getlist('selected[]')
selected = [int(x) for x in selected]
data_string = pickle.dumps(
{
"operation": config.socket_CMD_extractor_add_rule,
"attrid": post.get("attrid"),
"rule": d
}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
maps = pickle.loads(data_string)
return render(
request, 'app/extract_modal_rule.html', {
"page": {
"title": "Extract"
},
'rule_maps': maps['rule'],
'action': maps['action'],
'selected': selected
})
def judge(request):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(config.socket_addr_judge)
except socket_error as serr:
return render(
request, 'app/error.html', {
"page": {
"title": "Error"
},
"code": serr.errno,
"content": str(serr.strerror)
})
data_string = pickle.dumps({"operation": config.socket_CMD_judge_list}, -1)
tool.send_msg(sock, data_string)
data_string = tool.recv_msg(sock)
ll = pickle.loads(data_string)
return render(request, 'app/judge.html', {
"page": {
"title": "Judge"
},
'judge_list': ll
})
def __op_list(self, data_loaded, connection):
tool.send_msg(connection, pickle.dumps(self.__ext_queue, -1))
pass
def __op_list(self, data_loaded, connection):
tool.send_msg(connection, pickle.dumps(self.__judge_queue, -1))
pass
