def pop_items(self, data):
pdb.set_trace()
data = json.loads(data)
for i in ("batch", "node", "type", "key"):
if i not in data:
return {"errno": 1, "data": "args error: %s" % data}
batch = data["batch"]
node = data["node"]
type = data["type"]
key = data["key"]
r = REDIS_CONS.get(node)
if not r:
return {
"errno": 1,
"node": "%s not found" % node,
}
if type == "list":
items = spider.list_pop_n(r, key, batch)
elif type == "set":
items = spider.set_pop_n(r, key, batch)
else:
return {
"errno": 1,
"type": "%s is not allowed" % type,
}
b = []
for item in items:
b.append(msgpack.unpackb(item))
return {"errno": 0, "data": json.dumps(b)}
def pop_items(self, data):
pdb.set_trace()
data = json.loads(data)
for i in ("batch", "node", "type", "key"):
if i not in data:
return {
"errno": 1,
"data": "args error: %s" % data
}
batch = data["batch"]
node = data["node"]
type = data["type"]
key = data["key"]
r = REDIS_CONS.get(node)
if not r:
return {
"errno": 1,
"node": "%s not found" % node,
}
if type == "list":
items = spider.list_pop_n(r, key, batch)
elif type == "set":
items = spider.set_pop_n(r, key, batch)
else:
return {
"errno": 1,
"type": "%s is not allowed" % type,
}
b = []
for item in items:
b.append(msgpack.unpackb(item))
return {
"errno": 0,
"data": json.dumps(b)
}
def redis_nodes():
spider.CONFIG["site_id"] = 1111
nodes = {}
dst = {
"node": "default",
"name": "anyname",
"queue_pat": ("%s_%s", "site_id")
}
spider.eval_lua_scripts_for_node(nodes, dst)
assert "default" in nodes and dst["name"] == "anyname_1111"
node = nodes["default"]
spider.list_push_n(node, "list_push_n_test", *[1, 2, 3, 4])
assert node.llen("list_push_n_test") == 4
assert len(spider.list_pop_n(node, "list_push_n_test", 4)) == 4
spider.set_push_n(node, "set_push_n_test", *[1, 2, 3, 4])
assert node.scard("set_push_n_test") == 4
assert len(spider.set_pop_n(node, "set_push_n_test", 4)) == 4
dst = {
"node": "default",
"name": "anyname",
"queue_pat": ("%s_%s_suffix", "site_id")
}
spider.eval_lua_scripts_for_node(nodes, dst)
assert dst["name"] == "anyname_1111_suffix"
def redis_nodes():
spider.CONFIG["site_id"] = 1111
nodes = {}
dst = {
"node": "default",
"name": "anyname",
"queue_pat": ("%s_%s", "site_id")
}
spider.eval_lua_scripts_for_node(nodes, dst)
assert "default" in nodes and dst["name"] == "anyname_1111"
node = nodes["default"]
spider.list_push_n(node, "list_push_n_test", *[1,2,3,4])
assert node.llen("list_push_n_test") == 4
assert len(spider.list_pop_n(node, "list_push_n_test", 4)) == 4
spider.set_push_n(node, "set_push_n_test", *[1,2,3,4])
assert node.scard("set_push_n_test") == 4
assert len(spider.set_pop_n(node, "set_push_n_test", 4)) == 4
dst = {
"node": "default",
"name": "anyname",
"queue_pat": ("%s_%s_suffix", "site_id")
}
spider.eval_lua_scripts_for_node(nodes, dst)
assert dst["name"] == "anyname_1111_suffix"