def test_picklecompat():
obj = {'_encoding': 'utf-8',
'body': '',
'callback': '_response_downloaded',
'cookies': {},
'dont_filter': False,
'errback': None,
'headers': {'Referer': ['http://www.dmoz.org/']},
'meta': {'depth': 1, 'link_text': u'Fran\xe7ais', 'rule': 0},
'method': 'GET',
'priority': 0,
'url': u'http://www.dmoz.org/World/Fran%C3%A7ais/',
}
assert obj == picklecompat.loads(picklecompat.dumps(obj))
def add_task(server, json):
"""
:param json:str类型，表示要添加任务的详细信息
:return: list，表示添加的任务ids
"""
json = eval(json)
ids = []
for task in json.keys():
#电商meta中加入this_url_rule
this_meta = {}
if json[task]["type"] == 1:
this_url_rule = ''
if json[task].has_key("rules") and json[task]["rules"]:
for rule in json[task]["rules"].keys():
if re.compile(rule).match(task):
this_url_rule = rule
break
this_meta["this_url_rule"] = this_url_rule
json[task]["url"] = task
with dist_lock(BOT_NAME, server):
#获取当前任务id号
pipe = server.pipeline()
pipe.multi()
pipe.zcard('%s:running_task' % BOT_NAME)
pipe.zcard('%s:pausing_task' % BOT_NAME)
pipe.zcard('%s:done_canceled_task' % BOT_NAME)
id = sum(pipe.execute()) + 1
this_meta["id"] = id
#生成request
req = request_to_dict(
Request(task, meta=this_meta, priority=json[task]["priority"]))
print req
req = picklecompat.dumps(req)
#加入到对应task_X队列(zset)、task_information队列(hash)、running_task队列(zset)
server.execute_command('ZADD', '%s:task_%s' % (BOT_NAME, id),
-json[task]["priority"], req)
server.hset('%s:task_information' % BOT_NAME, id, json[task])
server.zadd('%s:running_task' % BOT_NAME, -json[task]["priority"],
id)
ids.append(id)
# time.sleep(3)
return ids