def read_list_from_quicklist(self, f):
count = self.read_length(f)
total_size = 0
self._callback.start_list(self._key,
self._expiry,
info={
'encoding': 'quicklist',
'zips': count,
'idle': self._idle,
'freq': self._freq
})
for i in range(0, count):
raw_string = self.read_string(f)
total_size += len(raw_string)
buff = BytesIO(raw_string)
zlbytes = read_unsigned_int(buff)
tail_offset = read_unsigned_int(buff)
num_entries = read_unsigned_short(buff)
for x in range(0, num_entries):
self._callback.rpush(self._key, self.read_ziplist_entry(buff))
zlist_end = read_unsigned_char(buff)
if zlist_end != 255:
raise Exception(
'read_quicklist', "Invalid zip list end - %d for key %s" %
(zlist_end, self._key))
self._callback.end_list(self._key,
info={
'encoding': 'quicklist',
'zips': count,
'sizeof_value': total_size
})
def read_stream(self, f):
listpacks = self.read_length(f)
self._callback.start_stream(self._key,
listpacks,
self._expiry,
info={
'encoding': 'listpack',
'idle': self._idle,
'freq': self._freq
})
for _lp in range(listpacks):
self._callback.stream_listpack(self._key, self.read_string(f),
self.read_string(f))
items = self.read_length(f)
last_entry_id = "%s-%s" % (self.read_length(f), self.read_length(f))
cgroups = self.read_length(f)
cgroups_data = []
for _cg in range(cgroups):
cgname = self.read_string(f)
last_cg_entry_id = "%s-%s" % (self.read_length(f),
self.read_length(f))
pending = self.read_length(f)
group_pending_entries = []
for _pel in range(pending):
eid = f.read(16)
delivery_time = read_milliseconds_time(f)
delivery_count = self.read_length(f)
group_pending_entries.append({
'id': eid,
'delivery_time': delivery_time,
'delivery_count': delivery_count
})
consumers = self.read_length(f)
consumers_data = []
for _c in range(consumers):
cname = self.read_string(f)
seen_time = read_milliseconds_time(f)
pending = self.read_length(f)
consumer_pending_entries = []
for _pel in range(pending):
eid = f.read(16)
consumer_pending_entries.append({'id': eid})
consumers_data.append({
'name': cname,
'seen_time': seen_time,
'pending': consumer_pending_entries
})
cgroups_data.append({
'name': cgname,
'last_entry_id': last_cg_entry_id,
'pending': group_pending_entries,
'consumers': consumers_data
})
self._callback.end_stream(self._key, items, last_entry_id,
cgroups_data)
def _getDOMElements(item, name, attrs):
lst = []
for key in attrs:
lst2 = re.compile(
'(<' + name + '[^>]*?(?:' + key + '=[\'"]' + attrs[key] +
'[\'"].*?>))', re.M | re.S).findall(item)
if len(lst2) == 0 and attrs[key].find(
" ") == -1: # Try matching without quotation marks
lst2 = re.compile(
'(<' + name + '[^>]*?(?:' + key + '=' + attrs[key] + '.*?>))',
re.M | re.S).findall(item)
if len(lst) == 0:
# print("Setting main list " + repr(lst2))
lst = lst2
lst2 = []
else:
# print("Setting new list " + repr(lst2))
test = list(range(len(lst)))
test.reverse()
for i in test: # Delete anything missing from the next list.
if not lst[i] in lst2:
# print("Purging mismatch " + str(len(lst)) + " - " + repr(lst[i]))
del (lst[i])
if len(lst) == 0 and attrs == {}:
# print("No list found, trying to match on name only")
lst = re.compile('(<' + name + '>)', re.M | re.S).findall(item)
if len(lst) == 0:
lst = re.compile('(<' + name + ' .*?>)', re.M | re.S).findall(item)
# print("Done: " + str(type(lst)))
return lst
def read_hash_from_ziplist(self, f):
raw_string = self.read_string(f)
buff = BytesIO(raw_string)
zlbytes = read_unsigned_int(buff)
tail_offset = read_unsigned_int(buff)
num_entries = read_unsigned_short(buff)
if (num_entries % 2):
raise Exception(
'read_hash_from_ziplist',
"Expected even number of elements, but found %d for key %s" %
(num_entries, self._key))
num_entries = num_entries // 2
self._callback.start_hash(self._key,
num_entries,
self._expiry,
info={
'encoding': 'ziplist',
'sizeof_value': len(raw_string),
'idle': self._idle,
'freq': self._freq
})
for x in range(0, num_entries):
field = self.read_ziplist_entry(buff)
value = self.read_ziplist_entry(buff)
self._callback.hset(self._key, field, value)
zlist_end = read_unsigned_char(buff)
if zlist_end != 255:
raise Exception(
'read_hash_from_ziplist',
"Invalid zip list end - %d for key %s" %
(zlist_end, self._key))
self._callback.end_hash(self._key)
def read_intset(self, f):
raw_string = self.read_string(f)
buff = BytesIO(raw_string)
encoding = read_unsigned_int(buff)
num_entries = read_unsigned_int(buff)
self._callback.start_set(self._key,
num_entries,
self._expiry,
info={
'encoding': 'intset',
'sizeof_value': len(raw_string),
'idle': self._idle,
'freq': self._freq
})
for x in range(0, num_entries):
if encoding == 8:
entry = read_signed_long(buff)
elif encoding == 4:
entry = read_signed_int(buff)
elif encoding == 2:
entry = read_signed_short(buff)
else:
raise Exception(
'read_intset',
'Invalid encoding %d for key %s' % (encoding, self._key))
self._callback.sadd(self._key, entry)
self._callback.end_set(self._key)
def _parse_duration_element(durationstr, elementstr):
#Extracts the specified portion of a duration, for instance, given:
#durationstr = 'T4H5M6.1234S'
#elementstr = 'H'
#
#returns 4
#
#Note that the string must start with a character, so its assumed the
#full duration string would be split at the 'T'
durationstartindex = 0
durationendindex = durationstr.find(elementstr)
for characterindex in compat.range(durationendindex - 1, 0, -1):
if durationstr[characterindex].isalpha() == True:
durationstartindex = characterindex
break
durationstartindex += 1
if ',' in durationstr:
#Replace the comma with a 'full-stop'
durationstr = durationstr.replace(',', '.')
return float(durationstr[durationstartindex:durationendindex])
def skip_object(self, f, enc_type):
skip_strings = 0
if enc_type == REDIS_RDB_TYPE_STRING:
skip_strings = 1
elif enc_type == REDIS_RDB_TYPE_LIST:
skip_strings = self.read_length(f)
elif enc_type == REDIS_RDB_TYPE_SET:
skip_strings = self.read_length(f)
elif enc_type == REDIS_RDB_TYPE_ZSET or enc_type == REDIS_RDB_TYPE_ZSET_2:
length = self.read_length(f)
for x in range(length):
self.skip_string(f)
self.skip_binary_double(
f
) if enc_type == REDIS_RDB_TYPE_ZSET_2 else self.skip_float(f)
elif enc_type == REDIS_RDB_TYPE_HASH:
skip_strings = self.read_length(f) * 2
elif enc_type == REDIS_RDB_TYPE_HASH_ZIPMAP:
skip_strings = 1
elif enc_type == REDIS_RDB_TYPE_LIST_ZIPLIST:
skip_strings = 1
elif enc_type == REDIS_RDB_TYPE_SET_INTSET:
skip_strings = 1
elif enc_type == REDIS_RDB_TYPE_ZSET_ZIPLIST:
skip_strings = 1
elif enc_type == REDIS_RDB_TYPE_HASH_ZIPLIST:
skip_strings = 1
elif enc_type == REDIS_RDB_TYPE_LIST_QUICKLIST:
skip_strings = self.read_length(f)
elif enc_type == REDIS_RDB_TYPE_MODULE:
raise Exception(
'skip_object',
'Unable to skip Redis Modules RDB objects (key %s)' %
self._key)
elif enc_type == REDIS_RDB_TYPE_MODULE_2:
self.skip_module(f)
elif enc_type == REDIS_RDB_TYPE_STREAM_LISTPACKS:
self.skip_stream(f)
else:
raise Exception(
'skip_object',
'Invalid object type %d for key %s' % (enc_type, self._key))
for x in range(0, skip_strings):
self.skip_string(f)
def pil(image):
"""Convert the PIL.Image object to the pixels buffer
Parameters::
image : PIL.Image object
"""
if image.mode not in ('1', 'L'):
image = image.convert('L')
width, height = image.size
pixels = image.load()
width_iter = range(width)
height_iter = range(height)
for i in width_iter:
for j in height_iter:
yield ctypes.c_uint8(pixels[j, i])
def lzf_decompress(self, compressed, expected_length):
if HAS_PYTHON_LZF:
return lzf.decompress(compressed, expected_length)
else:
in_stream = bytearray(compressed)
in_len = len(in_stream)
in_index = 0
out_stream = bytearray()
out_index = 0
while in_index < in_len:
ctrl = in_stream[in_index]
if not isinstance(ctrl, int):
raise Exception(
'lzf_decompress',
'ctrl should be a number %s for key %s' %
(str(ctrl), self._key))
in_index = in_index + 1
if ctrl < 32:
for x in range(0, ctrl + 1):
out_stream.append(in_stream[in_index])
# sys.stdout.write(chr(in_stream[in_index]))
in_index = in_index + 1
out_index = out_index + 1
else:
length = ctrl >> 5
if length == 7:
length = length + in_stream[in_index]
in_index = in_index + 1
ref = out_index - (
(ctrl & 0x1f) << 8) - in_stream[in_index] - 1
in_index = in_index + 1
for x in range(0, length + 2):
out_stream.append(out_stream[ref])
ref = ref + 1
out_index = out_index + 1
if len(out_stream) != expected_length:
raise Exception(
'lzf_decompress',
'Expected lengths do not match %d != %d for key %s' %
(len(out_stream), expected_length, self._key))
return bytes(out_stream)
def pil(image):
"""Convert the PIL.Image object to the pixels buffer
Parameters::
image : PIL.Image object
"""
if image.mode not in ('1', 'L'):
image = image.convert('L')
width, height = image.size
pixels = image.load()
width_iter = range(width)
height_iter = range(height)
for i in width_iter:
for j in height_iter:
yield ctypes.c_uint8(pixels[j, i])
def _decode_module_id(self, module_id):
"""
decode module id to string
based on @antirez moduleTypeNameByID function from redis/src/module.c
:param module_id: 64bit integer
:return: string
"""
name = [''] * 9
module_id >>= 10
for i in reversed(range(9)):
name[i] = self.charset[module_id & 63]
module_id >>= 6
return ''.join(name)
def skip_stream(self, f):
listpacks = self.read_length(f)
for _lp in range(listpacks):
self.skip_string(f)
self.skip_string(f)
self.read_length(f)
self.read_length(f)
self.read_length(f)
cgroups = self.read_length(f)
for _cg in range(cgroups):
self.skip_string(f)
self.read_length(f)
self.read_length(f)
pending = self.read_length(f)
for _pel in range(pending):
f.read(16)
f.read(8)
self.read_length(f)
consumers = self.read_length(f)
for _c in range(consumers):
self.skip_string(f)
f.read(8)
pending = self.read_length(f)
f.read(pending * 16)
def read_ziplist(self, f):
raw_string = self.read_string(f)
buff = BytesIO(raw_string)
zlbytes = read_unsigned_int(buff)
tail_offset = read_unsigned_int(buff)
num_entries = read_unsigned_short(buff)
self._callback.start_list(self._key,
self._expiry,
info={
'encoding': 'ziplist',
'sizeof_value': len(raw_string),
'idle': self._idle,
'freq': self._freq
})
for x in range(0, num_entries):
val = self.read_ziplist_entry(buff)
self._callback.rpush(self._key, val)
zlist_end = read_unsigned_char(buff)
if zlist_end != 255:
raise Exception(
'read_ziplist', "Invalid zip list end - %d for key %s" %
(zlist_end, self._key))
self._callback.end_list(self._key, info={'encoding': 'ziplist'})
def randomagent():
BR_VERS = [
['%s.0' % i for i in range(18, 50)],
[
'37.0.2062.103', '37.0.2062.120', '37.0.2062.124', '38.0.2125.101',
'38.0.2125.104', '38.0.2125.111', '39.0.2171.71', '39.0.2171.95',
'39.0.2171.99', '40.0.2214.93', '40.0.2214.111', '40.0.2214.115',
'42.0.2311.90', '42.0.2311.135', '42.0.2311.152', '43.0.2357.81',
'43.0.2357.124', '44.0.2403.155', '44.0.2403.157', '45.0.2454.101',
'45.0.2454.85', '46.0.2490.71', '46.0.2490.80', '46.0.2490.86',
'47.0.2526.73', '47.0.2526.80', '48.0.2564.116', '49.0.2623.112',
'50.0.2661.86', '51.0.2704.103', '52.0.2743.116', '53.0.2785.143',
'54.0.2840.71', '61.0.3163.100'
], ['11.0'], ['8.0', '9.0', '10.0', '10.6']
]
WIN_VERS = [
'Windows NT 10.0', 'Windows NT 7.0', 'Windows NT 6.3',
'Windows NT 6.2', 'Windows NT 6.1', 'Windows NT 6.0', 'Windows NT 5.1',
'Windows NT 5.0'
]
FEATURES = ['; WOW64', '; Win64; IA64', '; Win64; x64', '']
RAND_UAS = [
'Mozilla/5.0 ({win_ver}{feature}; rv:{br_ver}) Gecko/20100101 Firefox/{br_ver}',
'Mozilla/5.0 ({win_ver}{feature}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{br_ver} Safari/537.36',
'Mozilla/5.0 ({win_ver}{feature}; Trident/7.0; rv:{br_ver}) like Gecko',
'Mozilla/5.0 (compatible; MSIE {br_ver}; {win_ver}{feature}; Trident/6.0)'
]
index = randrange(len(RAND_UAS))
return RAND_UAS[index].format(win_ver=choice(WIN_VERS),
feature=choice(FEATURES),
br_ver=choice(BR_VERS[index]))
def read_object(self, f, enc_type):
if enc_type == REDIS_RDB_TYPE_STRING:
val = self.read_string(f)
self._callback.set(self._key,
val,
self._expiry,
info={
'encoding': 'string',
'idle': self._idle,
'freq': self._freq
})
elif enc_type == REDIS_RDB_TYPE_LIST:
# A redis list is just a sequence of strings
# We successively read strings from the stream and create a list from it
# The lists are in order i.e. the first string is the head,
# and the last string is the tail of the list
length = self.read_length(f)
self._callback.start_list(self._key,
self._expiry,
info={
'encoding': 'linkedlist',
'idle': self._idle,
'freq': self._freq
})
for count in range(0, length):
val = self.read_string(f)
self._callback.rpush(self._key, val)
self._callback.end_list(self._key, info={'encoding': 'linkedlist'})
elif enc_type == REDIS_RDB_TYPE_SET:
# A redis list is just a sequence of strings
# We successively read strings from the stream and create a set from it
# Note that the order of strings is non-deterministic
length = self.read_length(f)
self._callback.start_set(self._key,
length,
self._expiry,
info={
'encoding': 'hashtable',
'idle': self._idle,
'freq': self._freq
})
for count in range(0, length):
val = self.read_string(f)
self._callback.sadd(self._key, val)
self._callback.end_set(self._key)
elif enc_type == REDIS_RDB_TYPE_ZSET or enc_type == REDIS_RDB_TYPE_ZSET_2:
length = self.read_length(f)
self._callback.start_sorted_set(self._key,
length,
self._expiry,
info={
'encoding': 'skiplist',
'idle': self._idle,
'freq': self._freq
})
for count in range(0, length):
val = self.read_string(f)
score = read_binary_double(
f
) if enc_type == REDIS_RDB_TYPE_ZSET_2 else self.read_float(f)
self._callback.zadd(self._key, score, val)
self._callback.end_sorted_set(self._key)
elif enc_type == REDIS_RDB_TYPE_HASH:
length = self.read_length(f)
self._callback.start_hash(self._key,
length,
self._expiry,
info={
'encoding': 'hashtable',
'idle': self._idle,
'freq': self._freq
})
for count in range(0, length):
field = self.read_string(f)
value = self.read_string(f)
self._callback.hset(self._key, field, value)
self._callback.end_hash(self._key)
elif enc_type == REDIS_RDB_TYPE_HASH_ZIPMAP:
self.read_zipmap(f)
elif enc_type == REDIS_RDB_TYPE_LIST_ZIPLIST:
self.read_ziplist(f)
elif enc_type == REDIS_RDB_TYPE_SET_INTSET:
self.read_intset(f)
elif enc_type == REDIS_RDB_TYPE_ZSET_ZIPLIST:
self.read_zset_from_ziplist(f)
elif enc_type == REDIS_RDB_TYPE_HASH_ZIPLIST:
self.read_hash_from_ziplist(f)
elif enc_type == REDIS_RDB_TYPE_LIST_QUICKLIST:
self.read_list_from_quicklist(f)
elif enc_type == REDIS_RDB_TYPE_MODULE:
raise Exception(
'read_object',
'Unable to read Redis Modules RDB objects (key %s)' %
self._key)
# 刚开始是从这里进入的 read_module，因为 已经读到了 data_type = 7
elif enc_type == REDIS_RDB_TYPE_MODULE_2:
self.read_module(f)
elif enc_type == REDIS_RDB_TYPE_STREAM_LISTPACKS:
self.read_stream(f)
else:
raise Exception(
'read_object',
'Invalid object type %d for key %s' % (enc_type, self._key))