def init_data_size(self, key):
if self.data_size_list0:
self.data_size_list0 = []
random = xorshift128plus()
random.init_from_bin(key)
# 补全数组长为12~24-1
list_len = random.next() % (8 + 16) + (4 + 8)
for i in range(0, list_len):
self.data_size_list0.append((int)(random.next() % 2340 % 2040 % 1440))
self.data_size_list0.sort()
def rnd_data_len(self, buf_size, last_hash, random):
if buf_size > 1440:
return 0
random.init_from_bin_len(last_hash, buf_size)
if buf_size > 1300:
return random.next() % 31
if buf_size > 900:
return random.next() % 127
if buf_size > 400:
return random.next() % 521
return random.next() % 1021
def init_data_size(self, key):
if self.data_size_list0:
self.data_size_list0 = []
random = xorshift128plus()
random.init_from_bin(key)
list_len = random.next() % (8 + 16) + (4 + 8)
for i in range(0, list_len):
self.data_size_list0.append(
(int)(random.next() % 2340 % 2040 % 1440))
self.data_size_list0.sort()
old_len = len(self.data_size_list0)
self.check_and_patch_data_size(random)
if old_len != len(self.data_size_list0):
self.data_size_list0.sort()
def init_data_size(self, key):
if self.data_size_list:
self.data_size_list = []
self.data_size_list2 = []
random = xorshift128plus()
random.init_from_bin(key)
list_len = random.next() % 8 + 4
for i in range(0, list_len):
self.data_size_list.append((int)(random.next() % 2340 % 2040 % 1440))
self.data_size_list.sort()
list_len = random.next() % 16 + 8
for i in range(0, list_len):
self.data_size_list2.append((int)(random.next() % 2340 % 2040 % 1440))
self.data_size_list2.sort()
def recv_rnd_data_len(self, buf_size, last_hash, random):
if buf_size + self.server_info.overhead > self.recv_tcp_mss:
random.init_from_bin_len(last_hash, buf_size)
return random.next() % 521
if buf_size >= 1440 or buf_size + self.server_info.overhead == self.send_tcp_mss:
return 0
random.init_from_bin_len(last_hash, buf_size)
if buf_size > 1300:
return random.next() % 31
if buf_size > 900:
return random.next() % 127
if buf_size > 400:
return random.next() % 521
return random.next() % (self.recv_tcp_mss - buf_size - self.server_info.overhead)
def check_and_patch_data_size(self, random):
# append new item
# when the biggest item(first time) or the last append item(other time) are not big enough.
# but set a limit size (64) to avoid stack overflow.
if self.data_size_list0[-1] < 1300 and len(self.data_size_list0) < 64:
self.data_size_list0.append((int)(random.next() % 2340 % 2040 % 1440))
self.check_and_patch_data_size(random)
def init_data_size(self, key):
if self.data_size_list0:
self.data_size_list0 = []
random = xorshift128plus()
new_key = bytearray(key)
for i in range(0, 8):
new_key[i] ^= self.key_change_datetime_key_bytes[i]
random.init_from_bin(new_key)
list_len = random.next() % (8 + 16) + (4 + 8)
for i in range(0, list_len):
self.data_size_list0.append(int(random.next() % 2340 % 2040 %
1440))
self.data_size_list0.sort()
old_len = len(self.data_size_list0)
self.check_and_patch_data_size(random)
if old_len != len(self.data_size_list0):
self.data_size_list0.sort()
def rnd_data_len(self, buf_size, last_hash, random):
other_data_size = buf_size + self.server_info.overhead
random.init_from_bin_len(last_hash, buf_size)
if other_data_size >= self.data_size_list0[-1]:
if other_data_size >= 1440:
return 0
if other_data_size > 1300:
return random.next() % 31
if other_data_size > 900:
return random.next() % 127
if other_data_size > 400:
return random.next() % 521
return random.next() % 1021
pos = bisect.bisect_left(self.data_size_list0, other_data_size)
final_pos = pos + random.next() % (len(self.data_size_list0) - pos)
return self.data_size_list0[final_pos] - other_data_size
def recv_rnd_data_len(self, buf_size, last_hash, random):
if buf_size + self.server_info.overhead > self.recv_tcp_mss:
random.init_from_bin_len(last_hash, buf_size)
return random.next() % 521
if buf_size >= 1440 or buf_size + self.server_info.overhead == self.send_tcp_mss:
return 0
random.init_from_bin_len(last_hash, buf_size)
pos = bisect.bisect_left(self.data_size_list, buf_size + self.server_info.overhead)
final_pos = pos + random.next() % (len(self.data_size_list))
if final_pos < len(self.data_size_list):
return self.data_size_list[final_pos] - buf_size - self.server_info.overhead
pos = bisect.bisect_left(self.data_size_list2, buf_size + self.server_info.overhead)
final_pos = pos + random.next() % (len(self.data_size_list2))
if final_pos < len(self.data_size_list2):
return self.data_size_list2[final_pos] - buf_size - self.server_info.overhead
if final_pos < pos + len(self.data_size_list2) - 1:
return 0
if buf_size > 1300:
return random.next() % 31
if buf_size > 900:
return random.next() % 127
if buf_size > 400:
return random.next() % 521
return random.next() % 1021
def rnd_data_len(self, buf_size, last_hash, random):
other_data_size = buf_size + self.server_info.overhead
if other_data_size >= self.data_size_list0[-1]:
return 0
random.init_from_bin_len(last_hash, buf_size)
pos = bisect.bisect_left(self.data_size_list0, other_data_size)
final_pos = pos + random.next() % (len(self.data_size_list0) - pos)
return self.data_size_list0[final_pos] - other_data_size
def rnd_data_len(self, buf_size, last_hash, random):
other_data_size = buf_size + self.server_info.overhead
# if other_data_size > the bigest item in data_size_list0, not padding any data
if other_data_size >= self.data_size_list0[-1]:
return 0
random.init_from_bin_len(last_hash, buf_size)
pos = bisect.bisect_left(self.data_size_list0, other_data_size)
# random select a size in the leftover data_size_list0
final_pos = pos + random.next() % (len(self.data_size_list0) - pos)
return self.data_size_list0[final_pos] - other_data_size
def rnd_data_len(self, buf_size, last_hash, random):
other_data_size = buf_size + self.server_info.overhead
# 一定要在random使用前初始化，以保证服务器与客户端同步，保证包大小验证结果正确
random.init_from_bin_len(last_hash, buf_size)
# final_pos 总是分布在pos~(data_size_list0.len-1)之间
# 除非data_size_list0中的任何值均过小使其全部都无法容纳buf
if other_data_size >= self.data_size_list0[-1]:
if other_data_size >= 1440:
return 0
if other_data_size > 1300:
return random.next() % 31
if other_data_size > 900:
return random.next() % 127
if other_data_size > 400:
return random.next() % 521
return random.next() % 1021
pos = bisect.bisect_left(self.data_size_list0, other_data_size)
# random select a size in the leftover data_size_list0
final_pos = pos + random.next() % (len(self.data_size_list0) - pos)
return self.data_size_list0[final_pos] - other_data_size
def rnd_data_len(self, buf_size, last_hash, random):
if buf_size >= 1440:
return 0
random.init_from_bin_len(last_hash, buf_size)
pos = bisect.bisect_left(self.data_size_list,
buf_size + self.server_info.overhead)
final_pos = pos + random.next() % (len(self.data_size_list))
# 假设random均匀分布，则越长的原始数据长度越容易if false
if final_pos < len(self.data_size_list):
return self.data_size_list[
final_pos] - buf_size - self.server_info.overhead
# 上面if false后选择2号补全数组，此处有更精细的长度分段
pos = bisect.bisect_left(self.data_size_list2,
buf_size + self.server_info.overhead)
final_pos = pos + random.next() % (len(self.data_size_list2))
if final_pos < len(self.data_size_list2):
return self.data_size_list2[
final_pos] - buf_size - self.server_info.overhead
# final_pos 总是分布在pos~(data_size_list2.len-1)之间
if final_pos < pos + len(self.data_size_list2) - 1:
return 0
# 有1/len(self.data_size_list2)的概率不满足上一个if ?
# 理论上不会运行到此处，因此可以插入运行断言 ?
# assert False
if buf_size > 1300:
return random.next() % 31
if buf_size > 900:
return random.next() % 127
if buf_size > 400:
return random.next() % 521
return random.next() % 1021
def init_data_size(self, key):
if self.data_size_list0:
self.data_size_list0 = []
random = xorshift128plus()
# key xor with key_change_datetime_key
new_key = bytearray(key)
new_key_str = ''
for i in range(0, 8):
new_key[i] ^= self.key_change_datetime_key_bytes[i]
new_key_str += chr(new_key[i])
for i in range(8, len(new_key)):
new_key_str += chr(new_key[i])
random.init_from_bin(to_bytes(new_key_str))
# 补全数组长为12~24-1
list_len = random.next() % (8 + 16) + (4 + 8)
for i in range(0, list_len):
self.data_size_list0.append(int(random.next() % 2340 % 2040 % 1440))
self.data_size_list0.sort()
old_len = len(self.data_size_list0)
self.check_and_patch_data_size(random)
# if check_and_patch_data_size are work, re-sort again.
if old_len != len(self.data_size_list0):
self.data_size_list0.sort()
def rnd_start_pos(self, rand_len, random):
if rand_len > 0:
return random.next() % 8589934609 % rand_len
return 0
def udp_rnd_data_len(self, last_hash, random):
random.init_from_bin(last_hash)
return random.next() % 127
def check_and_patch_data_size(self, random):
if self.data_size_list0[-1] < 1300 and len(self.data_size_list0) < 64:
self.data_size_list0.append(
(int)(random.next() % 2340 % 2040 % 1440))
self.check_and_patch_data_size(random)
def readTemperature(self):
return random.next(35.9, 37.9)
def dataset_fillCohnKanade(dsFolder,
ckFolder,
ckEmoFolder,
config,
vperc=0.3,
vseed=0):
"""
Fill dataset with Cohn Kanade + data.
"""
subjects = [x for x in os.listdir(ckFolder) if isdir(join(ckFolder, x))]
print "INFO: %d subjects found in CK+ database" % len(subjects)
for subj in subjects:
print "INFO: Processing subject %s " % subj
labelFolders = [
x for x in os.listdir(join(ckEmoFolder, subj))
if isdir(join(ckEmoFolder, join(subj, x)))
]
imageFolders = [
x for x in os.listdir(join(ckFolder, subj))
if isdir(join(ckEmoFolder, join(subj, x)))
]
shots = [x for x in imageFolders if x in labelFolders]
for s in shots:
print "INFO: Processing shot %s " % s
pics = [
x for x in os.listdir(join(ckFolder, join(subj, s)))
if isfile(join(ckFolder, join(subj, join(s, x))))
]
pics.sort()
labels = [
x for x in os.listdir(join(ckEmoFolder, join(subj, s)))
if isfile(join(ckEmoFolder, join(subj, join(s, x))))
]
if len(labels) < 1 or len(pics) < 1:
# label forlder could contain no file at all, in this case skip the current shot or mark it as neutral?
print "WARN: subject %s shot %s has #%d emo labels and #%d pictures, (skip:incomplete)" % (
subj, s, len(labels), len(pics))
continue
emo = None
with open(join(ckEmoFolder, join(subj, join(s, labels[0]))),
"r") as f:
buf = f.read()
if len(buf) == 0:
print "WARN: subject %s shot %s has void emo label '%s', (skip:noemo)" % (
subj, s,
join(ckEmoFolder, join(subj, join(s, labels[0]))))
# A label file could be void, in this case skip the current shot
continue
try:
emo = config['CLASSES'][int(float(strip(buf)))]
except:
print "ERR: cannot parse emotional label for subject %s shot %s (skip:unknown_emo)" % (
subj, s)
continue
# Last picture is the final emotion (most intense), first picture is neutral
to_copy = [(pics[-1], emo), (pics[0], config['CLASSES'][0])]
for pic, emo in to_copy:
print "INFO: Picture '%s' has been marked as %s" % (pic, emo)
orig = join(ckFolder, join(subj, join(s, pic)))
IMAGES_FOLDER = config['TRAINING_IMAGES']
if random.next() <= vperc:
IMAGES_FOLDER = config['VALIDATION_IMAGES']
dest = join(dsFolder, join(IMAGES_FOLDER, join(emo, pic)))
try:
shutil.copy(orig, dest)
except:
print "ERR: cannot copy image '%s' to dataset '%s' " % (
orig, dest)
continue