def testBase64(self):
dec = "Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure."
enc = "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz\nIHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg\ndGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu\ndWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo\nZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=\n"
enc2 = SimpleHash.base64_encode(dec)
self.assertEquals(enc, enc2)
dec2 = SimpleHash.base64_decode(enc)
self.assertEquals(dec, dec2)
def test(self):
send_buf = []
self.task_session_1.send = lambda x: send_buf.append(x)
# session_2 [GetResource] -> session_1
msg_get = message.tasks.GetResource(task_id=self.task_id)
self.task_session_1._react_to_get_resource(msg_get)
# session_1 [ResourceList] -> session_2
msg_list = send_buf.pop()
self.task_session_2._react_to_resource_list(msg_list)
# client_2 downloads resources specified in the message
self.client_2.resource_server._download_resources(async_=False)
# verify downloaded resources
for relative_path in self.resources_relative:
location_1 = os.path.join(self.resource_dir_1, relative_path)
location_2 = os.path.join(self.resource_dir_2, relative_path)
assert os.path.exists(location_1)
assert os.path.exists(location_2)
sha_256_1 = SimpleHash.hash_file_base64(location_1)
sha_256_2 = SimpleHash.hash_file_base64(location_2)
assert sha_256_1 == sha_256_2
def build_header_delta_from_header(cls, header, absolute_root,
chosen_files):
if not isinstance(header, TaskResourceHeader):
raise TypeError(
"Incorrect header type: {}. Should be TaskResourceHeader".
format(type(header)))
cur_tr = TaskResourceHeader(header.dir_name)
dirs = [
name for name in os.listdir(absolute_root)
if os.path.isdir(os.path.join(absolute_root, name))
]
files = [
name for name in os.listdir(absolute_root)
if os.path.isfile(os.path.join(absolute_root, name))
]
for d in dirs:
if header.__has_sub_header(d):
cur_tr.sub_dir_headers.append(
cls.build_header_delta_from_header(
header.__get_sub_header(d),
os.path.join(absolute_root, d), chosen_files))
else:
cur_tr.sub_dir_headers.append(
cls.__build(d, os.path.join(absolute_root, d),
chosen_files))
for f in files:
if chosen_files and os.path.join(absolute_root,
f) not in chosen_files:
continue
file_hash = 0
if header.__has_file(f):
file_hash = SimpleHash.hash_file_base64(
os.path.join(absolute_root, f))
if file_hash == header.__get_file_hash(f):
continue
if not file_hash:
file_hash = SimpleHash.hash_file_base64(
os.path.join(absolute_root, f))
cur_tr.files_data.append((f, file_hash))
return cur_tr
def test(self):
file_path = os.path.join(self.path, 'file.txt')
with open(file_path, 'wb') as out:
out.write(b'The quick brown fox jumps over the lazy dog\n')
b64 = b"vkF3aLXDxcHZvLLnwRkZbddrVXA=\n"
self.assertEqual(b64, SimpleHash.hash_file_base64(file_path))
def get_short_hash(self):
"""Return short message representation for signature
:return str: short hash of serialized and sorted message dictionary
representation
"""
sorted_dict = self._sort_obj(self.dict_repr())
return SimpleHash.hash(CBORSerializer.dumps(sorted_dict))
def __build(cls, dirName, absoluteRoot):
curTh = TaskResourceHeader(dirName)
dirs = [
name for name in os.listdir(absoluteRoot)
if os.path.isdir(os.path.join(absoluteRoot, name))
]
files = [
name for name in os.listdir(absoluteRoot)
if os.path.isfile(os.path.join(absoluteRoot, name))
]
filesData = []
for f in files:
hsh = SimpleHash.hash_file_base64(os.path.join(absoluteRoot, f))
filesData.append((f, hsh))
#print "{}, {}, {}".format( relativeRoot, absoluteRoot, filesData )
curTh.filesData = filesData
subDirHeaders = []
for d in dirs:
childSubDirHeader = cls.__build(d, os.path.join(absoluteRoot, d))
subDirHeaders.append(childSubDirHeader)
curTh.subDirHeaders = subDirHeaders
#print "{} {} {}\n".format( absoluteRoot, len( subDirHeaders ), len( filesData ) )
return curTh
def build_from_chosen(cls, dir_name, absolute_root, chosen_files=None):
cur_th = TaskResourceHeader(dir_name)
abs_dirs = split_path(absolute_root)
for f in chosen_files:
dir_, file_name = os.path.split(f)
dirs = split_path(dir_)[len(abs_dirs):]
last_header = cur_th
for d in dirs:
child_sub_dir_header = TaskResourceHeader(d)
if last_header.__has_sub_header(d):
last_header = last_header.__get_sub_header(d)
else:
last_header.sub_dir_headers.append(child_sub_dir_header)
last_header = child_sub_dir_header
hsh = SimpleHash.hash_file_base64(f)
last_header.files_data.append((file_name, hsh))
return cur_th
def __build(cls, dir_name, absolute_root):
cur_th = TaskResource(dir_name)
dirs = [
name for name in os.listdir(absolute_root)
if os.path.isdir(os.path.join(absolute_root, name))
]
files = [
name for name in os.listdir(absolute_root)
if os.path.isfile(os.path.join(absolute_root, name))
]
files_data = []
for f in files:
file_data = cls.read_file(os.path.join(absolute_root, f))
hsh = SimpleHash.hash_base64(file_data)
files_data.append((f, hsh, file_data))
# print "{}, {}, {}".format(relative_root, absolute_root, files_data)
cur_th.files_data = files_data
sub_dir_resources = []
for d in dirs:
child_sub_dir_header = cls.__build(d,
os.path.join(absolute_root, d))
sub_dir_resources.append(child_sub_dir_header)
cur_th.sub_dir_resources = sub_dir_resources
# print "{} {} {}\n".format(absolute_root, len(sub_dir_headers), len(files_data))
return cur_th
def build_parts_header_delta_from_chosen(cls, header, absolute_root,
res_parts):
if not isinstance(header, TaskResourceHeader):
raise TypeError(
"Incorrect header type: {}. Should be TaskResourceHeader".
format(type(header)))
cur_th = TaskResourceHeader(header.dir_name)
abs_dirs = split_path(absolute_root)
delta_parts = []
for file_, parts in res_parts.items():
dir_, file_name = os.path.split(file_)
dirs = split_path(dir_)[len(abs_dirs):]
last_header = cur_th
last_ref_header = header
last_header, last_ref_header, ref_header_found = cls.__resolve_dirs(
dirs, last_header, last_ref_header)
hsh = SimpleHash.hash_file_base64(file_)
if ref_header_found:
if last_ref_header.__has_file(file_name):
if hsh == last_ref_header.__get_file_hash(file_name):
continue
last_header.files_data.append((file_name, hsh, parts))
delta_parts += parts
return cur_th, delta_parts
def build_delta_from_header(cls, header, absolute_root):
if not isinstance(header, TaskResourceHeader):
raise TypeError(
"Incorrect header type: {}. Should be TaskResourceHeader".
format(type(header)))
cur_tr = TaskResource(header.dir_name)
dirs = [
name for name in os.listdir(absolute_root)
if os.path.isdir(os.path.join(absolute_root, name))
]
files = [
name for name in os.listdir(absolute_root)
if os.path.isfile(os.path.join(absolute_root, name))
]
for d in dirs:
if d in [sdh.dir_name for sdh in header.sub_dir_headers]:
idx = [sdh.dir_name for sdh in header.sub_dir_headers].index(d)
cur_tr.sub_dir_resources.append(
cls.build_delta_from_header(header.sub_dir_headers[idx],
os.path.join(absolute_root,
d)))
else:
cur_tr.sub_dir_resources.append(
cls.__build(d, os.path.join(absolute_root, d)))
for f in files:
if f in [file_[0] for file_ in header.files_data]:
idx = [file_[0] for file_ in header.files_data].index(f)
if SimpleHash.hash_file_base64(os.path.join(
absolute_root, f)) == header.files_data[idx][1]:
continue
fdata = cls.read_file(os.path.join(absolute_root, f))
if fdata is None:
return None
cur_tr.files_data.append((f, SimpleHash.hash_base64(fdata), fdata))
return cur_tr
def buildHeaderDeltaFromHeader(cls, header, absoluteRoot):
assert isinstance(header, TaskResourceHeader)
curTr = TaskResourceHeader(header.dirName)
dirs = [
name for name in os.listdir(absoluteRoot)
if os.path.isdir(os.path.join(absoluteRoot, name))
]
files = [
name for name in os.listdir(absoluteRoot)
if os.path.isfile(os.path.join(absoluteRoot, name))
]
for d in dirs:
if d in [sdh.dirName for sdh in header.subDirHeaders]:
idx = [sdh.dirName for sdh in header.subDirHeaders].index(d)
curTr.subDirHeaders.append(
cls.buildHeaderDeltaFromHeader(
header.subDirHeaders[idx],
os.path.join(absoluteRoot, d)))
else:
curTr.subDirHeaders.append(
cls.__build(d, os.path.join(absoluteRoot, d)))
for f in files:
fileHash = 0
if f in [file[0] for file in header.filesData]:
idx = [file[0] for file in header.filesData].index(f)
fileHash = SimpleHash.hash_file_base64(
os.path.join(absoluteRoot, f))
if fileHash == header.filesData[idx][1]:
continue
if not fileHash:
fileHash = SimpleHash.hash_file_base64(
os.path.join(absoluteRoot, f))
curTr.filesData.append((f, fileHash))
return curTr
def extract(self, toPath):
for dir in self.subDirResources:
if not os.path.exists(os.path.join(toPath, dir.dirName)):
os.makedirs(os.path.join(toPath, dir.dirName))
dir.extract(os.path.join(toPath, dir.dirName))
for f in self.filesData:
if not os.path.exists(os.path.join(
toPath, f[0])) or SimpleHash.hash_file_base64(
os.path.join(toPath, f[0])) != f[1]:
self.writeFile(os.path.join(toPath, f[0]), f[2])
def extract(self, to_path):
for dir_ in self.sub_dir_resources:
if not os.path.exists(os.path.join(to_path, dir_.dir_name)):
os.makedirs(os.path.join(to_path, dir_.dir_name))
dir_.extract(os.path.join(to_path, dir_.dir_name))
for f in self.files_data:
if not os.path.exists(os.path.join(
to_path, f[0])) or SimpleHash.hash_file_base64(
os.path.join(to_path, f[0])) != f[1]:
self.write_file(os.path.join(to_path, f[0]), f[2])
def testHash(self):
ex1 = ""
hex1 = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
b641 = "2jmj7l5rSw0yVb/vlWAYkK/YBwk=\n"
hash1 = "\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t"
ex2 = "The quick brown fox jumps over the lazy dog"
hex2 = "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12"
b642 = "L9ThxnotKPzthJ7hu3bnORuT6xI=\n"
hash2 = "/\xd4\xe1\xc6z-(\xfc\xed\x84\x9e\xe1\xbbv\xe79\x1b\x93\xeb\x12"
self.assertEquals(hash1, SimpleHash.hash(ex1))
self.assertEquals(hash2, SimpleHash.hash(ex2))
self.assertEquals(hex1, SimpleHash.hash_hex(ex1))
self.assertEquals(hex2, SimpleHash.hash_hex(ex2))
self.assertEquals(b641, SimpleHash.hash_base64(ex1))
self.assertEquals(b642, SimpleHash.hash_base64(ex2))
def __build(cls, dir_name, absolute_root, chosen_files=None):
cur_th = TaskResourceHeader(dir_name)
dirs = [
name for name in os.listdir(absolute_root)
if os.path.isdir(os.path.join(absolute_root, name))
]
files = [
name for name in os.listdir(absolute_root)
if os.path.isfile(os.path.join(absolute_root, name))
]
files_data = []
for f in files:
if chosen_files and os.path.join(absolute_root,
f) not in chosen_files:
continue
hsh = SimpleHash.hash_file_base64(os.path.join(absolute_root, f))
files_data.append((f, hsh))
# print "{}, {}, {}".format(relative_root, absolute_root, files_data)
cur_th.files_data = files_data
sub_dir_headers = []
for d in dirs:
child_sub_dir_header = cls.__build(d,
os.path.join(absolute_root,
d), chosen_files)
sub_dir_headers.append(child_sub_dir_header)
cur_th.sub_dir_headers = sub_dir_headers
# print "{} {} {}\n".format(absolute_root, len(sub_dir_headers), len(files_data))
return cur_th
def hash(self):
return SimpleHash.hash_base64(self.toString())
def compute_sha1(source_path: str):
pkg_sha1 = SimpleHash.hash_file(source_path)
return binascii.hexlify(pkg_sha1).decode('utf8')
def hash(self):
return SimpleHash.hash_base64(self.to_string().encode('utf-8'))
def test_fileHash(self):
file_ = path.join(path.dirname(__file__), 'file.txt')
b64 = "vkF3aLXDxcHZvLLnwRkZbddrVXA=\n"
self.assertEquals(b64, SimpleHash.hash_file_base64(file_))
def hash(self):
return 'sha1:' + binascii.hexlify(SimpleHash.hash_file(
self.filename)).decode()