def delta_size():
with open("deltasize", "w") as f:
matches = []
misses = []
sizes = []
for step in xrange(1, 51):
count_mat = 0
count_mis = 0
size = 0
subprocess.call(
["dd", "if=/dev/urandom", "of=1.dat", "bs=4096", "count=100"])
subprocess.call(
["dd", "if=/dev/urandom", "of=2.dat", "bs=4096", "count=100"])
b = delta.blockchecksums("2.dat", 64) #"linux-1.1.94.tar.gz", 64)
d = delta.delta("1.dat", b, 64,
step) #"linux-1.1.95.tar.gz", b, 64, step)
for offset, data in d:
if data:
count_mis += 1
size += len(data)
else:
count_mat += 1
matches.append(count_mat)
misses.append(count_mis)
sizes.append(size)
f.write("mat <- c({})\n".format(", ".join([str(x) for x in matches])))
f.write("mis <- c({})\n".format(", ".join([str(x) for x in misses])))
f.write("size <- c({})\n".format(", ".join([str(x) for x in sizes])))
def delta_random():
with open("deltarand", "w") as f:
matches = []
misses = []
sizes = []
for x in xrange(0, 101):
count_mat = 0
count_mis = 0
size = 0
count = 100
subprocess.call([
"dd", "if=/dev/urandom", "of=1.dat", "bs=4096",
"count={}".format(count)
])
subprocess.call([
"dd", "if=/dev/urandom", "of=2.dat", "bs=4096",
"count={}".format(count)
])
b = delta.blockchecksums("2.dat", 8)
d = delta.delta("1.dat", b, 8)
for offset, data in d:
if data:
count_mis += 1
size += len(data)
else:
count_mat += 1
matches.append(count_mat)
misses.append(count_mis)
sizes.append(size)
f.write("mat <- c({})\n".format(", ".join([str(x) for x in matches])))
f.write("mis <- c({})\n".format(", ".join([str(x) for x in misses])))
f.write("size <- c({})\n".format(", ".join([str(x) for x in sizes])))
def delta_random():
with open("deltarand", "w") as f:
matches = []
misses = []
sizes = []
for x in xrange(0,101):
count_mat = 0
count_mis = 0
size = 0
count = 100
subprocess.call(["dd", "if=/dev/urandom", "of=1.dat", "bs=4096", "count={}".format(count)])
subprocess.call(["dd", "if=/dev/urandom", "of=2.dat", "bs=4096", "count={}".format(count)])
b = delta.blockchecksums("2.dat", 8)
d = delta.delta("1.dat", b, 8)
for offset, data in d:
if data:
count_mis += 1
size += len(data)
else:
count_mat += 1
matches.append(count_mat)
misses.append(count_mis)
sizes.append(size)
f.write("mat <- c({})\n".format(", ".join([str(x) for x in matches])))
f.write("mis <- c({})\n".format(", ".join([str(x) for x in misses])))
f.write("size <- c({})\n".format(", ".join([str(x) for x in sizes])))
def delta_size():
with open("deltasize", "w") as f:
matches = []
misses = []
sizes = []
for step in xrange(1,51):
count_mat = 0
count_mis = 0
size = 0
subprocess.call(["dd", "if=/dev/urandom", "of=1.dat", "bs=4096", "count=100"])
subprocess.call(["dd", "if=/dev/urandom", "of=2.dat", "bs=4096", "count=100"])
b = delta.blockchecksums("2.dat", 64)#"linux-1.1.94.tar.gz", 64)
d = delta.delta("1.dat", b, 64, step)#"linux-1.1.95.tar.gz", b, 64, step)
for offset, data in d:
if data:
count_mis += 1
size += len(data)
else:
count_mat += 1
matches.append(count_mat)
misses.append(count_mis)
sizes.append(size)
f.write("mat <- c({})\n".format(", ".join([str(x) for x in matches])))
f.write("mis <- c({})\n".format(", ".join([str(x) for x in misses])))
f.write("size <- c({})\n".format(", ".join([str(x) for x in sizes])))
def test_delta_equal(self):
data = 'hello'
with open('.tmp', 'wb') as f:
f.write(data)
d = delta('.tmp', blockchecksums('.tmp'))
self.failUnlessEqual([(0, '')], d)
def test_delta_equal(self):
data = 'hello'
with open('.tmp', 'wb') as f:
f.write(data)
d = delta('.tmp', blockchecksums('.tmp'))
os.remove('.tmp')
self.failUnlessEqual([(0, '')], d)
def test_delta_new(self):
data = 'hello'
with open('.tmp1', 'wb') as f:
f.write(data)
new = open('.tmp2', 'wb')
new.close()
d = delta('.tmp1', blockchecksums('.tmp2'))
self.failUnlessEqual([(0, 'hello')], d)
def checksum_dist(filename, n, w):
blocksize = n * 1024
b = delta.blockchecksums(filename, blocksize)
dist = []
print "BLOCKSIZE {0}".format(blocksize)
for i in xrange(1,w):
num = len([len(x) for x in b.values() if len(x) == i])
print "{0}: {1}".format(i, num)
dist.append(num)
return [float(x)/sum(dist) for x in dist]
def checksum_dist(filename, n, w):
blocksize = n * 1024
b = delta.blockchecksums(filename, blocksize)
dist = []
print "BLOCKSIZE {0}".format(blocksize)
for i in xrange(1, w):
num = len([len(x) for x in b.values() if len(x) == i])
print "{0}: {1}".format(i, num)
dist.append(num)
return [float(x) / sum(dist) for x in dist]
def test_blockchecksums(self):
data = 'hello'
md5 = hashlib.md5()
md5.update(data)
c1 = zlib.adler32(data) & 0xffffffff
c2 = md5.hexdigest()
h1 = { unicode(c1 >> 16): [(0, c1, c2)] }
with open('.tmp','wb') as f:
f.write(data)
h2 = blockchecksums('.tmp')
self.failUnlessEqual(h1, h2)
def test_delta_new(self):
data = 'hello'
with open('.tmp1', 'wb') as f:
f.write(data)
new = open('.tmp2', 'wb')
new.close()
d = delta('.tmp1', blockchecksums('.tmp2'))
os.remove('.tmp1')
os.remove('.tmp2')
self.failUnlessEqual([(0, base64.b64encode(data))], d)
def test_delta_new(self):
data = 'hello'
with open('.tmp1', 'wb') as f:
f.write(data)
new = open('.tmp2', 'wb')
new.close()
d = delta('.tmp1', blockchecksums('.tmp2'))
os.remove('.tmp1')
os.remove('.tmp2')
self.failUnlessEqual([(0, base64.b64encode(data))], d)
def blockchecksums(self, path):
"""
Get blockchecksums for the given file.
@param path: path.
@type path: str
@return: blockchecksums.
@rtype: str (json dict)
"""
path = self._get_path(path)
try:
bs = blockchecksums(path)
except OSError as e:
bs = {}
return json.dumps(bs)
def test_blockchecksums(self):
data = 'hello'
md5 = hashlib.md5()
md5.update(data)
c1 = zlib.adler32(data) & 0xffffffff
c2 = md5.hexdigest()
h1 = {unicode(c1 >> 16): [(0, c1, c2)]}
with open('.tmp', 'wb') as f:
f.write(data)
h2 = blockchecksums('.tmp')
os.remove('.tmp')
self.failUnlessEqual(h1, h2)
def test_delta_patch(self):
data = 'hello'
with open('.tmp1', 'wb') as f:
f.write(data)
new = open('.tmp2', 'wb')
new.close()
d = delta('.tmp1', blockchecksums('.tmp2'))
patchname = patch('.tmp2', d)
self.assertTrue(filecmp.cmp('.tmp1', patchname))
os.remove('.tmp1')
os.remove('.tmp2')
os.remove(patchname)
def test_delta_patch(self):
data = 'hello'
with open('.tmp1', 'wb') as f:
f.write(data)
new = open('.tmp2', 'wb')
new.close()
d = delta('.tmp1', blockchecksums('.tmp2'))
patchname = patch('.tmp2', d)
self.assertTrue(filecmp.cmp('.tmp1',patchname))
os.remove('.tmp1')
os.remove('.tmp2')
os.remove(patchname)
def _process(self, t, request_number, msg):
"""
Overwritten method for processing incoming requests to except
and execute synchronization specific requests.
This is a hook into the paramiko.SFTPServer implementation
See L{paramiko.SFTPServer._process}
"""
if t == CMD_BLOCKCHK:
path = msg.get_string()
bs = blockchecksums(self.server._get_path(path))
j = json.dumps(bs)
message = paramiko.Message()
message.add_int(request_number)
message.add_string(j)
self._send_packet(t, str(message))
return
elif t == CMD_DELTA:
path = msg.get_string()
bs = json.loads(msg.get_string())
d = delta(self.server._get_path(path), bs)
j = json.dumps(d)
message = paramiko.Message()
message.add_int(request_number)
message.add_string(j)
self._send_packet(t, str(message))
elif t == CMD_PATCH:
path = msg.get_string()
d = json.loads(msg.get_string())
patched = patch(self.server._get_path(path), d)
self._send_status(request_number, self.server.rename(patched, path))
elif t == CMD_OTP:
self._send_status(request_number, self.server.onetimepass())
else:
return paramiko.SFTPServer._process(self, t, request_number, msg)
def blockchecksums(self, path):
return blockchecksums(path)
def checksums(self, path):
return blockchecksums(path)