def test_patch(self):
s = librsync.signature(self.rand1)
d = librsync.delta(self.rand2, s)
self.rand1.seek(0)
self.rand2.seek(0)
o = librsync.patch(self.rand1, d)
self.assertEqual(o.read(), self.rand2.read())
def test_patch(self):
s = librsync.signature(self.rand1)
d = librsync.delta(self.rand2, s)
self.rand1.seek(0)
self.rand2.seek(0)
o = librsync.patch(self.rand1, d)
self.assertEqual(o.read(), self.rand2.read())
def add_diff(self, signature, filename=None):
if not filename:
filename = self.name
with open(filename, "r") as fp:
delta = librsync.delta(fp, signature)
# TODO: chunk big diffs
self["data"] = delta.read()
return self
def recieve_signature_and_send_delta(client_id, client, ip, file_name,
db_conn):
while pm.SharedPort.server_sig_port_used:
continue
pm.SharedPort.server_sig_port_used = True
sig_socket = socket.socket()
address = ('', pm.SharedPort.server_sig_port)
sig_socket.bind(address)
#print "Client sig socket is ready at: {}".format(data_socket.getsockname())
sig_socket.listen(10)
client_sig_sock, addr = sig_socket.accept()
# Receive Signature
sigdata = ""
while True:
data = client_sig_sock.recv(BUFFER_SIZE)
sigdata += data
if not data:
break
client_sig_sock.close()
sig_socket.close()
pm.SharedPort.server_sig_port_used = False
logging.info("Signature Received!")
# logging.info("Sync-ing and uploading %s", file_name)
# send SENDDEL msg
msg = pm.get_senddel_msg(client_id, file_name, db_conn)
client.send(msg)
#calculate delta
signature = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+')
signature.write(sigdata)
signature.seek(0)
src = open(file_name, 'rb')
delta = sync.delta(src, signature)
# connect to delta socket
client_del_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
wait_net_service(client_del_socket, ip, pm.SharedPort.client_del_port)
# send delta
l = delta.read(BUFFER_SIZE)
while l:
client_del_socket.send(l)
l = delta.read(BUFFER_SIZE)
delta.close()
client_del_socket.close()
logging.info("Delta Sent!")
def CalculateDelta(self, directory, fileName):
if os.path.exists(directory + "/" + ".Delta_" + fileName):
os.remove(directory + "/" + ".Delta_" + fileName)
# The destination file.
src = file(self.directoryPath + fileName, 'rb')
# The source file.
dst = file(directory + "/" + "." + fileName, 'rb')
# Step 1: prepare signature of the destination file
signature = librsync.signature(dst)
# Step 2: prepare a delta of the source file
delta = librsync.delta(src, signature)
#Save delta in a file
dill.dump(delta, open(directory + "/" + ".Delta_" + fileName, "w"))
def process_updated(path, updated, base_path, sigvault):
""" Process upated files, create a new SigVault if needed,
and create a new archives with delta (from the previous SigVault signatures).
"""
if updated:
updated_archive = tarfile.open(path, 'w:gz')
for f in updated:
f_abs = os.path.join(base_path, f)
delta = librsync.delta(open(f_abs, 'rb'), sigvault.extract(f))
delta_size = os.fstat(delta.fileno()).st_size
delta_info = tarfile.TarInfo(f)
delta_info.size = delta_size
updated_archive.addfile(delta_info, delta)
updated_archive.close()
return path
def process_updated(path, updated, base_path, sigvault):
""" Process upated files, create a new SigVault if needed,
and create a new archives with delta (from the previous SigVault signatures).
"""
if updated:
updated_archive = tarfile.open(path, 'w:gz')
for f in updated:
f_abs = os.path.join(base_path, f)
delta = librsync.delta(open(f_abs, 'rb'),
sigvault.extract(f))
delta_size = os.fstat(delta.fileno()).st_size
delta_info = tarfile.TarInfo(f)
delta_info.size = delta_size
updated_archive.addfile(delta_info, delta)
updated_archive.close()
return path
def get(self, uid, name, field, item, buf):
self._log('device->get, name=%s' % str(name))
if RSYNC:
sig = StringIO(b64decode(buf))
mnt = get_mnt_path(uid)
path = os.path.join(mnt, field, name, item)
fd = os.open(path, os.O_RDONLY)
try:
res = os.read(fd, RECORD_LEN)
except:
log_err(self, 'failed to get, cannot read, name=%s' % str(name))
return
finally:
os.close(fd)
if not res or len(res) == RECORD_LEN:
log_err(self, 'failed to get, name=%s' % str(name))
return
if RSYNC:
tmp = StringIO(res)
delta = librsync.delta(tmp, sig)
res = delta.read()
self._log('device->get, name=%s, finished' % str(name))
return self._encode(res)
def _sync(self, src, dst, dst_path):
syn = open(dst_path, 'wb')
signature = librsync.signature(dst)
delta = librsync.delta(src, signature)
librsync.patch(dst, delta, syn)
return True
def test_signature(self):
s = librsync.signature(self.rand1)
d = librsync.delta(self.rand2, s)
#!/usr/bin/python
import librsync
import tempfile
# get the file objects
dst = file('old.SLDASM')
src = file('new.SLDASM')
synced = tempfile.SpooledTemporaryFile(max_size=500000000, mode='w+b')
# do the librsync stuff
signature = librsync.signature(dst)
delta = librsync.delta(src, signature)
librsync.patch(dst, delta, synced)
# write the synced file
synced.seek(0)
syncedfile = open('synced.SLDASM', 'wb')
buf = synced.read()
syncedfile.write(buf)
syncedfile.close()
# write the signature to file
signature.seek(0)
sigfile = open('asm.signature', 'wb')
buf = signature.read()
sigfile.write(buf)
sigfile.close()
# write the delta to file
delta.seek(0)
delfile = open('asm.delta', 'wb')
def test_string_patch(self):
src_sig = librsync.signature(BytesIO(self.src))
delta = librsync.delta(BytesIO(self.dst), src_sig).read()
out = librsync.patch(BytesIO(self.src), BytesIO(delta))
self.assertEqual(self.dst, out.read())
def test_signature(self):
s = librsync.signature(self.rand1)
d = librsync.delta(self.rand2, s)
#!/usr/bin/python
import librsync
import tempfile
# get the file objects
dst = file('old.SLDPRT')
src = file('new.SLDPRT')
synced = tempfile.SpooledTemporaryFile(max_size=500000000,mode='w+b')
# do the librsync stuff
signature = librsync.signature(dst)
delta = librsync.delta(src, signature)
librsync.patch(dst,delta,synced)
# write the synced file
synced.seek(0)
syncedfile = open('synced.SLDPRT','wb')
buf = synced.read()
syncedfile.write(buf)
syncedfile.close()
# write the signature to file
signature.seek(0)
sigfile = open('prt.signature','wb')
buf = signature.read()
sigfile.write(buf)
sigfile.close()
# write the delta to file
delta.seek(0)
def service_message(msg, client_socket, db_conn):
global tempdelFiles, tempFiles, tempmvFiles, locked, conflict, delreq, mvreq
global SERVER_IP
if db_conn is None:
db_conn = pm.open_db()
msg_code, client_id, file_name, data = msg.split(pm.msgCode.delim)
if msg_code == pm.msgCode.REQTOT:
header = pm.get_senddat_msg(client_id, file_name, db_conn)
#logging.info("sending : header = %s", header)
client_socket.send(header)
client_data_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#logging.debug("waiting for data socket to be ready..")
wait_net_service(client_data_socket, SERVER_IP, S_DATA_SOCK_PORT)
#client_data_socket.connect(server_data_address)
#logging.debug("Connected")
with open(file_name, 'rb') as f:
l = f.read(BUFFER_SIZE)
while l:
client_data_socket.send(l)
l = f.read(BUFFER_SIZE)
f.close()
logging.debug("file sent: %s", file_name)
client_data_socket.close()
return 1
if msg_code == pm.msgCode.SENDSMT:
#logging.info("updating server_m_time of %s to %s",file_name,data)
pm.update_db(db_conn, file_name, "server_m_time", data)
db_conn.commit()
return 0
if msg_code == pm.msgCode.SENDSIG:
#compute delta and send
sig_socket = socket.socket()
addr = ('', pm.SharedPort.client_sig_port)
sig_socket.bind(addr)
#print "Client sig socket is ready at: {}".format(data_socket.getsockname())
sig_socket.listen(10)
server_sig_sock, addr = sig_socket.accept()
# Receive Signature
sigdata = ""
while True:
data = server_sig_sock.recv(BUFFER_SIZE)
sigdata += data
if not data:
break
server_sig_sock.close()
sig_socket.close()
logging.info("Signature Received!")
# logging.info("Sync-ing and uploading %s", file_name)
# send SENDDEL msg
msg = pm.get_senddel_msg(client_id, file_name, db_conn)
client_socket.send(msg)
signature = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL,
mode='wb+')
signature.write(sigdata)
signature.seek(0)
src = open(file_name, 'rb')
delta = sync.delta(src, signature)
delta.seek(0)
# connect to delta socket
client_del_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
wait_net_service(client_del_socket, SERVER_IP,
pm.SharedPort.server_del_port)
# send delta
l = delta.read(BUFFER_SIZE)
while l:
client_del_socket.send(l)
l = delta.read(BUFFER_SIZE)
delta.close()
client_del_socket.close()
logging.info("Delta Sent!")
return 1
if msg_code == pm.msgCode.REQSIG:
# next update : create a data socket to send large signature
msg = pm.get_sensig_msg(client_id, file_name, db_conn)
client_socket.send(msg)
send_signature(file_name)
return 1
if msg_code == pm.msgCode.SENDDEL:
# receive delta
del_socket = socket.socket()
addr = ('', pm.SharedPort.client_del_port)
del_socket.bind(addr)
#print "Client del socket is ready at: {}".format(data_socket.getsockname())
del_socket.listen(10)
client_del_sock, addr = del_socket.accept()
# Receive Signature
deldata = ""
while True:
data = client_del_sock.recv(BUFFER_SIZE)
deldata += data
if not data:
break
client_del_sock.close()
del_socket.close()
logging.info("Delta has been received!")
delta = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+')
delta.write(deldata)
delta.seek(0)
### Ask to Merge
### Hard to produce this case
### Happens when both updates are done at a same time instance
last_m_time = os.path.getmtime(file_name)
db_m_time = pm.get_data(db_conn, file_name, "client_m_time")
if last_m_time > db_m_time:
print pm.bcolors.FAIL + "WARNING!"
print file_name, " has changed since last update. Do you want to merge server's update? [Y|N]"
print "Local updates will be lost if you select 'Yes'" + pm.bcolors.ENDC
ans = raw_input()
if ans == 'n' or ans == 'N':
return 1
dest = open(file_name, 'rb')
synced_file = open(file_name, 'wb')
sync.patch(dest, delta, synced_file) # patch the delta
synced_file.close()
# crucial for 2 opens
tempFiles.append(file_name)
tempFiles.append(file_name)
logging.info("Updation Successful for file %s", file_name)
pm.update_db(db_conn, file_name, "client_m_time",
os.path.getmtime(file_name))
db_conn.commit()
#send server_m_time to client for update
ret_msg = pm.get_sendcmt_msg(client_id, file_name, db_conn)
#logging.info("returning msg for updating SMT: %s",ret_msg)
client_socket.send(ret_msg)
return 1
if msg_code == pm.msgCode.SENDDAT:
#create a new socket and send the data via it
data_socket = socket.socket()
addr = ('', C_DATA_SOCK_PORT)
data_socket.bind(addr)
#print "Client data socket is ready at: {}".format(data_socket.getsockname())
data_socket.listen(10)
client_data_sock, addr = data_socket.accept()
# directory traversing
# if sub directories do not exist then create accordingly
subpath = file_name.split('/')
for i in range(len(subpath) - 1):
dname = subpath[i]
if dname == '.':
continue
if os.path.exists(dname) is False:
os.mkdir(dname)
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server copy cannot be downloaded!" + pm.bcolors.ENDC
conflict[file_name] = 1
client_data_sock.close()
data_socket.close()
return 0
with open(file_name, 'wb') as f:
while True:
data = client_data_sock.recv(BUFFER_SIZE)
if not data:
break
f.write(data)
f.close()
tempFiles.append(file_name)
logging.info("file recieved: %s", file_name)
client_data_sock.close()
data_socket.close()
return 1
if msg_code == pm.msgCode.SREQ:
# SERVER sync
if os.path.exists(file_name) is True:
# if server file exists in the client directory
s_server_m_time, s_client_m_time = data.split('<##>')
c_client_m_time = pm.get_data(db_conn, file_name, "client_m_time")
c_server_m_time = pm.get_data(db_conn, file_name, "server_m_time")
# print "server: server_m_time ", s_server_m_time, "client_m_time ", s_client_m_time
# print "client: server_m_time ", c_server_m_time, "client_m_time ", c_client_m_time
if s_server_m_time > c_server_m_time:
# server has updated copy
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server copy cannot be downloaded!" + pm.bcolors.ENDC
conflict[file_name] = 1
return 0
msg = pm.get_sensig_msg(client_id, file_name, db_conn)
client_socket.send(msg)
send_signature(file_name)
return 1
elif s_server_m_time == c_server_m_time:
# same copy
msg = pm.get_sendnoc_msg(client_id, file_name, db_conn)
client_socket.send(msg)
return 1
else:
# send a request to send the total file
logging.info("Requesting File: %s", file_name)
sm_time, cm_time = data.split('<##>')
#print "timestamp", sm_time, cm_time
pm.update_db(db_conn, file_name, "client_m_time", cm_time)
pm.update_db(db_conn, file_name, "server_m_time", sm_time)
db_conn.commit()
msg = pm.get_reqtot_msg(client_id, file_name, db_conn)
client_socket.send(msg)
return 1
if msg_code == pm.msgCode.DELREQ:
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server action cannot be done!" + pm.bcolors.ENDC
delreq[file_name] = 1
return 0
if os.path.exists(file_name):
tempdelFiles.append(file_name)
if os.path.isdir(file_name): # if directory
shutil.rmtree(file_name)
else:
os.remove(file_name)
pm.delete_record(db_conn, file_name)
db_conn.commit()
logging.info("%s has been deleted successfully!", file_name)
return 0
if msg_code == pm.msgCode.MVREQ:
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server action cannot be done!" + pm.bcolors.ENDC
mvreq[file_name] = 1
return 0
#old filename = data
if os.path.exists(data):
os.rename(data, file_name)
if os.path.isdir(file_name) is False:
pm.update_db_filename(db_conn, data, file_name)
db_conn.commit()
tempmvFiles.append(file_name)
logging.info("%s is renamed/moved to %s", data, file_name)
return 0
if msg_code == pm.msgCode.CONFLICT:
print "The ", file_name, " is being accessed by some other client device! Updation is conflictiing"
return 1
if msg_code == pm.msgCode.TERMIN:
return 0 # close connection
def test_string_patch(self):
src_sig = librsync.signature(BytesIO(self.src))
delta = librsync.delta(BytesIO(self.dst), src_sig).read()
out = librsync.patch(BytesIO(self.src), BytesIO(delta))
self.assertEqual(self.dst, out.read())
def delta(self, signature):
"Generates delta for local file using remote signature."
return librsync.delta(open(self.path, 'rb'), signature)
def delta(self, signature):
"Generates delta for local file using remote signature."
return librsync.delta(open(self.path, 'rb'), signature)
def _sync(self,src,dst,dst_path): syn = open(dst_path,'wb') signature = librsync.signature(dst) delta = librsync.delta(src, signature) librsync.patch(dst,delta,syn) return True
