def recieve_delta_and_patch(file_name):
# receive delta
del_socket = socket.socket()
address = ('', pm.SharedPort.server_del_port)
del_socket.bind(address)
#print "Client del socket is ready at: {}".format(data_socket.getsockname())
del_socket.listen(10)
client_del_sock, addr = del_socket.accept()
# Receive delta
deldata = ""
while True:
data = client_del_sock.recv(BUFFER_SIZE)
deldata += data
if not data:
break
client_del_sock.close()
del_socket.close()
logging.info("Delta has been received!")
delta = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+')
delta.write(deldata)
delta.seek(0)
dest = open(file_name, 'rb')
synced_file = open(file_name, 'wb')
sync.patch(dest, delta, synced_file) # patch the delta
synced_file.close()
logging.info("Updation Successful for file %s", file_name)
def PatchLocalFile(self, deltaPath):
finalLocalName = self.GenerateLocalFileName(deltaPath)
finalLocalFilePath = "/home/"+getpass.getuser()+"/RemoteSync/"+finalLocalName
finalLocalHiddenFilePath = self.tempPath+"/."+finalLocalName
if os.path.exists(finalLocalFilePath):
os.remove(finalLocalFilePath)
open(finalLocalFilePath, "w").close
restoredDelta = dill.load(open(deltaPath)) #Open delta from synced file
dst = file(finalLocalFilePath, 'r+')
librsync.patch(dst, restoredDelta, dst) #Patching a local file using the received Delta file
self.Cleanup(deltaPath)
dst.close()
def test_patch(self):
s = librsync.signature(self.rand1)
d = librsync.delta(self.rand2, s)
self.rand1.seek(0)
self.rand2.seek(0)
o = librsync.patch(self.rand1, d)
self.assertEqual(o.read(), self.rand2.read())
def test_patch(self):
s = librsync.signature(self.rand1)
d = librsync.delta(self.rand2, s)
self.rand1.seek(0)
self.rand2.seek(0)
o = librsync.patch(self.rand1, d)
self.assertEqual(o.read(), self.rand2.read())
def patch(filename, delta):
with open(filename, "r") as dst:
o = None
try:
fd, tmp_filename = mkstemp(dir=os.path.dirname(filename))
o = librsync.patch(dst, delta, os.fdopen(fd, "w"))
finally:
if o:
o.close()
os.rename(tmp_filename, filename)
def patch(self, uid, name, buf):
if not buf:
log_warnning(self, 'cannot patch, no content, name=%s' % str(name))
return
dest = self.get_path(uid, name)
src = get_temp(dest)
tmp = b64decode(buf)
if RSYNC:
delta = StringIO(tmp)
with open(dest, 'wb') as f_dest:
with open(src, 'rb') as f_src:
try:
librsync.patch(f_src, delta, f_dest)
except:
log_warnning(self, 'failed to patch, name=%s' % str(name))
self._fs.rename(uid, src, dest)
return
self._fs.remove(uid, src)
else:
with open(src, 'wb') as f:
f.write(tmp)
self._fs.rename(uid, src, dest)
def patch(self, delta):
"Applies remote delta to local file."
# Create a temp file in which to store our synced copy. We will handle
# deleting it manually, since we may move it instead.
with (tempfile.NamedTemporaryFile(prefix='.sync',
suffix=os.path.basename(self.path),
dir=os.path.dirname(self.path), delete=False)) as output:
try:
# Open the local file, data may be read from it.
with open(self.path, 'rb') as reference:
# Patch the local file into our temporary file.
r = librsync.patch(reference, delta, output)
os.rename(output.name, self.path)
return r
finally:
try:
os.remove(output.name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def patch(self, delta):
"Applies remote delta to local file."
# Create a temp file in which to store our synced copy. We will handle
# deleting it manually, since we may move it instead.
with (tempfile.NamedTemporaryFile(prefix='.sync',
suffix=os.path.basename(self.path),
dir=os.path.dirname(self.path),
delete=False)) as output:
try:
# Open the local file, data may be read from it.
with open(self.path, 'rb') as reference:
# Patch the local file into our temporary file.
r = librsync.patch(reference, delta, output)
os.rename(output.name, self.path)
return r
finally:
try:
os.remove(output.name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
#!/usr/bin/python
import librsync
import tempfile
# get the file objects
dst = file('old.SLDASM')
src = file('new.SLDASM')
synced = tempfile.SpooledTemporaryFile(max_size=500000000, mode='w+b')
# do the librsync stuff
signature = librsync.signature(dst)
delta = librsync.delta(src, signature)
librsync.patch(dst, delta, synced)
# write the synced file
synced.seek(0)
syncedfile = open('synced.SLDASM', 'wb')
buf = synced.read()
syncedfile.write(buf)
syncedfile.close()
# write the signature to file
signature.seek(0)
sigfile = open('asm.signature', 'wb')
buf = signature.read()
sigfile.write(buf)
sigfile.close()
# write the delta to file
delta.seek(0)
delfile = open('asm.delta', 'wb')
def test_string_patch(self):
src_sig = librsync.signature(BytesIO(self.src))
delta = librsync.delta(BytesIO(self.dst), src_sig).read()
out = librsync.patch(BytesIO(self.src), BytesIO(delta))
self.assertEqual(self.dst, out.read())
def _sync(self, src, dst, dst_path):
syn = open(dst_path, 'wb')
signature = librsync.signature(dst)
delta = librsync.delta(src, signature)
librsync.patch(dst, delta, syn)
return True
#!/usr/bin/python
import librsync
import tempfile
# get the file objects
dst = file('old.SLDPRT')
src = file('new.SLDPRT')
synced = tempfile.SpooledTemporaryFile(max_size=500000000,mode='w+b')
# do the librsync stuff
signature = librsync.signature(dst)
delta = librsync.delta(src, signature)
librsync.patch(dst,delta,synced)
# write the synced file
synced.seek(0)
syncedfile = open('synced.SLDPRT','wb')
buf = synced.read()
syncedfile.write(buf)
syncedfile.close()
# write the signature to file
signature.seek(0)
sigfile = open('prt.signature','wb')
buf = signature.read()
sigfile.write(buf)
sigfile.close()
# write the delta to file
delta.seek(0)
def patch_diff(base_path, diff, created_archive=None, updated_archive=None):
# First, we iterate the created files
if diff['created']:
for crtd in diff['created']:
created_tar = tarfile.open(created_archive, 'r:gz')
try:
src_file = created_tar.extractfile(crtd)
abspath = os.path.join(base_path, crtd)
dirname = os.path.dirname(abspath)
# Create directories if they doesn't exist yet
if not os.path.exists(dirname):
os.makedirs(dirname)
# We copy the file from the archive directly to its destination
with open(abspath, 'wb') as f:
shutil.copyfileobj(src_file, f)
except KeyError as exc:
# It means that a file is missing in the archive.
log.exception(exc)
raise Exception("Diff seems corrupted.")
finally:
created_tar.close()
# Next, we iterate updated files in order to patch them
if diff['updated']:
for updtd in diff['updated']:
try:
updated_tar = tarfile.open(updated_archive, 'r:gz')
abspath = os.path.join(base_path, updtd)
# Load the librsync delta
delta_file = updated_tar.extractfile(updtd)
# A tempfile file to store the patched file/result
# before replacing the original
patched = tempfile.NamedTemporaryFile()
# Patch the current version of the file with the delta
# and store the result in the previously created tempfile
with open(abspath, 'rb') as f:
librsync.patch(f, delta_file, patched)
patched.seek(0)
# Now we replace the orignal file with the patched version
with open(abspath, 'wb') as f:
shutil.copyfileobj(patched, f)
patched.close()
except KeyError as exc:
# It means that a file is missing in the archive.
log.exception(exc)
raise Exception("Diff seems corrupted.")
finally:
updated_tar.close()
# Then, we iterate the deleted files
for dltd in diff['deleted']:
abspath = os.path.join(base_path, dltd)
if os.path.isfile(abspath):
os.remove(abspath)
# Finally, we iterate the deleted directories
for dltd_drs in diff['deleted_dirs']:
abspath = os.path.join(base_path, dltd_drs)
if os.path.isdir(abspath):
os.rmdir(abspath)
def patch_diff(base_path, diff, created_archive=None, updated_archive=None):
# First, we iterate the created files
if diff['created']:
for crtd in diff['created']:
created_tar = tarfile.open(created_archive, 'r:gz')
try:
src_file = created_tar.extractfile(crtd)
abspath = os.path.join(base_path, crtd)
dirname = os.path.dirname(abspath)
# Create directories if they doesn't exist yet
if not os.path.exists(dirname):
os.makedirs(dirname)
# We copy the file from the archive directly to its destination
with open(abspath, 'wb') as f:
shutil.copyfileobj(src_file, f)
except KeyError as exc:
# It means that a file is missing in the archive.
log.exception(exc)
raise Exception("Diff seems corrupted.")
finally:
created_tar.close()
# Next, we iterate updated files in order to patch them
if diff['updated']:
for updtd in diff['updated']:
try:
updated_tar = tarfile.open(updated_archive, 'r:gz')
abspath = os.path.join(base_path, updtd)
# Load the librsync delta
delta_file = updated_tar.extractfile(updtd)
# A tempfile file to store the patched file/result
# before replacing the original
patched = tempfile.NamedTemporaryFile()
# Patch the current version of the file with the delta
# and store the result in the previously created tempfile
with open(abspath, 'rb') as f:
librsync.patch(f, delta_file, patched)
patched.seek(0)
# Now we replace the orignal file with the patched version
with open(abspath, 'wb') as f:
shutil.copyfileobj(patched, f)
patched.close()
except KeyError as exc:
# It means that a file is missing in the archive.
log.exception(exc)
raise Exception("Diff seems corrupted.")
finally:
updated_tar.close()
# Then, we iterate the deleted files
for dltd in diff['deleted']:
abspath = os.path.join(base_path, dltd)
if os.path.isfile(abspath):
os.remove(abspath)
# Finally, we iterate the deleted directories
for dltd_drs in diff['deleted_dirs']:
abspath = os.path.join(base_path, dltd_drs)
if os.path.isdir(abspath):
os.rmdir(abspath)
def service_message(msg, client_socket, db_conn):
global tempdelFiles, tempFiles, tempmvFiles, locked, conflict, delreq, mvreq
global SERVER_IP
if db_conn is None:
db_conn = pm.open_db()
msg_code, client_id, file_name, data = msg.split(pm.msgCode.delim)
if msg_code == pm.msgCode.REQTOT:
header = pm.get_senddat_msg(client_id, file_name, db_conn)
#logging.info("sending : header = %s", header)
client_socket.send(header)
client_data_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#logging.debug("waiting for data socket to be ready..")
wait_net_service(client_data_socket, SERVER_IP, S_DATA_SOCK_PORT)
#client_data_socket.connect(server_data_address)
#logging.debug("Connected")
with open(file_name, 'rb') as f:
l = f.read(BUFFER_SIZE)
while l:
client_data_socket.send(l)
l = f.read(BUFFER_SIZE)
f.close()
logging.debug("file sent: %s", file_name)
client_data_socket.close()
return 1
if msg_code == pm.msgCode.SENDSMT:
#logging.info("updating server_m_time of %s to %s",file_name,data)
pm.update_db(db_conn, file_name, "server_m_time", data)
db_conn.commit()
return 0
if msg_code == pm.msgCode.SENDSIG:
#compute delta and send
sig_socket = socket.socket()
addr = ('', pm.SharedPort.client_sig_port)
sig_socket.bind(addr)
#print "Client sig socket is ready at: {}".format(data_socket.getsockname())
sig_socket.listen(10)
server_sig_sock, addr = sig_socket.accept()
# Receive Signature
sigdata = ""
while True:
data = server_sig_sock.recv(BUFFER_SIZE)
sigdata += data
if not data:
break
server_sig_sock.close()
sig_socket.close()
logging.info("Signature Received!")
# logging.info("Sync-ing and uploading %s", file_name)
# send SENDDEL msg
msg = pm.get_senddel_msg(client_id, file_name, db_conn)
client_socket.send(msg)
signature = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL,
mode='wb+')
signature.write(sigdata)
signature.seek(0)
src = open(file_name, 'rb')
delta = sync.delta(src, signature)
delta.seek(0)
# connect to delta socket
client_del_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
wait_net_service(client_del_socket, SERVER_IP,
pm.SharedPort.server_del_port)
# send delta
l = delta.read(BUFFER_SIZE)
while l:
client_del_socket.send(l)
l = delta.read(BUFFER_SIZE)
delta.close()
client_del_socket.close()
logging.info("Delta Sent!")
return 1
if msg_code == pm.msgCode.REQSIG:
# next update : create a data socket to send large signature
msg = pm.get_sensig_msg(client_id, file_name, db_conn)
client_socket.send(msg)
send_signature(file_name)
return 1
if msg_code == pm.msgCode.SENDDEL:
# receive delta
del_socket = socket.socket()
addr = ('', pm.SharedPort.client_del_port)
del_socket.bind(addr)
#print "Client del socket is ready at: {}".format(data_socket.getsockname())
del_socket.listen(10)
client_del_sock, addr = del_socket.accept()
# Receive Signature
deldata = ""
while True:
data = client_del_sock.recv(BUFFER_SIZE)
deldata += data
if not data:
break
client_del_sock.close()
del_socket.close()
logging.info("Delta has been received!")
delta = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+')
delta.write(deldata)
delta.seek(0)
### Ask to Merge
### Hard to produce this case
### Happens when both updates are done at a same time instance
last_m_time = os.path.getmtime(file_name)
db_m_time = pm.get_data(db_conn, file_name, "client_m_time")
if last_m_time > db_m_time:
print pm.bcolors.FAIL + "WARNING!"
print file_name, " has changed since last update. Do you want to merge server's update? [Y|N]"
print "Local updates will be lost if you select 'Yes'" + pm.bcolors.ENDC
ans = raw_input()
if ans == 'n' or ans == 'N':
return 1
dest = open(file_name, 'rb')
synced_file = open(file_name, 'wb')
sync.patch(dest, delta, synced_file) # patch the delta
synced_file.close()
# crucial for 2 opens
tempFiles.append(file_name)
tempFiles.append(file_name)
logging.info("Updation Successful for file %s", file_name)
pm.update_db(db_conn, file_name, "client_m_time",
os.path.getmtime(file_name))
db_conn.commit()
#send server_m_time to client for update
ret_msg = pm.get_sendcmt_msg(client_id, file_name, db_conn)
#logging.info("returning msg for updating SMT: %s",ret_msg)
client_socket.send(ret_msg)
return 1
if msg_code == pm.msgCode.SENDDAT:
#create a new socket and send the data via it
data_socket = socket.socket()
addr = ('', C_DATA_SOCK_PORT)
data_socket.bind(addr)
#print "Client data socket is ready at: {}".format(data_socket.getsockname())
data_socket.listen(10)
client_data_sock, addr = data_socket.accept()
# directory traversing
# if sub directories do not exist then create accordingly
subpath = file_name.split('/')
for i in range(len(subpath) - 1):
dname = subpath[i]
if dname == '.':
continue
if os.path.exists(dname) is False:
os.mkdir(dname)
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server copy cannot be downloaded!" + pm.bcolors.ENDC
conflict[file_name] = 1
client_data_sock.close()
data_socket.close()
return 0
with open(file_name, 'wb') as f:
while True:
data = client_data_sock.recv(BUFFER_SIZE)
if not data:
break
f.write(data)
f.close()
tempFiles.append(file_name)
logging.info("file recieved: %s", file_name)
client_data_sock.close()
data_socket.close()
return 1
if msg_code == pm.msgCode.SREQ:
# SERVER sync
if os.path.exists(file_name) is True:
# if server file exists in the client directory
s_server_m_time, s_client_m_time = data.split('<##>')
c_client_m_time = pm.get_data(db_conn, file_name, "client_m_time")
c_server_m_time = pm.get_data(db_conn, file_name, "server_m_time")
# print "server: server_m_time ", s_server_m_time, "client_m_time ", s_client_m_time
# print "client: server_m_time ", c_server_m_time, "client_m_time ", c_client_m_time
if s_server_m_time > c_server_m_time:
# server has updated copy
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server copy cannot be downloaded!" + pm.bcolors.ENDC
conflict[file_name] = 1
return 0
msg = pm.get_sensig_msg(client_id, file_name, db_conn)
client_socket.send(msg)
send_signature(file_name)
return 1
elif s_server_m_time == c_server_m_time:
# same copy
msg = pm.get_sendnoc_msg(client_id, file_name, db_conn)
client_socket.send(msg)
return 1
else:
# send a request to send the total file
logging.info("Requesting File: %s", file_name)
sm_time, cm_time = data.split('<##>')
#print "timestamp", sm_time, cm_time
pm.update_db(db_conn, file_name, "client_m_time", cm_time)
pm.update_db(db_conn, file_name, "server_m_time", sm_time)
db_conn.commit()
msg = pm.get_reqtot_msg(client_id, file_name, db_conn)
client_socket.send(msg)
return 1
if msg_code == pm.msgCode.DELREQ:
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server action cannot be done!" + pm.bcolors.ENDC
delreq[file_name] = 1
return 0
if os.path.exists(file_name):
tempdelFiles.append(file_name)
if os.path.isdir(file_name): # if directory
shutil.rmtree(file_name)
else:
os.remove(file_name)
pm.delete_record(db_conn, file_name)
db_conn.commit()
logging.info("%s has been deleted successfully!", file_name)
return 0
if msg_code == pm.msgCode.MVREQ:
if file_name in locked and locked[file_name] == 1:
print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server action cannot be done!" + pm.bcolors.ENDC
mvreq[file_name] = 1
return 0
#old filename = data
if os.path.exists(data):
os.rename(data, file_name)
if os.path.isdir(file_name) is False:
pm.update_db_filename(db_conn, data, file_name)
db_conn.commit()
tempmvFiles.append(file_name)
logging.info("%s is renamed/moved to %s", data, file_name)
return 0
if msg_code == pm.msgCode.CONFLICT:
print "The ", file_name, " is being accessed by some other client device! Updation is conflictiing"
return 1
if msg_code == pm.msgCode.TERMIN:
return 0 # close connection
def test_string_patch(self):
src_sig = librsync.signature(BytesIO(self.src))
delta = librsync.delta(BytesIO(self.dst), src_sig).read()
out = librsync.patch(BytesIO(self.src), BytesIO(delta))
self.assertEqual(self.dst, out.read())
def _sync(self,src,dst,dst_path): syn = open(dst_path,'wb') signature = librsync.signature(dst) delta = librsync.delta(src, signature) librsync.patch(dst,delta,syn) return True