def test_patch(self): s = librsync.signature(self.rand1) d = librsync.delta(self.rand2, s) self.rand1.seek(0) self.rand2.seek(0) o = librsync.patch(self.rand1, d) self.assertEqual(o.read(), self.rand2.read())
def add_diff(self, signature, filename=None): if not filename: filename = self.name with open(filename, "r") as fp: delta = librsync.delta(fp, signature) # TODO: chunk big diffs self["data"] = delta.read() return self
def recieve_signature_and_send_delta(client_id, client, ip, file_name, db_conn): while pm.SharedPort.server_sig_port_used: continue pm.SharedPort.server_sig_port_used = True sig_socket = socket.socket() address = ('', pm.SharedPort.server_sig_port) sig_socket.bind(address) #print "Client sig socket is ready at: {}".format(data_socket.getsockname()) sig_socket.listen(10) client_sig_sock, addr = sig_socket.accept() # Receive Signature sigdata = "" while True: data = client_sig_sock.recv(BUFFER_SIZE) sigdata += data if not data: break client_sig_sock.close() sig_socket.close() pm.SharedPort.server_sig_port_used = False logging.info("Signature Received!") # logging.info("Sync-ing and uploading %s", file_name) # send SENDDEL msg msg = pm.get_senddel_msg(client_id, file_name, db_conn) client.send(msg) #calculate delta signature = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+') signature.write(sigdata) signature.seek(0) src = open(file_name, 'rb') delta = sync.delta(src, signature) # connect to delta socket client_del_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) wait_net_service(client_del_socket, ip, pm.SharedPort.client_del_port) # send delta l = delta.read(BUFFER_SIZE) while l: client_del_socket.send(l) l = delta.read(BUFFER_SIZE) delta.close() client_del_socket.close() logging.info("Delta Sent!")
def CalculateDelta(self, directory, fileName): if os.path.exists(directory + "/" + ".Delta_" + fileName): os.remove(directory + "/" + ".Delta_" + fileName) # The destination file. src = file(self.directoryPath + fileName, 'rb') # The source file. dst = file(directory + "/" + "." + fileName, 'rb') # Step 1: prepare signature of the destination file signature = librsync.signature(dst) # Step 2: prepare a delta of the source file delta = librsync.delta(src, signature) #Save delta in a file dill.dump(delta, open(directory + "/" + ".Delta_" + fileName, "w"))
def process_updated(path, updated, base_path, sigvault): """ Process upated files, create a new SigVault if needed, and create a new archives with delta (from the previous SigVault signatures). """ if updated: updated_archive = tarfile.open(path, 'w:gz') for f in updated: f_abs = os.path.join(base_path, f) delta = librsync.delta(open(f_abs, 'rb'), sigvault.extract(f)) delta_size = os.fstat(delta.fileno()).st_size delta_info = tarfile.TarInfo(f) delta_info.size = delta_size updated_archive.addfile(delta_info, delta) updated_archive.close() return path
def get(self, uid, name, field, item, buf): self._log('device->get, name=%s' % str(name)) if RSYNC: sig = StringIO(b64decode(buf)) mnt = get_mnt_path(uid) path = os.path.join(mnt, field, name, item) fd = os.open(path, os.O_RDONLY) try: res = os.read(fd, RECORD_LEN) except: log_err(self, 'failed to get, cannot read, name=%s' % str(name)) return finally: os.close(fd) if not res or len(res) == RECORD_LEN: log_err(self, 'failed to get, name=%s' % str(name)) return if RSYNC: tmp = StringIO(res) delta = librsync.delta(tmp, sig) res = delta.read() self._log('device->get, name=%s, finished' % str(name)) return self._encode(res)
def _sync(self, src, dst, dst_path): syn = open(dst_path, 'wb') signature = librsync.signature(dst) delta = librsync.delta(src, signature) librsync.patch(dst, delta, syn) return True
def test_signature(self): s = librsync.signature(self.rand1) d = librsync.delta(self.rand2, s)
#!/usr/bin/python import librsync import tempfile # get the file objects dst = file('old.SLDASM') src = file('new.SLDASM') synced = tempfile.SpooledTemporaryFile(max_size=500000000, mode='w+b') # do the librsync stuff signature = librsync.signature(dst) delta = librsync.delta(src, signature) librsync.patch(dst, delta, synced) # write the synced file synced.seek(0) syncedfile = open('synced.SLDASM', 'wb') buf = synced.read() syncedfile.write(buf) syncedfile.close() # write the signature to file signature.seek(0) sigfile = open('asm.signature', 'wb') buf = signature.read() sigfile.write(buf) sigfile.close() # write the delta to file delta.seek(0) delfile = open('asm.delta', 'wb')
def test_string_patch(self): src_sig = librsync.signature(BytesIO(self.src)) delta = librsync.delta(BytesIO(self.dst), src_sig).read() out = librsync.patch(BytesIO(self.src), BytesIO(delta)) self.assertEqual(self.dst, out.read())
#!/usr/bin/python import librsync import tempfile # get the file objects dst = file('old.SLDPRT') src = file('new.SLDPRT') synced = tempfile.SpooledTemporaryFile(max_size=500000000,mode='w+b') # do the librsync stuff signature = librsync.signature(dst) delta = librsync.delta(src, signature) librsync.patch(dst,delta,synced) # write the synced file synced.seek(0) syncedfile = open('synced.SLDPRT','wb') buf = synced.read() syncedfile.write(buf) syncedfile.close() # write the signature to file signature.seek(0) sigfile = open('prt.signature','wb') buf = signature.read() sigfile.write(buf) sigfile.close() # write the delta to file delta.seek(0)
def service_message(msg, client_socket, db_conn): global tempdelFiles, tempFiles, tempmvFiles, locked, conflict, delreq, mvreq global SERVER_IP if db_conn is None: db_conn = pm.open_db() msg_code, client_id, file_name, data = msg.split(pm.msgCode.delim) if msg_code == pm.msgCode.REQTOT: header = pm.get_senddat_msg(client_id, file_name, db_conn) #logging.info("sending : header = %s", header) client_socket.send(header) client_data_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #logging.debug("waiting for data socket to be ready..") wait_net_service(client_data_socket, SERVER_IP, S_DATA_SOCK_PORT) #client_data_socket.connect(server_data_address) #logging.debug("Connected") with open(file_name, 'rb') as f: l = f.read(BUFFER_SIZE) while l: client_data_socket.send(l) l = f.read(BUFFER_SIZE) f.close() logging.debug("file sent: %s", file_name) client_data_socket.close() return 1 if msg_code == pm.msgCode.SENDSMT: #logging.info("updating server_m_time of %s to %s",file_name,data) pm.update_db(db_conn, file_name, "server_m_time", data) db_conn.commit() return 0 if msg_code == pm.msgCode.SENDSIG: #compute delta and send sig_socket = socket.socket() addr = ('', pm.SharedPort.client_sig_port) sig_socket.bind(addr) #print "Client sig socket is ready at: {}".format(data_socket.getsockname()) sig_socket.listen(10) server_sig_sock, addr = sig_socket.accept() # Receive Signature sigdata = "" while True: data = server_sig_sock.recv(BUFFER_SIZE) sigdata += data if not data: break server_sig_sock.close() sig_socket.close() logging.info("Signature Received!") # logging.info("Sync-ing and uploading %s", file_name) # send SENDDEL msg msg = pm.get_senddel_msg(client_id, file_name, db_conn) client_socket.send(msg) signature = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+') signature.write(sigdata) signature.seek(0) src = open(file_name, 'rb') delta = sync.delta(src, signature) delta.seek(0) # connect to delta socket client_del_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) wait_net_service(client_del_socket, SERVER_IP, pm.SharedPort.server_del_port) # send delta l = delta.read(BUFFER_SIZE) while l: client_del_socket.send(l) l = delta.read(BUFFER_SIZE) delta.close() client_del_socket.close() logging.info("Delta Sent!") return 1 if msg_code == pm.msgCode.REQSIG: # next update : create a data socket to send large signature msg = pm.get_sensig_msg(client_id, file_name, db_conn) client_socket.send(msg) send_signature(file_name) return 1 if msg_code == pm.msgCode.SENDDEL: # receive delta del_socket = socket.socket() addr = ('', pm.SharedPort.client_del_port) del_socket.bind(addr) #print "Client del socket is ready at: {}".format(data_socket.getsockname()) del_socket.listen(10) client_del_sock, addr = del_socket.accept() # Receive Signature deldata = "" while True: data = client_del_sock.recv(BUFFER_SIZE) deldata += data if not data: break client_del_sock.close() del_socket.close() logging.info("Delta has been received!") delta = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+') delta.write(deldata) delta.seek(0) ### Ask to Merge ### Hard to produce this case ### Happens when both updates are done at a same time instance last_m_time = os.path.getmtime(file_name) db_m_time = pm.get_data(db_conn, file_name, "client_m_time") if last_m_time > db_m_time: print pm.bcolors.FAIL + "WARNING!" print file_name, " has changed since last update. Do you want to merge server's update? [Y|N]" print "Local updates will be lost if you select 'Yes'" + pm.bcolors.ENDC ans = raw_input() if ans == 'n' or ans == 'N': return 1 dest = open(file_name, 'rb') synced_file = open(file_name, 'wb') sync.patch(dest, delta, synced_file) # patch the delta synced_file.close() # crucial for 2 opens tempFiles.append(file_name) tempFiles.append(file_name) logging.info("Updation Successful for file %s", file_name) pm.update_db(db_conn, file_name, "client_m_time", os.path.getmtime(file_name)) db_conn.commit() #send server_m_time to client for update ret_msg = pm.get_sendcmt_msg(client_id, file_name, db_conn) #logging.info("returning msg for updating SMT: %s",ret_msg) client_socket.send(ret_msg) return 1 if msg_code == pm.msgCode.SENDDAT: #create a new socket and send the data via it data_socket = socket.socket() addr = ('', C_DATA_SOCK_PORT) data_socket.bind(addr) #print "Client data socket is ready at: {}".format(data_socket.getsockname()) data_socket.listen(10) client_data_sock, addr = data_socket.accept() # directory traversing # if sub directories do not exist then create accordingly subpath = file_name.split('/') for i in range(len(subpath) - 1): dname = subpath[i] if dname == '.': continue if os.path.exists(dname) is False: os.mkdir(dname) if file_name in locked and locked[file_name] == 1: print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server copy cannot be downloaded!" + pm.bcolors.ENDC conflict[file_name] = 1 client_data_sock.close() data_socket.close() return 0 with open(file_name, 'wb') as f: while True: data = client_data_sock.recv(BUFFER_SIZE) if not data: break f.write(data) f.close() tempFiles.append(file_name) logging.info("file recieved: %s", file_name) client_data_sock.close() data_socket.close() return 1 if msg_code == pm.msgCode.SREQ: # SERVER sync if os.path.exists(file_name) is True: # if server file exists in the client directory s_server_m_time, s_client_m_time = data.split('<##>') c_client_m_time = pm.get_data(db_conn, file_name, "client_m_time") c_server_m_time = pm.get_data(db_conn, file_name, "server_m_time") # print "server: server_m_time ", s_server_m_time, "client_m_time ", s_client_m_time # print "client: server_m_time ", c_server_m_time, "client_m_time ", c_client_m_time if s_server_m_time > c_server_m_time: # server has updated copy if file_name in locked and locked[file_name] == 1: print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server copy cannot be downloaded!" + pm.bcolors.ENDC conflict[file_name] = 1 return 0 msg = pm.get_sensig_msg(client_id, file_name, db_conn) client_socket.send(msg) send_signature(file_name) return 1 elif s_server_m_time == c_server_m_time: # same copy msg = pm.get_sendnoc_msg(client_id, file_name, db_conn) client_socket.send(msg) return 1 else: # send a request to send the total file logging.info("Requesting File: %s", file_name) sm_time, cm_time = data.split('<##>') #print "timestamp", sm_time, cm_time pm.update_db(db_conn, file_name, "client_m_time", cm_time) pm.update_db(db_conn, file_name, "server_m_time", sm_time) db_conn.commit() msg = pm.get_reqtot_msg(client_id, file_name, db_conn) client_socket.send(msg) return 1 if msg_code == pm.msgCode.DELREQ: if file_name in locked and locked[file_name] == 1: print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server action cannot be done!" + pm.bcolors.ENDC delreq[file_name] = 1 return 0 if os.path.exists(file_name): tempdelFiles.append(file_name) if os.path.isdir(file_name): # if directory shutil.rmtree(file_name) else: os.remove(file_name) pm.delete_record(db_conn, file_name) db_conn.commit() logging.info("%s has been deleted successfully!", file_name) return 0 if msg_code == pm.msgCode.MVREQ: if file_name in locked and locked[file_name] == 1: print pm.bcolors.FAIL + "CONFLICT : Local copy is currently being modified! Server action cannot be done!" + pm.bcolors.ENDC mvreq[file_name] = 1 return 0 #old filename = data if os.path.exists(data): os.rename(data, file_name) if os.path.isdir(file_name) is False: pm.update_db_filename(db_conn, data, file_name) db_conn.commit() tempmvFiles.append(file_name) logging.info("%s is renamed/moved to %s", data, file_name) return 0 if msg_code == pm.msgCode.CONFLICT: print "The ", file_name, " is being accessed by some other client device! Updation is conflictiing" return 1 if msg_code == pm.msgCode.TERMIN: return 0 # close connection
def delta(self, signature): "Generates delta for local file using remote signature." return librsync.delta(open(self.path, 'rb'), signature)
def _sync(self,src,dst,dst_path): syn = open(dst_path,'wb') signature = librsync.signature(dst) delta = librsync.delta(src, signature) librsync.patch(dst,delta,syn) return True