def rsync(self, job): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): if self.test: print "Error: the path %s does not exists" % job['path'] return False, {} args = [ 'rsync', '-a', '--whole-file', '--human-readable', '--xattrs', '--ignore-existing', ] node = job['node'] node_ip = rsync_ip(node['replication_ip']) rsync_module = '%s:%s' % (node_ip, job['remote_path']) args.append(job['path']) args.append(rsync_module) if not self.test: return self._rsync(args) == 0, {} else: print " ".join(args) return True, {}
def create_remote_directory(self, job): """ Creates a temporal directory, at remote server. :param job: information about the partition being synced """ node = job['node'] args = [ "ssh", rsync_ip(node['replication_ip']), "mkdir", "-p", job['remote_path'] ] if not self.test: proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) results = proc.stdout.read() ret_val = proc.wait() #TODO: ret_val check (results, ret_val) else: print " ".join(args)
def _rsync_db(self, broker, device, http, local_id, replicate_method='complete_rsync', replicate_timeout=None): """ Sync a whole db using rsync. :param broker: DB broker object of DB to be synced :param device: device to sync to :param http: ReplConnection object :param local_id: unique ID of the local database replica :param replicate_method: remote operation to perform after rsync :param replicate_timeout: timeout to wait in seconds """ device_ip = rsync_ip(device['replication_ip']) if self.vm_test_mode: remote_file = '%s::%s%s/%s/tmp/%s' % ( device_ip, self.server_type, device['replication_port'], device['device'], local_id) else: remote_file = '%s::%s/%s/tmp/%s' % ( device_ip, self.server_type, device['device'], local_id) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, False): return False with Timeout(replicate_timeout or self.node_timeout): response = http.replicate(replicate_method, local_id) return response and response.status >= 200 and response.status < 300
def _rsync_db(self, broker, device, http, local_id, replicate_method='complete_rsync', replicate_timeout=None): """ Sync a whole db using rsync. :param broker: DB broker object of DB to be synced :param device: device to sync to :param http: ReplConnection object :param local_id: unique ID of the local database replica :param replicate_method: remote operation to perform after rsync :param replicate_timeout: timeout to wait in seconds """ device_ip = rsync_ip(device['ip']) if self.vm_test_mode: remote_file = '%s::%s%s/%s/tmp/%s' % (device_ip, self.server_type, device['port'], device['device'], local_id) else: remote_file = '%s::%s/%s/tmp/%s' % (device_ip, self.server_type, device['device'], local_id) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, False): return False with Timeout(replicate_timeout or self.node_timeout): response = http.replicate(replicate_method, local_id) return response and response.status >= 200 and response.status < 300
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False args.append(join(rsync_module, node['device'], 'objects', job['partition'])) return self._rsync(args) == 0
def create_remote_directory(self, job): """ Creates a temporal directory, at remote server. :param job: information about the partition being synced """ node = job['node'] args = ["ssh", rsync_ip(node['replication_ip']), "mkdir", "-p", job['remote_path']] if not self.test: proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) results = proc.stdout.read() ret_val = proc.wait() #TODO: ret_val check (results, ret_val) else: print " ".join(args)
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False args.append( join(rsync_module, node['device'], 'objects', job['partition'])) return self._rsync(args) == 0
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job["path"]): return False args = [ "rsync", "--recursive", "--whole-file", "--human-readable", "--xattrs", "--itemize-changes", "--ignore-existing", "--timeout=%s" % self.rsync_io_timeout, "--contimeout=%s" % self.rsync_io_timeout, "--bwlimit=%s" % self.rsync_bwlimit, ] node_ip = rsync_ip(node["replication_ip"]) if self.vm_test_mode: rsync_module = "%s::object%s" % (node_ip, node["replication_port"]) else: rsync_module = "%s::object" % node_ip had_any = False for suffix in suffixes: spath = join(job["path"], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False data_dir = get_data_dir(job["policy_idx"]) args.append(join(rsync_module, node["device"], data_dir, job["partition"])) return self._rsync(args) == 0
def rsync(partition, device): node_ip = rsync_ip('127.0.0.1') rsync_module = '%s::object' % (node_ip) spath = join('/SSD/%s' % partition) args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=30' ] args.append(spath) # args.append(join(rsync_module,'srv/node',device['device'],'objects',partition)) args.append( join(rsync_module, '/srv/node', device['device'], 'objects', partition)) start_time = time.time() ret_val = None try: with Timeout(900): proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) results = proc.stdout.read() ret_val = proc.wait() except Timeout: print("Killing long-running rsync: %s", str(args)) proc.kill() return 1 total_time = time.time() - start_time for result in results.split('\n'): if result == '': continue if result.startswith('cd+'): continue if not ret_val: print(result) else: print(result) if ret_val: print('Bad rsync return code: %(args)s -> %(ret)d', { 'args': str(args), 'ret': ret_val }) elif results: print("Successful rsync of %(src)s at %(dst)s (%(time).03f)", { 'src': args[-2], 'dst': args[-1], 'time': total_time }) else: print("Successful rsync of %(src)s at %(dst)s (%(time).03f)", { 'src': args[-2], 'dst': args[-1], 'time': total_time })
def rsync(partition, device, suffix): node_ip = rsync_ip('127.0.0.1') port='' if device == 'd2': port = '6020' if device == 'd3': port = '6030' rsync_module = '%s::object%s' %(node_ip,port) spath = join('/srv/node/ssd/objects/%s' %(partition),suffix[1]) print(suffix) args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=30', '--contimeout=30', ] args.append(spath) args.append(join(rsync_module, device, 'objects', partition)) start_time = time.time() ret_val = None try: with Timeout(900): proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) results = proc.stdout.read() ret_val = proc.wait() except Timeout: print("Killing long-running rsync: %s", str(args)) proc.kill() return 1 total_time = time.time() - start_time for result in results.split('\n'): if result == '': continue if result.startswith('cd+'): continue if not ret_val: print(result) else: print(result) if ret_val: print('Bad rsync return code: %(args)s -> %(ret)d', {'args': str(args), 'ret': ret_val}) elif results: print("Successful rsync of %(src)s at %(dst)s (%(time).03f)", {'src': args[-2], 'dst': args[-1], 'time': total_time}) else: print("Successful rsync of %(src)s at %(dst)s (%(time).03f)", {'src': args[-2], 'dst': args[-1], 'time': total_time}) return ret_val
def rsync(partition, device): node_ip = rsync_ip(device['ip']) rsync_module = 'swift@%s:' %(node_ip) spath = '/SSD/'+str(partition) args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=30', '--archive' ] args.append(spath) # args.append(join(rsync_module,'srv/node',device['device'],'objects',partition)) args.append(join(rsync_module,'srv/node',device['device'],'objects/')) logging.info("===args===%s",str(args)) start_time = time.time() ret_val = None try: with Timeout(900): proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) results = proc.stdout.read() ret_val = proc.wait() except Timeout: logging.info("Killing long-running rsync: %s", str(args)) proc.kill() return 1 total_time = time.time() - start_time logging.info("===Total time===%s",str(total_time)) for result in results.split('\n'): if result == '': continue if result.startswith('cd+'): continue if not ret_val: logging.info(result) else: logging.info(result) if ret_val: logging.info('Bad rsync return code: %(args)s -> %(ret)d', {'args': str(args), 'ret': ret_val}) elif results: logging.info("Successful rsync of %(src)s at %(dst)s (%(time).03f)", {'src': args[-2], 'dst': args[-1], 'time': total_time}) else: logging.info("Successful rsync of %(src)s at %(dst)s (%(time).03f)", {'src': args[-2], 'dst': args[-1], 'time': total_time})
def rsync(self, node, job, suffixes): """ 从一个远程结点上的partition同步一个文件 Synchronize local suffix directories from a partition with a remote node. :param node: the "dev" entry for the remote node to sync with :param job: information about the partition being synced :param suffixes: a list of suffixes which need to be pushed :returns: boolean indicating success or failure """ if not os.path.exists(job['path']): return False args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False args.append(join(rsync_module, node['device'], 'objects', job['partition'])) return self._rsync(args) == 0
def rsync(self, node, job, suffixes): """ Synchronize local suffix directories from a partition with a remote node. :param node: the "dev" entry for the remote node to sync with :param job: information about the partition being synced :param suffixes: a list of suffixes which need to be pushed :returns: boolean indicating success or failure """ if not os.path.exists(job['path']): return False args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False args.append( join(rsync_module, node['device'], 'objects', job['partition'])) return self._rsync(args) == 0
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False, {} args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] if self.rsync_compress and \ job['region'] != node['region']: # Allow for compression, but only if the remote node is in # a different region than the local one. args.append('--compress') node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False, {} data_dir = get_data_dir(job['policy']) args.append( join(rsync_module, node['device'], data_dir, job['partition'])) return self._rsync(args) == 0, {}
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) f = open("/home/hduser/log.txt","w") f.write(str(calframe)) f.close() if not os.path.exists(job['path']): return False, set() args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False, set() data_dir = get_data_dir(job['policy_idx']) args.append(join(rsync_module, node['device'], data_dir, job['partition'])) return self._rsync(args) == 0, set()
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False, {} args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, ] if self.rsync_compress and \ job['region'] != node['region']: # Allow for compression, but only if the remote node is in # a different region than the local one. args.append('--compress') node_ip = rsync_ip(node['replication_ip']) if self.vm_test_mode: rsync_module = '%s::object%s' % (node_ip, node['replication_port']) else: rsync_module = '%s::object' % node_ip had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False, {} data_dir = get_data_dir(job['policy']) args.append(join(rsync_module, node['device'], data_dir, job['partition'])) return self._rsync(args) == 0, {}
def rsync(self, node, job, suffixes): """ Synchronize local suffix directories from a partition with a remote node. :param node: the "dev" entry for the remote node to sync with :param job: information about the partition being synced :param suffixes: a list of suffixes which need to be pushed :returns: boolean indicating success or failure """ if not os.path.exists(job["path"]): return False args = [ "rsync", "--recursive", "--whole-file", "--human-readable", "--xattrs", "--itemize-changes", "--ignore-existing", "--timeout=%s" % self.rsync_io_timeout, "--contimeout=%s" % self.rsync_io_timeout, ] node_ip = rsync_ip(node["ip"]) if self.vm_test_mode: rsync_module = "%s::object%s" % (node_ip, node["port"]) else: rsync_module = "%s::object" % node_ip had_any = False for suffix in suffixes: spath = join(job["path"], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False args.append(join(rsync_module, node["device"], "objects", job["partition"])) return self._rsync(args) == 0
def test_rsync_ip_ipv4_localhost(self): self.assertEqual(utils.rsync_ip('127.0.0.1'), '127.0.0.1')
def test_rsync_ip_ipv6_ipv4_compatible(self): self.assertEqual( utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]')
def test_rsync_ip_ipv6_random_ip(self): self.assertEqual( utils.rsync_ip('fe80:0000:0000:0000:0202:b3ff:fe1e:8329'), '[fe80:0000:0000:0000:0202:b3ff:fe1e:8329]')
def test_rsync_ip_ipv6_ipv4_compatible(self): self.assertEqual(utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]')