def _rsync_db(self, broker, device, http, local_id, replicate_method='complete_rsync', replicate_timeout=None, different_region=False): """ Sync a whole db using rsync. :param broker: DB broker object of DB to be synced :param device: device to sync to :param http: ReplConnection object :param local_id: unique ID of the local database replica :param replicate_method: remote operation to perform after rsync :param replicate_timeout: timeout to wait in seconds :param different_region: if True, the destination node is in a different region """ rsync_module = rsync_module_interpolation(self.rsync_module, device) rsync_path = '%s/tmp/%s' % (device['device'], local_id) remote_file = '%s/%s' % (rsync_module, rsync_path) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file, different_region=different_region): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, whole_file=False, different_region=different_region): return False with Timeout(replicate_timeout or self.node_timeout): response = http.replicate(replicate_method, local_id, os.path.basename(broker.db_file)) return response and 200 <= response.status < 300
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False, {} args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, '--exclude=.*.%s' % ''.join('[0-9a-zA-Z]' for i in range(6)) ] if self.rsync_compress and \ job['region'] != node['region']: # Allow for compression, but only if the remote node is in # a different region than the local one. args.append('--compress') rsync_module = rsync_module_interpolation(self.rsync_module, node) had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False, {} data_dir = get_data_dir(job['policy']) args.append( join(rsync_module, node['device'], data_dir, job['partition'])) return self._rsync(args) == 0, {}
def _rsync_db(self, broker, device, http, local_id, replicate_method='complete_rsync', replicate_timeout=None, different_region=False): """ Sync a whole db using rsync. :param broker: DB broker object of DB to be synced :param device: device to sync to :param http: ReplConnection object :param local_id: unique ID of the local database replica :param replicate_method: remote operation to perform after rsync :param replicate_timeout: timeout to wait in seconds :param different_region: if True, the destination node is in a different region """ rsync_module = rsync_module_interpolation(self.rsync_module, device) rsync_path = '%s/tmp/%s' % (device['device'], local_id) remote_file = '%s/%s' % (rsync_module, rsync_path) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file, different_region=different_region): return False # perform block-level sync if the db was modified during the first sync if os.path.exists(broker.db_file + '-journal') or \ os.path.getmtime(broker.db_file) > mtime: # grab a lock so nobody else can modify it with broker.lock(): if not self._rsync_file(broker.db_file, remote_file, whole_file=False, different_region=different_region): return False with Timeout(replicate_timeout or self.node_timeout): response = http.replicate(replicate_method, local_id) return response and 200 <= response.status < 300
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job["path"]): return False, {} args = [ "rsync", "--recursive", "--whole-file", "--human-readable", "--xattrs", "--itemize-changes", "--ignore-existing", "--timeout=%s" % self.rsync_io_timeout, "--contimeout=%s" % self.rsync_io_timeout, "--bwlimit=%s" % self.rsync_bwlimit, ] if self.rsync_compress and job["region"] != node["region"]: # Allow for compression, but only if the remote node is in # a different region than the local one. args.append("--compress") rsync_module = rsync_module_interpolation(self.rsync_module, node) had_any = False for suffix in suffixes: spath = join(job["path"], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False, {} data_dir = get_data_dir(job["policy"]) args.append(join(rsync_module, node["device"], data_dir, job["partition"])) return self._rsync(args) == 0, {}
def rsync(self, node, job, suffixes): """ Uses rsync to implement the sync method. This was the first sync method in Swift. """ if not os.path.exists(job['path']): return False, {} args = [ 'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs', '--itemize-changes', '--ignore-existing', '--timeout=%s' % self.rsync_io_timeout, '--contimeout=%s' % self.rsync_io_timeout, '--bwlimit=%s' % self.rsync_bwlimit, '--exclude=.*.%s' % ''.join('[0-9a-zA-Z]' for i in range(6)) ] if self.rsync_compress and \ job['region'] != node['region']: # Allow for compression, but only if the remote node is in # a different region than the local one. args.append('--compress') rsync_module = rsync_module_interpolation(self.rsync_module, node) had_any = False for suffix in suffixes: spath = join(job['path'], suffix) if os.path.exists(spath): args.append(spath) had_any = True if not had_any: return False, {} data_dir = get_data_dir(job['policy']) args.append(join(rsync_module, node['device'], data_dir, job['partition'])) return self._rsync(args) == 0, {}
def get_ring(ring_name, required_replicas, required_devices, server=None, force_validate=None, ipport2server=None, config_paths=None): if not server: server = ring_name ring = Ring('/etc/swift', ring_name=ring_name) if ipport2server is None: ipport2server = {} # used internally, even if not passed in if config_paths is None: config_paths = defaultdict(dict) store_config_paths(server, config_paths) repl_name = '%s-replicator' % server repl_configs = {i: readconf(c, section_name=repl_name) for i, c in config_paths[repl_name].items()} servers_per_port = any(int(c.get('servers_per_port', '0')) for c in repl_configs.values()) add_ring_devs_to_ipport2server(ring, server, ipport2server, servers_per_port=servers_per_port) if not VALIDATE_RSYNC and not force_validate: return ring # easy sanity checks if ring.replica_count != required_replicas: raise SkipTest('%s has %s replicas instead of %s' % ( ring.serialized_path, ring.replica_count, required_replicas)) devs = [dev for dev in ring.devs if dev is not None] if len(devs) != required_devices: raise SkipTest('%s has %s devices instead of %s' % ( ring.serialized_path, len(devs), required_devices)) for dev in devs: # verify server is exposing mounted device ipport = (dev['ip'], dev['port']) _, server_number = get_server_number(ipport, ipport2server) conf = repl_configs[server_number] for device in os.listdir(conf['devices']): if device == dev['device']: dev_path = os.path.join(conf['devices'], device) full_path = os.path.realpath(dev_path) if not os.path.exists(full_path): raise SkipTest( 'device %s in %s was not found (%s)' % (device, conf['devices'], full_path)) break else: raise SkipTest( "unable to find ring device %s under %s's devices (%s)" % ( dev['device'], server, conf['devices'])) # verify server is exposing rsync device rsync_export = conf.get('rsync_module', '').rstrip('/') if not rsync_export: rsync_export = '{replication_ip}::%s' % server cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev) p = Popen(cmd, shell=True, stdout=PIPE) stdout, _stderr = p.communicate() if p.returncode: raise SkipTest('unable to connect to rsync ' 'export %s (%s)' % (rsync_export, cmd)) for line in stdout.splitlines(): if line.rsplit(None, 1)[-1] == dev['device']: break else: raise SkipTest("unable to find ring device %s under rsync's " "exported devices for %s (%s)" % (dev['device'], rsync_export, cmd)) return ring
def get_ring(ring_name, required_replicas, required_devices, server=None, force_validate=None, ipport2server=None, config_paths=None): if not server: server = ring_name ring = Ring('/etc/swift', ring_name=ring_name) if ipport2server is None: ipport2server = {} # used internally, even if not passed in if config_paths is None: config_paths = defaultdict(dict) store_config_paths(server, config_paths) repl_name = '%s-replicator' % server repl_configs = {i: readconf(c, section_name=repl_name) for i, c in config_paths[repl_name].items()} servers_per_port = any(int(c.get('servers_per_port', '0')) for c in repl_configs.values()) add_ring_devs_to_ipport2server(ring, server, ipport2server, servers_per_port=servers_per_port) if not VALIDATE_RSYNC and not force_validate: return ring # easy sanity checks if ring.replica_count != required_replicas: raise SkipTest('%s has %s replicas instead of %s' % ( ring.serialized_path, ring.replica_count, required_replicas)) devs = [dev for dev in ring.devs if dev is not None] if len(devs) != required_devices: raise SkipTest('%s has %s devices instead of %s' % ( ring.serialized_path, len(ring.devs), required_devices)) for dev in devs: # verify server is exposing mounted device ipport = (dev['ip'], dev['port']) _, server_number = get_server_number(ipport, ipport2server) conf = repl_configs[server_number] for device in os.listdir(conf['devices']): if device == dev['device']: dev_path = os.path.join(conf['devices'], device) full_path = os.path.realpath(dev_path) if not os.path.exists(full_path): raise SkipTest( 'device %s in %s was not found (%s)' % (device, conf['devices'], full_path)) break else: raise SkipTest( "unable to find ring device %s under %s's devices (%s)" % ( dev['device'], server, conf['devices'])) # verify server is exposing rsync device rsync_export = conf.get('rsync_module', '').rstrip('/') if not rsync_export: rsync_export = '{replication_ip}::%s' % server if config_true_value(conf.get('vm_test_mode', 'no')): rsync_export += '{replication_port}' cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev) p = Popen(cmd, shell=True, stdout=PIPE) stdout, _stderr = p.communicate() if p.returncode: raise SkipTest('unable to connect to rsync ' 'export %s (%s)' % (rsync_export, cmd)) for line in stdout.splitlines(): if line.rsplit(None, 1)[-1] == dev['device']: break else: raise SkipTest("unable to find ring device %s under rsync's " "exported devices for %s (%s)" % (dev['device'], rsync_export, cmd)) return ring