def rsync(self, job):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job['path']):
if self.test:
print "Error: the path %s does not exists" % job['path']
return False, {}
args = [
'rsync',
'-a',
'--whole-file',
'--human-readable',
'--xattrs',
'--ignore-existing',
]
node = job['node']
node_ip = rsync_ip(node['replication_ip'])
rsync_module = '%s:%s' % (node_ip, job['remote_path'])
args.append(job['path'])
args.append(rsync_module)
if not self.test:
return self._rsync(args) == 0, {}
else:
print " ".join(args)
return True, {}
def create_remote_directory(self, job):
"""
Creates a temporal directory, at remote server.
:param job: information about the partition being synced
"""
node = job['node']
args = [
"ssh",
rsync_ip(node['replication_ip']), "mkdir", "-p", job['remote_path']
]
if not self.test:
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
results = proc.stdout.read()
ret_val = proc.wait()
#TODO: ret_val check
(results, ret_val)
else:
print " ".join(args)
def _rsync_db(self, broker, device, http, local_id,
replicate_method='complete_rsync', replicate_timeout=None):
"""
Sync a whole db using rsync.
:param broker: DB broker object of DB to be synced
:param device: device to sync to
:param http: ReplConnection object
:param local_id: unique ID of the local database replica
:param replicate_method: remote operation to perform after rsync
:param replicate_timeout: timeout to wait in seconds
"""
device_ip = rsync_ip(device['replication_ip'])
if self.vm_test_mode:
remote_file = '%s::%s%s/%s/tmp/%s' % (
device_ip, self.server_type, device['replication_port'],
device['device'], local_id)
else:
remote_file = '%s::%s/%s/tmp/%s' % (
device_ip, self.server_type, device['device'], local_id)
mtime = os.path.getmtime(broker.db_file)
if not self._rsync_file(broker.db_file, remote_file):
return False
# perform block-level sync if the db was modified during the first sync
if os.path.exists(broker.db_file + '-journal') or \
os.path.getmtime(broker.db_file) > mtime:
# grab a lock so nobody else can modify it
with broker.lock():
if not self._rsync_file(broker.db_file, remote_file, False):
return False
with Timeout(replicate_timeout or self.node_timeout):
response = http.replicate(replicate_method, local_id)
return response and response.status >= 200 and response.status < 300
def _rsync_db(self, broker, device, http, local_id,
replicate_method='complete_rsync', replicate_timeout=None):
"""
Sync a whole db using rsync.
:param broker: DB broker object of DB to be synced
:param device: device to sync to
:param http: ReplConnection object
:param local_id: unique ID of the local database replica
:param replicate_method: remote operation to perform after rsync
:param replicate_timeout: timeout to wait in seconds
"""
device_ip = rsync_ip(device['ip'])
if self.vm_test_mode:
remote_file = '%s::%s%s/%s/tmp/%s' % (device_ip,
self.server_type, device['port'], device['device'],
local_id)
else:
remote_file = '%s::%s/%s/tmp/%s' % (device_ip,
self.server_type, device['device'], local_id)
mtime = os.path.getmtime(broker.db_file)
if not self._rsync_file(broker.db_file, remote_file):
return False
# perform block-level sync if the db was modified during the first sync
if os.path.exists(broker.db_file + '-journal') or \
os.path.getmtime(broker.db_file) > mtime:
# grab a lock so nobody else can modify it
with broker.lock():
if not self._rsync_file(broker.db_file, remote_file, False):
return False
with Timeout(replicate_timeout or self.node_timeout):
response = http.replicate(replicate_method, local_id)
return response and response.status >= 200 and response.status < 300
def rsync(self, node, job, suffixes):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job['path']):
return False
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False
args.append(join(rsync_module, node['device'],
'objects', job['partition']))
return self._rsync(args) == 0
def create_remote_directory(self, job):
"""
Creates a temporal directory, at remote server.
:param job: information about the partition being synced
"""
node = job['node']
args = ["ssh", rsync_ip(node['replication_ip']),
"mkdir", "-p", job['remote_path']]
if not self.test:
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
results = proc.stdout.read()
ret_val = proc.wait()
#TODO: ret_val check
(results, ret_val)
else:
print " ".join(args)
def rsync(self, node, job, suffixes):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job['path']):
return False
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False
args.append(
join(rsync_module, node['device'], 'objects', job['partition']))
return self._rsync(args) == 0
def rsync(self, job):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job['path']):
if self.test:
print "Error: the path %s does not exists" % job['path']
return False, {}
args = [
'rsync',
'-a',
'--whole-file',
'--human-readable',
'--xattrs',
'--ignore-existing',
]
node = job['node']
node_ip = rsync_ip(node['replication_ip'])
rsync_module = '%s:%s' % (node_ip, job['remote_path'])
args.append(job['path'])
args.append(rsync_module)
if not self.test:
return self._rsync(args) == 0, {}
else:
print " ".join(args)
return True, {}
def rsync(self, node, job, suffixes):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job["path"]):
return False
args = [
"rsync",
"--recursive",
"--whole-file",
"--human-readable",
"--xattrs",
"--itemize-changes",
"--ignore-existing",
"--timeout=%s" % self.rsync_io_timeout,
"--contimeout=%s" % self.rsync_io_timeout,
"--bwlimit=%s" % self.rsync_bwlimit,
]
node_ip = rsync_ip(node["replication_ip"])
if self.vm_test_mode:
rsync_module = "%s::object%s" % (node_ip, node["replication_port"])
else:
rsync_module = "%s::object" % node_ip
had_any = False
for suffix in suffixes:
spath = join(job["path"], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False
data_dir = get_data_dir(job["policy_idx"])
args.append(join(rsync_module, node["device"], data_dir, job["partition"]))
return self._rsync(args) == 0
def rsync(partition, device):
node_ip = rsync_ip('127.0.0.1')
rsync_module = '%s::object' % (node_ip)
spath = join('/SSD/%s' % partition)
args = [
'rsync', '--recursive', '--whole-file', '--human-readable', '--xattrs',
'--itemize-changes', '--ignore-existing', '--timeout=30'
]
args.append(spath)
# args.append(join(rsync_module,'srv/node',device['device'],'objects',partition))
args.append(
join(rsync_module, '/srv/node', device['device'], 'objects',
partition))
start_time = time.time()
ret_val = None
try:
with Timeout(900):
proc = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
results = proc.stdout.read()
ret_val = proc.wait()
except Timeout:
print("Killing long-running rsync: %s", str(args))
proc.kill()
return 1
total_time = time.time() - start_time
for result in results.split('\n'):
if result == '':
continue
if result.startswith('cd+'):
continue
if not ret_val:
print(result)
else:
print(result)
if ret_val:
print('Bad rsync return code: %(args)s -> %(ret)d', {
'args': str(args),
'ret': ret_val
})
elif results:
print("Successful rsync of %(src)s at %(dst)s (%(time).03f)", {
'src': args[-2],
'dst': args[-1],
'time': total_time
})
else:
print("Successful rsync of %(src)s at %(dst)s (%(time).03f)", {
'src': args[-2],
'dst': args[-1],
'time': total_time
})
def rsync(partition, device, suffix):
node_ip = rsync_ip('127.0.0.1')
port=''
if device == 'd2':
port = '6020'
if device == 'd3':
port = '6030'
rsync_module = '%s::object%s' %(node_ip,port)
spath = join('/srv/node/ssd/objects/%s' %(partition),suffix[1])
print(suffix)
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=30',
'--contimeout=30',
]
args.append(spath)
args.append(join(rsync_module, device, 'objects', partition))
start_time = time.time()
ret_val = None
try:
with Timeout(900):
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
results = proc.stdout.read()
ret_val = proc.wait()
except Timeout:
print("Killing long-running rsync: %s", str(args))
proc.kill()
return 1
total_time = time.time() - start_time
for result in results.split('\n'):
if result == '':
continue
if result.startswith('cd+'):
continue
if not ret_val:
print(result)
else:
print(result)
if ret_val:
print('Bad rsync return code: %(args)s -> %(ret)d',
{'args': str(args), 'ret': ret_val})
elif results:
print("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
{'src': args[-2], 'dst': args[-1], 'time': total_time})
else:
print("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
{'src': args[-2], 'dst': args[-1], 'time': total_time})
return ret_val
def rsync(partition, device):
node_ip = rsync_ip(device['ip'])
rsync_module = 'swift@%s:' %(node_ip)
spath = '/SSD/'+str(partition)
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=30',
'--archive'
]
args.append(spath)
# args.append(join(rsync_module,'srv/node',device['device'],'objects',partition))
args.append(join(rsync_module,'srv/node',device['device'],'objects/'))
logging.info("===args===%s",str(args))
start_time = time.time()
ret_val = None
try:
with Timeout(900):
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
results = proc.stdout.read()
ret_val = proc.wait()
except Timeout:
logging.info("Killing long-running rsync: %s", str(args))
proc.kill()
return 1
total_time = time.time() - start_time
logging.info("===Total time===%s",str(total_time))
for result in results.split('\n'):
if result == '':
continue
if result.startswith('cd+'):
continue
if not ret_val:
logging.info(result)
else:
logging.info(result)
if ret_val:
logging.info('Bad rsync return code: %(args)s -> %(ret)d',
{'args': str(args), 'ret': ret_val})
elif results:
logging.info("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
{'src': args[-2], 'dst': args[-1], 'time': total_time})
else:
logging.info("Successful rsync of %(src)s at %(dst)s (%(time).03f)",
{'src': args[-2], 'dst': args[-1], 'time': total_time})
def rsync(self, node, job, suffixes):
"""
从一个远程结点上的partition同步一个文件
Synchronize local suffix directories from a partition with a remote
node.
:param node: the "dev" entry for the remote node to sync with
:param job: information about the partition being synced
:param suffixes: a list of suffixes which need to be pushed
:returns: boolean indicating success or failure
"""
if not os.path.exists(job['path']):
return False
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False
args.append(join(rsync_module, node['device'],
'objects', job['partition']))
return self._rsync(args) == 0
def rsync(self, node, job, suffixes):
"""
Synchronize local suffix directories from a partition with a remote
node.
:param node: the "dev" entry for the remote node to sync with
:param job: information about the partition being synced
:param suffixes: a list of suffixes which need to be pushed
:returns: boolean indicating success or failure
"""
if not os.path.exists(job['path']):
return False
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False
args.append(
join(rsync_module, node['device'], 'objects', job['partition']))
return self._rsync(args) == 0
def rsync(self, node, job, suffixes):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job['path']):
return False, {}
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
if self.rsync_compress and \
job['region'] != node['region']:
# Allow for compression, but only if the remote node is in
# a different region than the local one.
args.append('--compress')
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False, {}
data_dir = get_data_dir(job['policy'])
args.append(
join(rsync_module, node['device'], data_dir, job['partition']))
return self._rsync(args) == 0, {}
def rsync(self, node, job, suffixes):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
curframe = inspect.currentframe()
calframe = inspect.getouterframes(curframe, 2)
f = open("/home/hduser/log.txt","w")
f.write(str(calframe))
f.close()
if not os.path.exists(job['path']):
return False, set()
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False, set()
data_dir = get_data_dir(job['policy_idx'])
args.append(join(rsync_module, node['device'],
data_dir, job['partition']))
return self._rsync(args) == 0, set()
def rsync(self, node, job, suffixes):
"""
Uses rsync to implement the sync method. This was the first
sync method in Swift.
"""
if not os.path.exists(job['path']):
return False, {}
args = [
'rsync',
'--recursive',
'--whole-file',
'--human-readable',
'--xattrs',
'--itemize-changes',
'--ignore-existing',
'--timeout=%s' % self.rsync_io_timeout,
'--contimeout=%s' % self.rsync_io_timeout,
'--bwlimit=%s' % self.rsync_bwlimit,
]
if self.rsync_compress and \
job['region'] != node['region']:
# Allow for compression, but only if the remote node is in
# a different region than the local one.
args.append('--compress')
node_ip = rsync_ip(node['replication_ip'])
if self.vm_test_mode:
rsync_module = '%s::object%s' % (node_ip, node['replication_port'])
else:
rsync_module = '%s::object' % node_ip
had_any = False
for suffix in suffixes:
spath = join(job['path'], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False, {}
data_dir = get_data_dir(job['policy'])
args.append(join(rsync_module, node['device'],
data_dir, job['partition']))
return self._rsync(args) == 0, {}
def rsync(self, node, job, suffixes):
"""
Synchronize local suffix directories from a partition with a remote
node.
:param node: the "dev" entry for the remote node to sync with
:param job: information about the partition being synced
:param suffixes: a list of suffixes which need to be pushed
:returns: boolean indicating success or failure
"""
if not os.path.exists(job["path"]):
return False
args = [
"rsync",
"--recursive",
"--whole-file",
"--human-readable",
"--xattrs",
"--itemize-changes",
"--ignore-existing",
"--timeout=%s" % self.rsync_io_timeout,
"--contimeout=%s" % self.rsync_io_timeout,
]
node_ip = rsync_ip(node["ip"])
if self.vm_test_mode:
rsync_module = "%s::object%s" % (node_ip, node["port"])
else:
rsync_module = "%s::object" % node_ip
had_any = False
for suffix in suffixes:
spath = join(job["path"], suffix)
if os.path.exists(spath):
args.append(spath)
had_any = True
if not had_any:
return False
args.append(join(rsync_module, node["device"], "objects", job["partition"]))
return self._rsync(args) == 0
def test_rsync_ip_ipv4_localhost(self):
self.assertEqual(utils.rsync_ip('127.0.0.1'), '127.0.0.1')
def test_rsync_ip_ipv6_ipv4_compatible(self):
self.assertEqual(
utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]')
def test_rsync_ip_ipv6_random_ip(self):
self.assertEqual(
utils.rsync_ip('fe80:0000:0000:0000:0202:b3ff:fe1e:8329'),
'[fe80:0000:0000:0000:0202:b3ff:fe1e:8329]')
def test_rsync_ip_ipv6_ipv4_compatible(self):
self.assertEqual(utils.rsync_ip('::ffff:192.0.2.128'),
'[::ffff:192.0.2.128]')
def test_rsync_ip_ipv6_random_ip(self):
self.assertEqual(
utils.rsync_ip('fe80:0000:0000:0000:0202:b3ff:fe1e:8329'),
'[fe80:0000:0000:0000:0202:b3ff:fe1e:8329]')
def test_rsync_ip_ipv4_localhost(self):
self.assertEqual(utils.rsync_ip('127.0.0.1'), '127.0.0.1')