def safe_crushmap_update(self):
with open(os.devnull,
'w') as fnull, tempfile.TemporaryFile() as backup:
LOG.info("Saving crushmap for safe update")
try:
subprocess.check_call( # pylint: disable=not-callable
"ceph osd getcrushmap",
stdin=fnull,
stdout=backup,
stderr=fnull,
shell=True)
except subprocess.CalledProcessError as exc:
raise exception.CephFailure("failed to backup crushmap: %s" %
str(exc))
try:
yield
except exception.CephFailure:
backup.seek(0, os.SEEK_SET)
LOG.warn("Crushmap update failed. Restoring from backup")
subprocess.call( # pylint: disable=not-callable
"ceph osd setcrushmap",
stdin=backup,
stdout=fnull,
stderr=fnull,
shell=True)
raise
def run(local_file, remote_file, hosts):
start = time.time()
log.info("Spawning tracker...")
eventlet.spawn(track)
eventlet.sleep(1)
local_host = (local_ip(), opts['port'])
log.info("Creating torrent (host %s:%s)..." % local_host)
torrent_file = mktorrent(local_file, '%s:%s' % local_host)
log.info("Seeding %s" % torrent_file)
eventlet.spawn(seed, torrent_file, local_file)
log.info("Transferring")
if not os.path.isfile(bittornado_tgz):
cwd = os.getcwd()
os.chdir(herd_root)
args = ['tar', 'czf', 'bittornado.tar.gz', 'BitTornado']
log.info("Executing: " + " ".join(args))
subprocess.call(args)
os.chdir(cwd)
pool = eventlet.GreenPool(100)
threads = []
remainingHosts = hosts
for host in hosts:
threads.append(pool.spawn(transfer, host, torrent_file, remote_file, opts['retry']))
for thread in threads:
host = thread.wait()
remainingHosts.remove(host)
log.info("Done: %-6s Remaining: %s" % (host, remainingHosts))
os.unlink(torrent_file)
try:
os.unlink(opts['data_file'])
except OSError:
pass
log.info("Finished, took %.2f seconds." % (time.time() - start))
def run(local_file, remote_file, hosts):
start = time.time()
print "Spawning tracker..."
eventlet.spawn(track)
eventlet.sleep(1)
local_host = (local_ip(), PORT)
print "Creating torrent (host %s:%s)..." % local_host
torrent_file = mktorrent(local_file, '%s:%s' % local_host)
print "Seeding %s" % torrent_file
eventlet.spawn(seed, torrent_file, local_file)
print "Transferring"
if not os.path.isfile(bittornado_tgz):
cwd = os.getcwd()
os.chdir(herd_root)
args = ['tar', 'czf', 'bittornado.tar.gz', 'BitTornado']
print "Executing", " ".join(args)
subprocess.call(args)
os.chdir(cwd)
pool = eventlet.GreenPool(100)
threads = []
for host in hosts:
threads.append(pool.spawn(transfer, host, torrent_file, remote_file))
for thread in threads:
thread.wait()
os.unlink(torrent_file)
try:
os.unlink(DATA_FILE)
except OSError:
pass
print "Finished, took %.2f seconds." % (time.time() - start)
def run(local_file, remote_file, hosts):
start = time.time()
print "Spawning tracker..."
tracker = eventlet.spawn(track)
eventlet.sleep(1)
local_host = (local_ip(), PORT)
print "Creating torrent (host %s:%s)..." % local_host
torrent_file = mktorrent(local_file, '%s:%s' % local_host)
print "Seeding"
seeder = eventlet.spawn(seed, torrent_file, local_file)
print "Transferring"
if not os.path.isfile('./bittornado.tar.gz'):
subprocess.call("tar cfz ./bittornado.tar.gz ./BitTornado".split(' '))
pool = eventlet.GreenPool(100)
threads = []
for host in hosts:
threads.append(pool.spawn(transfer, host, torrent_file, remote_file))
for thread in threads:
thread.wait()
os.unlink(torrent_file)
try:
os.unlink(DATA_FILE)
except OSError:
pass
print "Finished, took %.2f seconds." % (time.time() - start)
def run(local_file, remote_file, hosts):
start = time.time()
print "Spawning tracker..."
tracker = eventlet.spawn(track)
eventlet.sleep(1)
local_host = (local_ip(), PORT)
print "Creating torrent (host %s:%s)..." % local_host
torrent_file = mktorrent(local_file, '%s:%s' % local_host)
print "Seeding"
seeder = eventlet.spawn(seed, torrent_file, local_file)
print "Transferring"
if not os.path.isfile('./bittornado.tar.gz'):
subprocess.call("tar cfz ./bittornado.tar.gz ./BitTornado".split(' '))
pool = eventlet.GreenPool(100)
threads = []
for host in hosts:
threads.append(pool.spawn(transfer, host, torrent_file, remote_file))
for thread in threads:
thread.wait()
os.unlink(torrent_file)
try:
os.unlink(DATA_FILE)
except OSError:
pass
print "Finished, took %.2f seconds." % (time.time() - start)
def execute_command(cmd, **kwargs):
try:
subprocess.call(cmd, **kwargs)
except subprocess.CalledProcessError:
if kwargs['shell']:
raise exception.CommandError(cmd=cmd)
else:
raise exception.CommandError(cmd=' '.join(cmd))
def renamer(old, new):
"""
Attempt to fix / hide race conditions like empty object directories
being removed by backend processes during uploads, by retrying.
:param old: old path to be renamed
:param new: new path to be renamed to
"""
try:
mkdirs(os.path.dirname(new))
os.rename(old, new)
except OSError, err:
mkdirs(os.path.dirname(new))
if err.errno == errno.EXDEV:
subprocess.call(['mv', old, new])
else:
os.rename(old, new)
def ssh(host,command):
global password
if not os.path.exists(opts['log_dir']):
os.makedirs(opts['log_dir'])
incommand = ['sshpass','-p',password,'ssh',host,command]
result = subprocess.call(incommand)
print ('SSH complete')
return result
def ssh(host, command):
global password
if not os.path.exists(opts['log_dir']):
os.makedirs(opts['log_dir'])
incommand = ['sshpass', '-p', password, 'ssh', host, command]
result = subprocess.call(incommand)
print('SSH complete')
return result
def copy(self, src, dest):
# With large files this is 2x-3x faster than shutil.copy(src, dest),
# especially when copying to a UNC target.
# shutil.copyfileobj(...) with a proper buffer is better than
# shutil.copy(...) but still 20% slower than a shell copy.
# It can be replaced with Win32 API calls to avoid the process
# spawning overhead.
if subprocess.call(['cmd.exe', '/C', 'copy', '/Y', src, dest]):
raise IOError(_('The file copy from %(src)s to %(dest)s failed'))
def scp(host, local_file, remote_file):
incommand = [
'sshpass', '-p', password, 'scp', local_file,
'%s:%s' % (host, remote_file)
]
print(incommand)
return subprocess.call(incommand,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def on_timeout_expired(timeout):
try:
process.wait(timeout=self._timeout)
except subprocess.TimeoutExpired:
# Set the error prior to kill the process else the error is not picked up due
# to eventlet scheduling.
error_holder['error'] = 'Action failed to complete in %s seconds' % (self._timeout)
# Action has timed out, kill the process and propagate the error. The process
# is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
try:
killcommand = shlex.split('sudo pkill -TERM -s %s' % process.pid)
subprocess.call(killcommand)
except:
LOG.exception('Unable to pkill.')
def on_timeout_expired(timeout):
try:
process.wait(timeout=self._timeout)
except subprocess.TimeoutExpired:
# Set the error prior to kill the process else the error is not picked up due
# to eventlet scheduling.
error_holder['error'] = 'Action failed to complete in %s seconds' % (self._timeout)
# Action has timed out, kill the process and propagate the error. The process
# is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
try:
killcommand = shlex.split('sudo pkill -TERM -s %s' % process.pid)
subprocess.call(killcommand)
except:
LOG.exception('Unable to pkill.')
def ssh(host, command):
if not os.path.exists(opts['log_dir']):
os.makedirs(opts['log_dir'])
with open("%s%s%s-ssh.log" % (opts['log_dir'], os.path.sep, host), 'a') as log:
result = subprocess.call(['ssh', '-o UserKnownHostsFile=/dev/null',
'-o LogLevel=quiet',
'-o StrictHostKeyChecking=no',
host, command], stdout=log,
stderr=log)
return result
def kill_process(process):
"""
Kill the provided process by sending it TERM signal using "pkill" shell
command.
Note: This function only works on Linux / Unix based systems.
:param process: Process object as returned by subprocess.Popen.
:type process: ``object``
"""
kill_command = shlex.split('sudo pkill -TERM -s %s' % (process.pid))
try:
if six.PY3:
status = subprocess.call(kill_command, timeout=100) # pylint: disable=not-callable
else:
status = subprocess.call(kill_command) # pylint: disable=not-callable
except Exception:
LOG.exception('Unable to pkill process.')
return status
def kill_process(process):
"""
Kill the provided process by sending it TERM signal using "pkill" shell
command.
Note: This function only works on Linux / Unix based systems.
:param process: Process object as returned by subprocess.Popen.
:type process: ``object``
"""
kill_command = shlex.split('sudo pkill -TERM -s %s' % (process.pid))
try:
if six.PY3:
status = subprocess.call(kill_command, timeout=100) # pylint: disable=not-callable
else:
status = subprocess.call(kill_command) # pylint: disable=not-callable
except Exception:
LOG.exception('Unable to pkill process.')
return status
def run(local_file, remote_file, hosts):
start = time.time()
log.info("Spawning tracker...")
eventlet.spawn(track)
eventlet.sleep(1)
local_host = (local_ip(), opts['port'])
log.info("Creating torrent (host %s:%s)..." % local_host)
torrent_file = mktorrent(local_file, '%s:%s' % local_host)
log.info("Seeding %s" % torrent_file)
eventlet.spawn(seed, torrent_file, local_file)
log.info("Transferring")
if not os.path.isfile(bittornado_tgz):
cwd = os.getcwd()
os.chdir(herd_root)
args = ['tar', 'czf', 'bittornado.tar.gz', 'BitTornado']
log.info("Executing: " + " ".join(args))
subprocess.call(args)
os.chdir(cwd)
pool = eventlet.GreenPool(100)
threads = []
remainingHosts = hosts
for host in hosts:
global username
host = '%s@%s' % (username, host)
threads.append(
pool.spawn(transfer, host, torrent_file, remote_file,
opts['retry']))
for thread in threads:
host = thread.wait()
remainingHosts.remove(host)
log.info("Done: %-6s Remaining: %s" % (host, remainingHosts))
os.unlink(torrent_file)
try:
os.unlink(opts['data_file'])
except OSError:
pass
log.info("Finished, took %.2f seconds." % (time.time() - start))
def _launch(port,logfile):
"""
Run _this_ module in a separate process. That
process will daemonize the service and quickly
exit. This function is supposed to be called
from the external API. Users currently are
not expected to manually or explicitely start
the port mapper. The port mapper should be started
the first time someone wants to use it.
"""
args = [_get_python_executable(), __file__]
if port:
args += ['-p', str(port)]
if logfile:
args += ['-l', logfile]
rcode = subprocess.call(args)
return rcode
def ssh(host, command):
return subprocess.call(['ssh', '-o UserKnownHostsFile=/dev/null',
'-o StrictHostKeyChecking=no',
host, command], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def lldp_update(self):
subprocess.call(['lldpcli', 'update'])
def scp(host, local_file, remote_file):
return subprocess.call(['scp', local_file, '%s:%s' % (host, remote_file)],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def scp(host, local_file, remote_file):
return subprocess.call(
['scp', local_file, '%s:%s' % (host, remote_file)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def init_sound():
if sys.platform == "linux" or sys.platform == "linux2":
Logger().write(LogVerbosity.Debug, "Settings sound to 100%")
call(["amixer", "sset", "PCM,0", "100%"])
def ssh(host, command):
return subprocess.call(['ssh', host, command],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def lldp_update(self):
subprocess.call(['lldpcli', 'update']) # pylint: disable=not-callable
def scp(host, local_file, remote_file):
incommand=['sshpass','-p',password,'scp',local_file,'%s:%s' %(host,remote_file)]
print(incommand)
return subprocess.call(incommand,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def scp(host, local_file, remote_file):
return subprocess.call(['scp', '-o UserKnownHostsFile=/dev/null',
'-o StrictHostKeyChecking=no',
local_file, '%s:%s' % (host, remote_file)],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def ssh(host, command):
return subprocess.call(['ssh', host, command], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
