def _tree_rcopy_dir(self, target, dirsuffix=None):
teh = TEventHandler()
srcdir = make_temp_dir()
destdir = make_temp_dir()
file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", dir=srcdir)
file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", dir=srcdir)
try:
worker = self.task.rcopy(srcdir, destdir, nodes=target, handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
# rcopy successful?
if not dirsuffix:
dirsuffix = target
rcopy_dest = join(destdir, basename(srcdir) + '.' + dirsuffix)
with open(join(rcopy_dest, basename(file1.name)), 'rb') as rfile1:
self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum')
with open(join(rcopy_dest, basename(file2.name)), 'rb') as rfile2:
self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo')
finally:
# src
file1 = None
file2 = None
os.rmdir(srcdir)
# dest
shutil.rmtree(destdir)
def _tree_copy_dir(self, target):
teh = TEventHandler()
srcdir = make_temp_dir()
destdir = make_temp_dir()
file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", dir=srcdir)
file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", dir=srcdir)
try:
# add '/' to dest so that distant does like the others
worker = self.task.copy(srcdir, destdir + '/', nodes=target,
handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
# copy successful?
copy_dest = join(destdir, srcdir)
with open(join(copy_dest, basename(file1.name)), 'rb') as rfile1:
self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum')
with open(join(copy_dest, basename(file2.name)), 'rb') as rfile2:
self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo')
finally:
# src
file1 = None
file2 = None
os.rmdir(srcdir)
# dest
shutil.rmtree(destdir)
def testLocalhostCopyDir(self):
dtmp_src = make_temp_dir("src")
dtmp_dst = make_temp_dir("testLocalhostCopyDir")
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = self._task.copy(dtmp_src, dtmp_dst, nodes=HOSTNAME)
self.assert_(worker != None)
self._task.resume()
self.assert_(os.path.exists(os.path.join(dtmp_dst, os.path.basename(dtmp_src), "lev1_a", "lev2")))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def testLocalhostExplicitSshCopyDirPreserve(self):
dtmp_src = make_temp_dir("src")
dtmp_dst = make_temp_dir("testLocalhostExplicitSshCopyDirPreserve")
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = WorkerSsh(HOSTNAME, source=dtmp_src, dest=dtmp_dst, handler=None, timeout=10, preserve=True)
self._task.schedule(worker)
self._task.resume()
self.assert_(os.path.exists(os.path.join(dtmp_dst, os.path.basename(dtmp_src), "lev1_a", "lev2")))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def testLocalhostCopyDir(self):
"""test simple localhost directory copy"""
dtmp_src = make_temp_dir('src')
dtmp_dst = make_temp_dir('testLocalhostCopyDir')
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = self._task.copy(dtmp_src, dtmp_dst, nodes='localhost')
self.assert_(worker != None)
self._task.resume()
self.assert_(os.path.exists(os.path.join(dtmp_dst, \
os.path.basename(dtmp_src), "lev1_a", "lev2")))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def testLocalhostExplicitSshReverseCopyDir(self):
dtmp_src = make_temp_dir('src')
dtmp_dst = make_temp_dir('testLocalhostExplicitSshReverseCopyDir')
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = WorkerSsh(HOSTNAME, source=dtmp_src,
dest=dtmp_dst, handler=None, timeout=30, reverse=True)
self._task.schedule(worker)
self._task.resume()
self.assert_(os.path.exists(os.path.join(dtmp_dst, \
"%s.%s" % (os.path.basename(dtmp_src), HOSTNAME), "lev1_a", "lev2")))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def test_rcopy(self):
"""test file rcopy setup in tree mode (1 gateway)"""
self.test_ok = False
self.tfile = make_temp_file("dummy-src")
self.tdir = make_temp_dir()
task_self().rcopy(self.tfile.name, self.tdir, "n60")
task_self().resume()
self.assertTrue(self.test_ok)
def testLocalhostExplicitSshReverseCopyDirPreserve(self):
"""test simple localhost preserve rcopy dir with explicit ssh worker"""
dtmp_src = make_temp_dir('src')
dtmp_dst = make_temp_dir('testLocalhostExplicitSshReverseCopyDirPreserve')
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = WorkerSsh("localhost", source=dtmp_src,
dest=dtmp_dst, handler=None, timeout=30, reverse=True)
self._task.schedule(worker)
self._task.resume()
self.assert_(os.path.exists(os.path.join(dtmp_dst, \
"%s.localhost" % os.path.basename(dtmp_src), "lev1_a", "lev2")))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def testClushConfigUserOverride(self):
"""test CLI.Config.ClushConfig (XDG_CONFIG_HOME user config)"""
# XXX Test should be improved when CLUSTERSHELL_CONFIG is available
# Improvement: override CLUSTERSHELL_CONFIG and set a sys clush config
# then verify that user config overrides CLUSTERSHELL_CONFIG as
# expected...
# For now, it has been tested manually. This test only really only
# ensures that user config is taken into account.
xdg_config_home_save = os.environ.get('XDG_CONFIG_HOME')
# Create fake XDG_CONFIG_HOME
dname = make_temp_dir()
try:
os.environ['XDG_CONFIG_HOME'] = dname
# create $XDG_CONFIG_HOME/clustershell/clush.conf
usercfgdir = os.path.join(dname, 'clustershell')
os.mkdir(usercfgdir)
cfgfile = open(os.path.join(usercfgdir, 'clush.conf'), 'w')
cfgfile.write("""
[Main]
fanout: 42
connect_timeout: 14
command_timeout: 0
history_size: 100
color: never
verbosity: 2
ssh_user: trump
ssh_path: ~/bin/ssh
ssh_options: -oSomeDummyUserOption=yes
""")
cfgfile.flush()
parser = OptionParser("dummy")
parser.install_config_options()
parser.install_display_options(verbose_options=True)
parser.install_connector_options()
options, _ = parser.parse_args([])
config = ClushConfig(options) # filename=None to use defaults!
self.assertEqual(config.color, WHENCOLOR_CHOICES[0])
self.assertEqual(config.verbosity, VERB_VERB) # takes biggest
self.assertEqual(config.fanout, 42)
self.assertEqual(config.connect_timeout, 14)
self.assertEqual(config.command_timeout, 0)
self.assertEqual(config.ssh_user, 'trump')
self.assertEqual(config.ssh_path, '~/bin/ssh')
self.assertEqual(config.ssh_options, '-oSomeDummyUserOption=yes')
cfgfile.close()
finally:
if xdg_config_home_save:
os.environ['XDG_CONFIG_HOME'] = xdg_config_home_save
else:
del os.environ['XDG_CONFIG_HOME']
shutil.rmtree(dname, ignore_errors=True)
def testLocalhostExplicitPdshCopyDir(self):
# test simple localhost copy dir with explicit pdsh worker
dtmp_src = make_temp_dir('src')
# pdcp worker doesn't create custom destination directory
dtmp_dst = make_temp_dir('testLocalhostExplicitPdshCopyDir')
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = WorkerPdsh(HOSTNAME, source=dtmp_src,
dest=dtmp_dst, handler=None, timeout=10)
self._task.schedule(worker)
self._task.resume()
self.assertTrue(os.path.exists(os.path.join(dtmp_dst, \
os.path.basename(dtmp_src), "lev1_a", "lev2")))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def test_copy_directory(self):
"""test copying directory with an ExecWorker"""
srcdir = make_temp_dir()
dstdir = make_temp_dir()
ref1 = make_temp_file("data1", dir=srcdir)
pathdstsrcdir = os.path.join(dstdir, os.path.basename(srcdir))
pathdst1 = os.path.join(pathdstsrcdir, os.path.basename(ref1.name))
try:
self.execw(nodes="localhost", handler=None, source=srcdir, dest=dstdir)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.path.isdir(pathdstsrcdir))
self.assertTrue(os.path.isfile(pathdst1))
self.assertEqual(open(pathdst1).readlines()[0], "data1")
finally:
os.unlink(pathdst1)
os.rmdir(pathdstsrcdir)
del ref1
os.rmdir(dstdir)
os.rmdir(srcdir)
def testLocalhostExplicitPdshReverseCopyDirPreserve(self):
# test simple localhost preserve rcopy dir with explicit pdsh worker
dtmp_src = make_temp_dir('src')
dtmp_dst = make_temp_dir('testLocalhostExplicitPdshReverseCopyDirPreserve')
try:
os.mkdir(os.path.join(dtmp_src, "lev1_a"))
os.mkdir(os.path.join(dtmp_src, "lev1_b"))
os.mkdir(os.path.join(dtmp_src, "lev1_a", "lev2"))
worker = WorkerPdsh(HOSTNAME, source=dtmp_src, dest=dtmp_dst,
handler=None, timeout=30, preserve=True,
reverse=True)
self._task.schedule(worker)
self._task.resume()
tgt = os.path.join(dtmp_dst, "%s.%s" % (os.path.basename(dtmp_src),
HOSTNAME), "lev1_a", "lev2")
self.assertTrue(os.path.exists(tgt))
finally:
shutil.rmtree(dtmp_dst, ignore_errors=True)
shutil.rmtree(dtmp_src, ignore_errors=True)
def testLocalhostExplicitSshReverseCopy(self):
dest = make_temp_dir("testLocalhostExplicitSshRCopy")
try:
worker = WorkerSsh(HOSTNAME, source="/etc/hosts", dest=dest, handler=None, timeout=10, reverse=True)
self._task.schedule(worker)
self._task.resume()
self.assertEqual(worker.source, "/etc/hosts")
self.assertEqual(worker.dest, dest)
self.assert_(os.path.exists(os.path.join(dest, "hosts.%s" % HOSTNAME)))
finally:
shutil.rmtree(dest, ignore_errors=True)
def testLocalhostRCopy(self):
try:
dest = make_temp_dir("testLocalhostRCopy")
# use fake node 'aaa' to test rank > 0
worker = self._task.rcopy("/etc/hosts", dest, "aaa,%s" % HOSTNAME, handler=None, timeout=10)
self._task.resume()
self.assertEqual(worker.source, "/etc/hosts")
self.assertEqual(worker.dest, dest)
self.assertTrue(os.path.exists(os.path.join(dest, "hosts.%s" % HOSTNAME)))
finally:
shutil.rmtree(dest, ignore_errors=True)
def test_rcopy_wrong_directory(self):
"""test ExecWorker reverse copying with wrong directory"""
dstbasedir = make_temp_dir()
dstdir = os.path.join(dstbasedir, "wrong")
src = make_temp_file("data")
try:
self.assertRaises(ValueError, self.execw, nodes='localhost',
handler=None, source=src.name, dest=dstdir,
stderr=True, reverse=True)
finally:
os.rmdir(dstbasedir)
def test_copy(self):
"""test copying with an ExecWorker and host placeholder"""
src = make_temp_file("data")
dstdir = make_temp_dir()
dstpath = os.path.join(dstdir, os.path.basename(src.name))
try:
pattern = dstpath + ".%h"
self.execw(nodes="localhost", handler=None, source=src.name, dest=pattern)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.path.isfile(dstpath + ".localhost"))
finally:
os.unlink(dstpath + ".localhost")
os.rmdir(dstdir)
def testLocalhostExplicitSshReverseCopy(self):
"""test simple localhost rcopy with explicit ssh worker"""
dest = make_temp_dir('testLocalhostExplicitSshRCopy')
try:
worker = WorkerSsh("localhost", source="/etc/hosts",
dest=dest, handler=None, timeout=10, reverse=True)
self._task.schedule(worker)
self._task.resume()
self.assertEqual(worker.source, "/etc/hosts")
self.assertEqual(worker.dest, dest)
self.assert_(os.path.exists(os.path.join(dest, "hosts.localhost")))
finally:
shutil.rmtree(dest, ignore_errors=True)
def test_copy_wrong_directory(self):
"""test copying wrong directory with an ExecWorker"""
srcdir = make_temp_dir()
dst = make_temp_file("data")
ref1 = make_temp_file("data1", dir=srcdir)
try:
self.execw(nodes="localhost", handler=None, source=srcdir, dest=dst.name, stderr=True)
self.assertEqual(task_self().max_retcode(), 1)
self.assertTrue(len(task_self().node_error("localhost")) > 0)
self.assertTrue(os.path.isfile(ref1.name))
finally:
del ref1
os.rmdir(srcdir)
def testLocalhostExplicitSshCopyWithOptions(self):
dest = make_temp_dir("testLocalhostExplicitSshCopyWithOptions")
self._task.set_info("scp_path", "/usr/bin/scp -l 10")
self._task.set_info("scp_options", "-oLogLevel=QUIET")
try:
worker = WorkerSsh(HOSTNAME, source="/etc/hosts", dest=dest, handler=None)
self._task.schedule(worker)
self._task.resume()
self.assertEqual(self._task.max_retcode(), 0)
self.assertTrue(os.path.exists(os.path.join(dest, "hosts")))
finally:
os.unlink(os.path.join(dest, "hosts"))
os.rmdir(dest)
# clear options after test
task_cleanup()
self.assertEqual(task_self().info("scp_path"), None)
def test_copy_wrong_directory(self):
"""test copying wrong directory with an ExecWorker"""
srcdir = make_temp_dir()
dst = make_temp_file("data")
ref1 = make_temp_file("data1", dir=srcdir)
try:
self.execw(nodes='localhost',
handler=None,
source=srcdir,
dest=dst.name,
stderr=True)
self.assertEqual(task_self().max_retcode(), 1)
self.assertTrue(len(task_self().node_error("localhost")) > 0)
self.assertTrue(os.path.isfile(ref1.name))
finally:
del ref1
os.rmdir(srcdir)
def testLocalhostExplicitPdshCopyWithOptions(self):
dest = make_temp_dir('testLocalhostExplicitPdshCopyWithOptions')
self._task.set_info("pdcp_path", "pdcp -p")
try:
worker = WorkerPdsh(HOSTNAME,
source="/etc/hosts",
dest=dest,
handler=None)
self._task.schedule(worker)
self._task.resume()
self.assertEqual(self._task.max_retcode(), 0)
self.assertTrue(os.path.exists(os.path.join(dest, "hosts")))
finally:
os.unlink(os.path.join(dest, "hosts"))
os.rmdir(dest)
# clear options after test
task_cleanup()
self.assertEqual(task_self().info("pdcp_path"), None)