def _tree_copy_dir(self, target):
teh = TEventHandler()
srcdir = make_temp_dir()
destdir = make_temp_dir()
file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", dir=srcdir)
file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", dir=srcdir)
try:
# add '/' to dest so that distant does like the others
worker = self.task.copy(srcdir, destdir + '/', nodes=target,
handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
# copy successful?
copy_dest = join(destdir, srcdir)
with open(join(copy_dest, basename(file1.name)), 'rb') as rfile1:
self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum')
with open(join(copy_dest, basename(file2.name)), 'rb') as rfile2:
self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo')
finally:
# src
file1 = None
file2 = None
os.rmdir(srcdir)
# dest
shutil.rmtree(destdir)
def _tree_rcopy_dir(self, target, dirsuffix=None):
teh = TEventHandler()
srcdir = make_temp_dir()
destdir = make_temp_dir()
file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", dir=srcdir)
file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", dir=srcdir)
try:
worker = self.task.rcopy(srcdir, destdir, nodes=target, handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
# rcopy successful?
if not dirsuffix:
dirsuffix = target
rcopy_dest = join(destdir, basename(srcdir) + '.' + dirsuffix)
with open(join(rcopy_dest, basename(file1.name)), 'rb') as rfile1:
self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum')
with open(join(rcopy_dest, basename(file2.name)), 'rb') as rfile2:
self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo')
finally:
# src
file1 = None
file2 = None
os.rmdir(srcdir)
# dest
shutil.rmtree(destdir)
def test_copy_wrong_directory(self):
"""test copying wrong directory with an ExecWorker"""
srcdir = make_temp_dir()
dst = make_temp_file("data")
ref1 = make_temp_file("data1", dir=srcdir)
try:
self.execw(nodes="localhost", handler=None, source=srcdir, dest=dst.name, stderr=True)
self.assertEqual(task_self().max_retcode(), 1)
self.assertTrue(len(task_self().node_error("localhost")) > 0)
self.assertTrue(os.path.isfile(ref1.name))
finally:
del ref1
os.rmdir(srcdir)
def test_copy_wrong_directory(self):
"""test copying wrong directory with an ExecWorker"""
srcdir = make_temp_dir()
dst = make_temp_file(b"data")
ref1 = make_temp_file(b"data1", dir=srcdir)
try:
self.execw(nodes='localhost', handler=None, source=srcdir,
dest=dst.name, stderr=True)
self.assertEqual(task_self().max_retcode(), 1)
self.assertTrue(len(task_self().node_error("localhost")) > 0)
self.assertTrue(os.path.isfile(ref1.name))
finally:
del ref1
os.rmdir(srcdir)
def test_002_changed(self):
"""test Defaults config file (changed)"""
conf_test = make_temp_file(dedent("""
[task.default]
stderr: true
stdout_msgtree: false
stderr_msgtree: false
engine: select
port_qlimit: 1000
auto_tree: false
local_workername: none
distant_workername: pdsh
[task.info]
debug: true
fanout: 256
grooming_delay: 0.5
connect_timeout: 12.5
command_timeout: 30.5"""))
self.defaults = Defaults(filenames=[conf_test.name])
self.assertTrue(self.defaults.stderr)
self.assertFalse(self.defaults.stdout_msgtree)
self.assertFalse(self.defaults.stderr_msgtree)
self.assertEqual(self.defaults.engine, 'select')
self.assertEqual(self.defaults.port_qlimit, 1000)
self.assertFalse(self.defaults.auto_tree)
self.assertEqual(self.defaults.local_workername, 'none')
self.assertEqual(self.defaults.distant_workername, 'pdsh')
# task_info
self.assertTrue(self.defaults.debug)
self.assertEqual(self.defaults.fanout, 256)
self.assertEqual(self.defaults.grooming_delay, 0.5)
self.assertEqual(self.defaults.connect_timeout, 12.5)
def test_001_defaults(self):
"""test Defaults config file (defaults)"""
conf_test = make_temp_file(
dedent("""
[nodeset]
fold_axis:
[task.default]
stderr: false
stdout_msgtree: true
stderr_msgtree: true
engine: auto
port_qlimit: 100
auto_tree: true
local_workername: exec
distant_workername: ssh
[task.info]
debug: false
fanout: 64
grooming_delay: 0.25
connect_timeout: 10
command_timeout: 0""").encode('ascii'))
self.defaults = Defaults(filenames=[conf_test.name])
self._assert_default_values()
def test_002_changed(self):
"""test Defaults config file (changed)"""
conf_test = make_temp_file(
dedent("""
[task.default]
stderr: true
stdout_msgtree: false
stderr_msgtree: false
engine: select
port_qlimit: 1000
auto_tree: false
local_workername: none
distant_workername: pdsh
[task.info]
debug: true
fanout: 256
grooming_delay: 0.5
connect_timeout: 12.5
command_timeout: 30.5""").encode('ascii'))
self.defaults = Defaults(filenames=[conf_test.name])
self.assertTrue(self.defaults.stderr)
self.assertFalse(self.defaults.stdout_msgtree)
self.assertFalse(self.defaults.stderr_msgtree)
self.assertEqual(self.defaults.engine, 'select')
self.assertEqual(self.defaults.port_qlimit, 1000)
self.assertFalse(self.defaults.auto_tree)
self.assertEqual(self.defaults.local_workername, 'none')
self.assertEqual(self.defaults.distant_workername, 'pdsh')
# task_info
self.assertTrue(self.defaults.debug)
self.assertEqual(self.defaults.fanout, 256)
self.assertEqual(self.defaults.grooming_delay, 0.5)
self.assertEqual(self.defaults.connect_timeout, 12.5)
def test_rcopy(self):
"""test file rcopy setup in tree mode (1 gateway)"""
self.test_ok = False
self.tfile = make_temp_file("dummy-src")
self.tdir = make_temp_dir()
task_self().rcopy(self.tfile.name, self.tdir, "n60")
task_self().resume()
self.assertTrue(self.test_ok)
def test_rcopy(self):
"""test file rcopy setup in tree mode (1 gateway)"""
self.test_ok = False
self.tfile = make_temp_file("dummy-src")
self.tdir = make_temp_dir()
task_self().rcopy(self.tfile.name, self.tdir, "n60")
task_self().resume()
self.assertTrue(self.test_ok)
def test_copy(self):
"""test file copy setup in tree mode (1 gateway)"""
self.test_ok = False
self.tfile = make_temp_file("dummy")
# add leading '/' like clush so that WorkerTree knows it's a dir
task_self().copy(self.tfile.name, join(dirname(self.tfile.name), ''),
"n60")
task_self().resume()
self.assertTrue(self.test_ok)
def test_copy(self):
"""test file copy setup in tree mode (1 gateway)"""
self.test_ok = False
self.tfile = make_temp_file("dummy")
# add leading '/' like clush so that WorkerTree knows it's a dir
task_self().copy(self.tfile.name,
join(dirname(self.tfile.name), ''),
"n60")
task_self().resume()
self.assertTrue(self.test_ok)
def test_shell_auto_tree_error(self):
"""test task shell auto tree [TopologyError]"""
# initialize an erroneous topology.conf file
topofile = make_temp_file(
'[Main]\n%s: dummy-gw\ndummy-gw: dummy-gw\n' % HOSTNAME)
task = task_self()
task.set_default("auto_tree", True)
task.TOPOLOGY_CONFIGS = [topofile.name]
self.assertRaises(TopologyError, task.run, "/bin/hostname",
nodes="dummy-node")
def test_rcopy_wrong_directory(self):
"""test ExecWorker reverse copying with wrong directory"""
dstbasedir = make_temp_dir()
dstdir = os.path.join(dstbasedir, "wrong")
src = make_temp_file("data")
try:
self.assertRaises(ValueError, self.execw, nodes='localhost',
handler=None, source=src.name, dest=dstdir,
stderr=True, reverse=True)
finally:
os.rmdir(dstbasedir)
def test_rcopy_wrong_directory(self):
"""test ExecWorker reverse copying with wrong directory"""
dstbasedir = make_temp_dir()
dstdir = os.path.join(dstbasedir, "wrong")
src = make_temp_file(b"data")
try:
self.assertRaises(ValueError, self.execw, nodes='localhost',
handler=None, source=src.name, dest=dstdir,
stderr=True, reverse=True)
finally:
os.rmdir(dstbasedir)
def test_shell_auto_tree_dummy(self):
"""test task shell auto tree"""
# initialize a dummy topology.conf file
topofile = make_temp_file(
'[Main]\n%s: dummy-gw\ndummy-gw: dummy-node\n' % HOSTNAME)
task = task_self()
task.set_default("auto_tree", True)
task.TOPOLOGY_CONFIGS = [topofile.name]
task.run("/bin/hostname", nodes="dummy-node", stderr=True)
# FIXME gateway errors are not yet being handled correctly
self.assertEqual(task.max_retcode(), 255)
def test_copy_preserve(self):
"""test copying with an ExecWorker (preserve=True)"""
src = make_temp_file("data")
past_time = 443757600
os.utime(src.name, (past_time, past_time))
dstpath = make_temp_filename()
try:
self.execw(nodes="localhost", handler=None, source=src.name, dest=dstpath, preserve=True)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.stat(dstpath).st_mtime, past_time)
finally:
os.unlink(dstpath)
def test_copy(self):
"""test copying with an ExecWorker and host placeholder"""
src = make_temp_file("data")
dstdir = make_temp_dir()
dstpath = os.path.join(dstdir, os.path.basename(src.name))
try:
pattern = dstpath + ".%h"
self.execw(nodes="localhost", handler=None, source=src.name, dest=pattern)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.path.isfile(dstpath + ".localhost"))
finally:
os.unlink(dstpath + ".localhost")
os.rmdir(dstdir)
def test_copy_preserve(self):
"""test copying with an ExecWorker (preserve=True)"""
src = make_temp_file(b"data")
past_time = 443757600
os.utime(src.name, (past_time, past_time))
dstpath = make_temp_filename()
try:
self.execw(nodes='localhost', handler=None, source=src.name,
dest=dstpath, preserve=True)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.stat(dstpath).st_mtime, past_time)
finally:
os.unlink(dstpath)
def test_copy(self):
"""test copying with an ExecWorker and host placeholder"""
src = make_temp_file(b"data")
dstdir = make_temp_dir()
dstpath = os.path.join(dstdir, os.path.basename(src.name))
try:
pattern = dstpath + ".%h"
self.execw(nodes='localhost', handler=None, source=src.name,
dest=pattern)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.path.isfile(dstpath + '.localhost'))
finally:
os.unlink(dstpath + '.localhost')
os.rmdir(dstdir)
def test_shell_auto_tree_dummy(self):
"""test task shell auto tree"""
# initialize a dummy topology.conf file
topofile = make_temp_file(dedent("""
[Main]
%s: dummy-gw
dummy-gw: dummy-node"""% HOSTNAME).encode())
task = task_self()
task.set_default("auto_tree", True)
task.TOPOLOGY_CONFIGS = [topofile.name]
self.assertRaises(RouteResolvingError, task.run, "/bin/hostname",
nodes="dummy-node", stderr=True)
self.assertEqual(task.max_retcode(), None)
def _tree_rcopy_dir(self, target, dirsuffix=None):
teh = TEventHandler()
srcdir = make_temp_dir()
destdir = make_temp_dir()
file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", dir=srcdir)
file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", dir=srcdir)
try:
worker = self.task.rcopy(srcdir,
destdir,
nodes=target,
handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
# rcopy successful?
if not dirsuffix:
dirsuffix = target
rcopy_dest = join(destdir, basename(srcdir) + '.' + dirsuffix)
with open(join(rcopy_dest, basename(file1.name)), 'rb') as rfile1:
self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum')
with open(join(rcopy_dest, basename(file2.name)), 'rb') as rfile2:
self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo')
finally:
# src
file1 = None
file2 = None
os.rmdir(srcdir)
# dest
shutil.rmtree(destdir)
def _tree_copy_dir(self, target):
teh = TEventHandler()
srcdir = make_temp_dir()
destdir = make_temp_dir()
file1 = make_temp_file(b'Lorem Ipsum Unum', suffix=".txt", dir=srcdir)
file2 = make_temp_file(b'Lorem Ipsum Duo', suffix=".txt", dir=srcdir)
try:
# add '/' to dest so that distant does like the others
worker = self.task.copy(srcdir,
destdir + '/',
nodes=target,
handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
# copy successful?
copy_dest = join(destdir, srcdir)
with open(join(copy_dest, basename(file1.name)), 'rb') as rfile1:
self.assertEqual(rfile1.read(), b'Lorem Ipsum Unum')
with open(join(copy_dest, basename(file2.name)), 'rb') as rfile2:
self.assertEqual(rfile2.read(), b'Lorem Ipsum Duo')
finally:
# src
file1 = None
file2 = None
os.rmdir(srcdir)
# dest
shutil.rmtree(destdir)
def _tree_copy_file(self, target):
teh = TEventHandler()
srcf = make_temp_file(b'Lorem Ipsum', 'test_tree_copy_file_src')
dest = make_temp_filename('test_tree_copy_file_dest')
try:
worker = self.task.copy(srcf.name, dest, nodes=target, handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
with open(dest, 'r') as destf:
self.assertEqual(destf.read(), 'Lorem Ipsum')
finally:
os.remove(dest)
def _tree_copy_file(self, target):
teh = TEventHandler()
srcf = make_temp_file(b'Lorem Ipsum', 'test_tree_copy_file_src')
dest = make_temp_filename('test_tree_copy_file_dest')
try:
worker = self.task.copy(srcf.name, dest, nodes=target, handler=teh)
self.task.run()
self.assertEqual(teh.ev_start_cnt, 1)
self.assertEqual(teh.ev_pickup_cnt, 1)
self.assertEqual(teh.ev_read_cnt, 0)
#self.assertEqual(teh.ev_written_cnt, 0) # FIXME
self.assertEqual(teh.ev_hup_cnt, 1)
self.assertEqual(teh.ev_timedout_cnt, 0)
self.assertEqual(teh.ev_close_cnt, 1)
with open(dest, 'r') as destf:
self.assertEqual(destf.read(), 'Lorem Ipsum')
finally:
os.remove(dest)
def test_copy_directory(self):
"""test copying directory with an ExecWorker"""
srcdir = make_temp_dir()
dstdir = make_temp_dir()
ref1 = make_temp_file("data1", dir=srcdir)
pathdstsrcdir = os.path.join(dstdir, os.path.basename(srcdir))
pathdst1 = os.path.join(pathdstsrcdir, os.path.basename(ref1.name))
try:
self.execw(nodes="localhost", handler=None, source=srcdir, dest=dstdir)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.path.isdir(pathdstsrcdir))
self.assertTrue(os.path.isfile(pathdst1))
self.assertEqual(open(pathdst1).readlines()[0], "data1")
finally:
os.unlink(pathdst1)
os.rmdir(pathdstsrcdir)
del ref1
os.rmdir(dstdir)
os.rmdir(srcdir)
def test_copy_directory(self):
"""test copying directory with an ExecWorker"""
srcdir = make_temp_dir()
dstdir = make_temp_dir()
ref1 = make_temp_file(b"data1", dir=srcdir)
pathdstsrcdir = os.path.join(dstdir, os.path.basename(srcdir))
pathdst1 = os.path.join(pathdstsrcdir, os.path.basename(ref1.name))
try:
self.execw(nodes='localhost', handler=None, source=srcdir,
dest=dstdir)
self.assertEqual(task_self().max_retcode(), 0)
self.assertTrue(os.path.isdir(pathdstsrcdir))
self.assertTrue(os.path.isfile(pathdst1))
self.assertEqual(open(pathdst1).readlines()[0], "data1")
finally:
os.unlink(pathdst1)
os.rmdir(pathdstsrcdir)
del ref1
os.rmdir(dstdir)
os.rmdir(srcdir)
def test_001_defaults(self):
"""test Defaults config file (defaults)"""
conf_test = make_temp_file(dedent("""
[task.default]
stderr: false
stdout_msgtree: true
stderr_msgtree: true
engine: auto
port_qlimit: 100
auto_tree: true
local_workername: exec
distant_workername: ssh
[task.info]
debug: false
fanout: 64
grooming_delay: 0.25
connect_timeout: 10
command_timeout: 0"""))
self.defaults = Defaults(filenames=[conf_test.name])
self._assert_default_values()
def test_000_empty(self):
"""test Defaults config file (empty)"""
conf_test = make_temp_file('')
self.defaults = Defaults(filenames=[conf_test.name])
self._assert_default_values()
def test_000_empty(self):
"""test Defaults config file (empty)"""
conf_test = make_temp_file(b'')
self.defaults = Defaults(filenames=[conf_test.name])
self._assert_default_values()
