class TestRemoteTarget(unittest.TestCase):
""" These tests assume RemoteContext working
in order for setUp and tearDown to work
"""
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def tearDown(self):
self.ctx.check_output(["rm", "-rf", self.filepath])
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_remove(self):
self.target.remove()
self.assertRaises(
subprocess.CalledProcessError,
self.ctx.check_output,
["cat", self.filepath]
)
def test_open(self):
f = self.target.open('r')
file_content = f.read()
f.close()
self.assertEqual(file_content, "hello")
self.assertTrue(self.target.fs.exists(self.filepath))
self.assertFalse(self.target.fs.isdir(self.filepath))
def test_context_manager(self):
with self.target.open('r') as f:
file_content = f.read()
self.assertEqual(file_content, "hello")
class TestRemoteTarget(unittest.TestCase):
""" These tests assume RemoteContext working
in order for setUp and tearDown to work
"""
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def tearDown(self):
self.ctx.check_output(["rm", "-rf", self.filepath])
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_remove(self):
self.target.remove()
self.assertRaises(
subprocess.CalledProcessError,
self.ctx.check_output,
["cat", self.filepath]
)
def test_open(self):
f = self.target.open('r')
file_content = f.read()
f.close()
self.assertEqual(file_content, "hello")
self.assertTrue(self.target.fs.exists(self.filepath))
self.assertFalse(self.target.fs.isdir(self.filepath))
def test_context_manager(self):
with self.target.open('r') as f:
file_content = f.read()
self.assertEqual(file_content, "hello")
class TestRemoteFilesystem(unittest.TestCase):
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_mkdir(self):
self.assertFalse(self.fs.isdir(self.directory))
self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False)
self.fs.mkdir(self.directory)
self.assertTrue(self.fs.isdir(self.directory))
# Shouldn't throw
self.fs.mkdir(self.directory)
self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True)
def test_list(self):
with self.target.open('w'):
pass
self.assertEqual([self.target.path], list(self.fs.listdir(self.directory)))
def test_close(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print >> p, 'test'
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
class TestRemoteFilesystem(unittest.TestCase):
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_mkdir(self):
self.assertFalse(self.fs.isdir(self.directory))
self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False)
self.fs.mkdir(self.directory)
self.assertTrue(self.fs.isdir(self.directory))
# Shouldn't throw
self.fs.mkdir(self.directory)
self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True)
def test_list(self):
with self.target.open('w'):
pass
self.assertEquals([self.target.path], list(self.fs.listdir(self.directory)))
def test_close(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print('test', file=p)
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
def test_write_cleanup_with_error(self):
t = RemoteTarget(self.path, working_ssh_host)
try:
with t.open('w'):
raise Exception('something broke')
except:
pass
self.assertFalse(t.exists())
def test_write_cleanup_with_error(self):
t = RemoteTarget(self.path, working_ssh_host)
try:
with t.open('w'):
raise Exception('something broke')
except:
pass
self.assertFalse(t.exists())
def test_del(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print('test', file=p)
tp = p.tmp_path
del p
self.assertFalse(self._exists(tp))
self.assertFalse(self._exists(self.path))
def test_del(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print >> p, 'test'
tp = p.tmp_path
del p
self.assertFalse(self._exists(tp))
self.assertFalse(self._exists(self.path))
def complete(self):
rt = RemoteTarget(host=self.host,
path=self.remote_path,
username=self.user)
if not rt.exists():
return False
# Check hashes:
local_target = luigi.LocalTarget(path=self.local_path)
with local_target.open('r') as reader:
local_hash = hashlib.sha512(reader.read()).hexdigest()
logger.info("LOCAL HASH: %s" % local_hash)
# Read from Remote
with rt.open('r') as reader:
remote_hash = hashlib.sha512(reader.read()).hexdigest()
logger.info("REMOTE HASH: %s" % remote_hash)
# If they match, we are good:
return remote_hash == local_hash
def test_gzip(self):
t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip)
p = t.open('w')
test_data = 'test'
p.write(test_data)
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
# Using gzip module as validation
cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file)
assert os.system(cmd) == 0
f = gzip.open(self.local_file, 'rb')
self.assertTrue(test_data == f.read())
f.close()
# Verifying our own gzip remote reader
f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r')
self.assertTrue(test_data == f.read())
f.close()
def test_gzip(self):
t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip)
p = t.open('w')
test_data = 'test'
p.write(test_data)
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
# Using gzip module as validation
cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file)
assert os.system(cmd) == 0
f = gzip.open(self.local_file, 'rb')
self.assertTrue(test_data == f.read())
f.close()
# Verifying our own gzip remote reader
f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r')
self.assertTrue(test_data == f.read())
f.close()
def test_recursion_on_delete(self):
target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host)
with self.assertRaises(RemoteCalledProcessError):
with target.open('w') as fh:
fh.write("test")
def test_write_with_success(self):
t = RemoteTarget(self.path, working_ssh_host)
with t.open('w') as p:
p.write("hello")
self.assertTrue(t.exists())
def test_write_with_success(self):
t = RemoteTarget(self.path, working_ssh_host)
with t.open('w') as p:
p.write("hello")
self.assertTrue(t.exists())
def test_recursion_on_delete(self):
target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host)
with self.assertRaises(RemoteCalledProcessError):
with target.open('w') as fh:
fh.write("test")
