def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_write_cleanup_with_error(self):
t = RemoteTarget(self.path, working_ssh_host)
try:
with t.open('w'):
raise Exception('something broke')
except:
pass
self.assertFalse(t.exists())
def test_write_cleanup_with_error(self):
t = RemoteTarget(self.path, working_ssh_host)
try:
with t.open('w'):
raise Exception('something broke')
except:
pass
self.assertFalse(t.exists())
def test_write_cleanup_no_close(self):
t = RemoteTarget(self.path, working_ssh_host)
def context():
f = t.open('w')
f.write('stuff')
context()
gc.collect() # force garbage collection of f variable
self.assertFalse(t.exists())
def test_write_cleanup_no_close(self):
t = RemoteTarget(self.path, working_ssh_host)
def context():
f = t.open('w')
f.write('stuff')
context()
gc.collect() # force garbage collection of f variable
self.assertFalse(t.exists())
class TestRemoteTarget(unittest.TestCase):
""" These tests assume RemoteContext working
in order for setUp and tearDown to work
"""
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def tearDown(self):
self.ctx.check_output(["rm", "-rf", self.filepath])
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_remove(self):
self.target.remove()
self.assertRaises(
subprocess.CalledProcessError,
self.ctx.check_output,
["cat", self.filepath]
)
def test_open(self):
f = self.target.open('r')
file_content = f.read()
f.close()
self.assertEqual(file_content, "hello")
self.assertTrue(self.target.fs.exists(self.filepath))
self.assertFalse(self.target.fs.isdir(self.filepath))
def test_context_manager(self):
with self.target.open('r') as f:
file_content = f.read()
self.assertEqual(file_content, "hello")
class TestRemoteTarget(unittest.TestCase):
""" These tests assume RemoteContext working
in order for setUp and tearDown to work
"""
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def tearDown(self):
self.ctx.check_output(["rm", "-rf", self.filepath])
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_remove(self):
self.target.remove()
self.assertRaises(
subprocess.CalledProcessError,
self.ctx.check_output,
["cat", self.filepath]
)
def test_open(self):
f = self.target.open('r')
file_content = f.read()
f.close()
self.assertEqual(file_content, "hello")
self.assertTrue(self.target.fs.exists(self.filepath))
self.assertFalse(self.target.fs.isdir(self.filepath))
def test_context_manager(self):
with self.target.open('r') as f:
file_content = f.read()
self.assertEqual(file_content, "hello")
def complete(self):
rt = RemoteTarget(host=self.host,
path=self.remote_path,
username=self.user)
if not rt.exists():
return False
# Check hashes:
local_target = luigi.LocalTarget(path=self.local_path)
with local_target.open('r') as reader:
local_hash = hashlib.sha512(reader.read()).hexdigest()
logger.info("LOCAL HASH: %s" % local_hash)
# Read from Remote
with rt.open('r') as reader:
remote_hash = hashlib.sha512(reader.read()).hexdigest()
logger.info("REMOTE HASH: %s" % remote_hash)
# If they match, we are good:
return remote_hash == local_hash
def test_write_with_success(self):
t = RemoteTarget(self.path, working_ssh_host)
with t.open('w') as p:
p.write("hello")
self.assertTrue(t.exists())
def test_write_with_success(self):
t = RemoteTarget(self.path, working_ssh_host)
with t.open('w') as p:
p.write("hello")
self.assertTrue(t.exists())
