class TestRemoteTarget(unittest.TestCase): """ These tests assume RemoteContext working in order for setUp and tearDown to work """ def setUp(self): self.ctx = RemoteContext(working_ssh_host) self.filepath = "/tmp/luigi_remote_test.dat" self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.ctx.check_output(["rm", "-rf", self.filepath]) self.ctx.check_output(["echo -n 'hello' >", self.filepath]) def tearDown(self): self.ctx.check_output(["rm", "-rf", self.filepath]) def test_exists(self): self.assertTrue(self.target.exists()) no_file = RemoteTarget( "/tmp/_file_that_doesnt_exist_", working_ssh_host, ) self.assertFalse(no_file.exists()) def test_remove(self): self.target.remove() self.assertRaises( subprocess.CalledProcessError, self.ctx.check_output, ["cat", self.filepath] ) def test_open(self): f = self.target.open('r') file_content = f.read() f.close() self.assertEqual(file_content, "hello") self.assertTrue(self.target.fs.exists(self.filepath)) self.assertFalse(self.target.fs.isdir(self.filepath)) def test_context_manager(self): with self.target.open('r') as f: file_content = f.read() self.assertEqual(file_content, "hello")
class TestRemoteTarget(unittest.TestCase): """ These tests assume RemoteContext working in order for setUp and tearDown to work """ def setUp(self): self.ctx = RemoteContext(working_ssh_host) self.filepath = "/tmp/luigi_remote_test.dat" self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.ctx.check_output(["rm", "-rf", self.filepath]) self.ctx.check_output(["echo -n 'hello' >", self.filepath]) def tearDown(self): self.ctx.check_output(["rm", "-rf", self.filepath]) def test_exists(self): self.assertTrue(self.target.exists()) no_file = RemoteTarget( "/tmp/_file_that_doesnt_exist_", working_ssh_host, ) self.assertFalse(no_file.exists()) def test_remove(self): self.target.remove() self.assertRaises( subprocess.CalledProcessError, self.ctx.check_output, ["cat", self.filepath] ) def test_open(self): f = self.target.open('r') file_content = f.read() f.close() self.assertEqual(file_content, "hello") self.assertTrue(self.target.fs.exists(self.filepath)) self.assertFalse(self.target.fs.isdir(self.filepath)) def test_context_manager(self): with self.target.open('r') as f: file_content = f.read() self.assertEqual(file_content, "hello")
class TestRemoteFilesystem(unittest.TestCase): def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root]) def test_mkdir(self): self.assertFalse(self.fs.isdir(self.directory)) self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False) self.fs.mkdir(self.directory) self.assertTrue(self.fs.isdir(self.directory)) # Shouldn't throw self.fs.mkdir(self.directory) self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True) def test_list(self): with self.target.open('w'): pass self.assertEqual([self.target.path], list(self.fs.listdir(self.directory)))
def test_close(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print >> p, 'test' self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path))
class TestRemoteFilesystem(unittest.TestCase): def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root]) def test_mkdir(self): self.assertFalse(self.fs.isdir(self.directory)) self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False) self.fs.mkdir(self.directory) self.assertTrue(self.fs.isdir(self.directory)) # Shouldn't throw self.fs.mkdir(self.directory) self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True) def test_list(self): with self.target.open('w'): pass self.assertEquals([self.target.path], list(self.fs.listdir(self.directory)))
def test_close(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print('test', file=p) self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path))
def test_write_cleanup_with_error(self): t = RemoteTarget(self.path, working_ssh_host) try: with t.open('w'): raise Exception('something broke') except: pass self.assertFalse(t.exists())
def test_write_cleanup_with_error(self): t = RemoteTarget(self.path, working_ssh_host) try: with t.open('w'): raise Exception('something broke') except: pass self.assertFalse(t.exists())
def test_del(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print('test', file=p) tp = p.tmp_path del p self.assertFalse(self._exists(tp)) self.assertFalse(self._exists(self.path))
def test_del(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print >> p, 'test' tp = p.tmp_path del p self.assertFalse(self._exists(tp)) self.assertFalse(self._exists(self.path))
def complete(self): rt = RemoteTarget(host=self.host, path=self.remote_path, username=self.user) if not rt.exists(): return False # Check hashes: local_target = luigi.LocalTarget(path=self.local_path) with local_target.open('r') as reader: local_hash = hashlib.sha512(reader.read()).hexdigest() logger.info("LOCAL HASH: %s" % local_hash) # Read from Remote with rt.open('r') as reader: remote_hash = hashlib.sha512(reader.read()).hexdigest() logger.info("REMOTE HASH: %s" % remote_hash) # If they match, we are good: return remote_hash == local_hash
def test_gzip(self): t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip) p = t.open('w') test_data = 'test' p.write(test_data) self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path)) # Using gzip module as validation cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file) assert os.system(cmd) == 0 f = gzip.open(self.local_file, 'rb') self.assertTrue(test_data == f.read()) f.close() # Verifying our own gzip remote reader f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r') self.assertTrue(test_data == f.read()) f.close()
def test_gzip(self): t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip) p = t.open('w') test_data = 'test' p.write(test_data) self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path)) # Using gzip module as validation cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file) assert os.system(cmd) == 0 f = gzip.open(self.local_file, 'rb') self.assertTrue(test_data == f.read()) f.close() # Verifying our own gzip remote reader f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r') self.assertTrue(test_data == f.read()) f.close()
def test_recursion_on_delete(self): target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host) with self.assertRaises(RemoteCalledProcessError): with target.open('w') as fh: fh.write("test")
def test_write_with_success(self): t = RemoteTarget(self.path, working_ssh_host) with t.open('w') as p: p.write("hello") self.assertTrue(t.exists())
def test_write_with_success(self): t = RemoteTarget(self.path, working_ssh_host) with t.open('w') as p: p.write("hello") self.assertTrue(t.exists())
def test_recursion_on_delete(self): target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host) with self.assertRaises(RemoteCalledProcessError): with target.open('w') as fh: fh.write("test")