def test_get(self): self.ctx.check_output(["echo -n 'hello' >", self.path]) t = RemoteTarget(self.path, working_ssh_host) t.get(self.local_file) f = open(self.local_file, 'r') file_content = f.read() self.assertEqual(file_content, 'hello')
def test_put(self): f = open(self.local_file, 'w') f.write('hello') f.close() t = RemoteTarget(self.path, working_ssh_host) t.put(self.local_file) self.assertTrue(self._exists(self.path))
def test_exists(self): self.assertTrue(self.target.exists()) no_file = RemoteTarget( "/tmp/_file_that_doesnt_exist_", working_ssh_host, ) self.assertFalse(no_file.exists())
def test_close(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print('test', file=p) self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path))
def test_close(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print >> p, 'test' self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path))
def test_write_cleanup_with_error(self): t = RemoteTarget(self.path, working_ssh_host) try: with t.open('w'): raise Exception('something broke') except: pass self.assertFalse(t.exists())
def test_del(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print >> p, 'test' tp = p.tmp_path del p self.assertFalse(self._exists(tp)) self.assertFalse(self._exists(self.path))
def test_write_cleanup_no_close(self): t = RemoteTarget(self.path, working_ssh_host) def context(): f = t.open('w') f.write('stuff') context() gc.collect() # force garbage collection of f variable self.assertFalse(t.exists())
def test_del(self): t = RemoteTarget(self.path, working_ssh_host) p = t.open('w') print('test', file=p) tp = p.tmp_path del p self.assertFalse(self._exists(tp)) self.assertFalse(self._exists(self.path))
def setUp(self): self.ctx = RemoteContext(working_ssh_host) self.filepath = "/tmp/luigi_remote_test.dat" self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.ctx.check_output(["rm", "-rf", self.filepath]) self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
class TestRemoteTarget(unittest.TestCase): """ These tests assume RemoteContext working in order for setUp and tearDown to work """ def setUp(self): self.ctx = RemoteContext(working_ssh_host) self.filepath = "/tmp/luigi_remote_test.dat" self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.ctx.check_output(["rm", "-rf", self.filepath]) self.ctx.check_output(["echo -n 'hello' >", self.filepath]) def tearDown(self): self.ctx.check_output(["rm", "-rf", self.filepath]) def test_exists(self): self.assertTrue(self.target.exists()) no_file = RemoteTarget( "/tmp/_file_that_doesnt_exist_", working_ssh_host, ) self.assertFalse(no_file.exists()) def test_remove(self): self.target.remove() self.assertRaises( subprocess.CalledProcessError, self.ctx.check_output, ["cat", self.filepath] ) def test_open(self): f = self.target.open('r') file_content = f.read() f.close() self.assertEqual(file_content, "hello") self.assertTrue(self.target.fs.exists(self.filepath)) self.assertFalse(self.target.fs.isdir(self.filepath)) def test_context_manager(self): with self.target.open('r') as f: file_content = f.read() self.assertEqual(file_content, "hello")
def get_luigi_target(path): try: from urlparse import urlparse # Python3 compatibility except ImportError: from urllib.parse import urlparse from luigi.s3 import S3Target from luigi.contrib.ssh import RemoteTarget from luigi.file import LocalTarget from luigi.format import GzipFormat file_format = None if path.endswith(".gz"): file_format = GzipFormat(compression_level=9) if path.startswith("s3n://"): # Try to always store files in S3 in compressed format. s3_headers = {} if file_is_compressible(path): file_format = GzipFormat(compression_level=9) s3_headers["Content-Encoding"] = "gzip" return S3Target(path, format=file_format, headers=s3_headers) elif path.startswith("ssh://"): ssh_key_file = config.get("ssh", "ssh-key-file", None) no_host_key_check = config.get("ssh", "no-host-key-check", None) p = urlparse(path) return RemoteTarget(p.path, p.hostname, format=format, username=p.username, sshpass=p.password, key_file=ssh_key_file, no_host_key_check=no_host_key_check) return LocalTarget(path, format=file_format)
class TestRemoteFilesystem(unittest.TestCase): def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root]) def test_mkdir(self): self.assertFalse(self.fs.isdir(self.directory)) self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False) self.fs.mkdir(self.directory) self.assertTrue(self.fs.isdir(self.directory)) # Shouldn't throw self.fs.mkdir(self.directory) self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True) def test_list(self): with self.target.open('w'): pass self.assertEquals([self.target.path], list(self.fs.listdir(self.directory)))
class TestRemoteFilesystem(unittest.TestCase): def setUp(self): self.fs = RemoteFileSystem(working_ssh_host) self.root = '/tmp/luigi-remote-test' self.directory = self.root + '/dir' self.filepath = self.directory + '/file' self.target = RemoteTarget( self.filepath, working_ssh_host, ) self.fs.remote_context.check_output(['rm', '-rf', self.root]) self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root]) def test_mkdir(self): self.assertFalse(self.fs.isdir(self.directory)) self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False) self.fs.mkdir(self.directory) self.assertTrue(self.fs.isdir(self.directory)) # Shouldn't throw self.fs.mkdir(self.directory) self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True) def test_list(self): with self.target.open('w'): pass self.assertEqual([self.target.path], list(self.fs.listdir(self.directory)))
def get_luigi_target(path): try: from urlparse import urlparse # Python3 compatibility except ImportError: from urllib.parse import urlparse from luigi.s3 import S3Target from luigi.contrib.ssh import RemoteTarget from luigi.file import LocalTarget from luigi.format import GzipFormat file_format = None if path.endswith(".gz"): file_format = GzipFormat() if path.startswith("s3n://"): return S3Target(path, format=file_format) elif path.startswith("ssh://"): ssh_key_file = config.get("ssh", "ssh-key-file", None) no_host_key_check = config.get("ssh", "no-host-key-check", None) p = urlparse(path) return RemoteTarget(p.path, p.hostname, format=format, username=p.username, sshpass=p.password, key_file=ssh_key_file, no_host_key_check=no_host_key_check) return LocalTarget(path, format=file_format)
def complete(self): rt = RemoteTarget(host=self.host, path=self.remote_path, username=self.user) if not rt.exists(): return False # Check hashes: local_target = luigi.LocalTarget(path=self.local_path) with local_target.open('r') as reader: local_hash = hashlib.sha512(reader.read()).hexdigest() logger.info("LOCAL HASH: %s" % local_hash) # Read from Remote with rt.open('r') as reader: remote_hash = hashlib.sha512(reader.read()).hexdigest() logger.info("REMOTE HASH: %s" % remote_hash) # If they match, we are good: return remote_hash == local_hash
def output(self): """ Returns the target output for this task. In this case, a successful execution of this task will create a file on a remote server using SSH. :return: the target output for this task. :rtype: object (:py:class:`~luigi.target.Target`) """ return RemoteTarget("/tmp/stuff", SSH_HOST)
def output(self): parameters.setParameters(parameters.data) status = RemoteTarget( parameters.data['remote_storage'] + 'demux.corrected.merged.aligned.annotated.aclame.json', SSH_HOST) # if status: stage.pushStatus(parameters.data['id'], stage.AnnotateAlignment) return status
def test_gzip(self): t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip) p = t.open('w') test_data = 'test' p.write(test_data) self.assertFalse(self._exists(self.path)) p.close() self.assertTrue(self._exists(self.path)) # Using gzip module as validation cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file) assert os.system(cmd) == 0 f = gzip.open(self.local_file, 'rb') self.assertTrue(test_data == f.read()) f.close() # Verifying our own gzip remote reader f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r') self.assertTrue(test_data == f.read()) f.close()
def __init__(self, path, file_type='regular', root_dir=None, format=None, **kwargs): self.is_remote = commons().is_remote if root_dir: full_path = os.path.join(root_dir, path) else: if self.is_remote: full_path = os.path.join(commons().remote_root, path) else: full_path = os.path.join(commons().local_root, path) self.file_type = file_type self.format = format if self.is_remote: host = commons().SSH_HOST port = commons().SSH_PORT kwargs['port'] = port self._target = RemoteTarget(full_path, host, format=format, **kwargs) if file_type == 'apk': # create temporary local copy self.local_path = os.path.join( tempfile.gettempdir(), 'luigi-{}-{}.apk'.format(os.path.basename(path), random.randint(0, 999999999))) self._target.get(self.local_path) else: self._target = LocalTarget(full_path, format=format, **kwargs) if self.is_remote and self.file_type == 'apk': path = self.local_path else: path = self._target.path super(ExternalFileTarget, self).__init__(path) # XXX: check if this is right
def output(self): parameters.setParameters(parameters.data) status = RemoteTarget( parameters.data['remote_storage'] + "barcodes.fasta", SSH_HOST) return status
def create_target(self, format=None): return RemoteTarget(self.path, working_ssh_host, format=format)
def output(self): parameters.setParameters(parameters.data) status = RemoteTarget(parameters.data['remote_storage'], SSH_HOST) # if status: stage.pushStatus(parameters.data['id'], stage.CreateEnv) return status
def output(self): status = RemoteTarget( parameters.data['remote_storage'] + "rawreads.fasta", SSH_HOST) # if status: stage.pushStatus(parameters.data['id'], self.get_params['status']) return status
def test_recursion_on_delete(self): target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host) with self.assertRaises(RemoteCalledProcessError): with target.open('w') as fh: fh.write("test")
def output(self): return RemoteTarget("/tmp/stuff", SSH_HOST)
def output(self): remote_path = os.path.join(self.remote_dir, 'remote_out.txt') return RemoteTarget(path=remote_path, host=self.ssh_host)
def output(self): # Avoid running if the target files already appear to be set up: return RemoteTarget(host=self.host, path="/heritrix/jobs/%s/crawler-beans.cxml" % self.get_job_name(0), username=self.user)
def output(self): return RemoteTarget("~/out", SSH_HOST)
def test_write_with_success(self): t = RemoteTarget(self.path, working_ssh_host) with t.open('w') as p: p.write("hello") self.assertTrue(t.exists())
def run(self): # Copy the local file over to the remote place rt = RemoteTarget(host=self.host, path=self.remote_path, username=self.user) rt.put(self.local_path)
def output(self): status = RemoteTarget( parameters.data['remote_storage'] + 'demux.corrected.merged', SSH_HOST) return status