def test_get(self):
self.ctx.check_output(["echo -n 'hello' >", self.path])
t = RemoteTarget(self.path, working_ssh_host)
t.get(self.local_file)
f = open(self.local_file, 'r')
file_content = f.read()
self.assertEqual(file_content, 'hello')
def test_get(self):
self.ctx.check_output(["echo -n 'hello' >", self.path])
t = RemoteTarget(self.path, working_ssh_host)
t.get(self.local_file)
f = open(self.local_file, 'r')
file_content = f.read()
self.assertEqual(file_content, 'hello')
def test_put(self):
f = open(self.local_file, 'w')
f.write('hello')
f.close()
t = RemoteTarget(self.path, working_ssh_host)
t.put(self.local_file)
self.assertTrue(self._exists(self.path))
def test_put(self):
f = open(self.local_file, 'w')
f.write('hello')
f.close()
t = RemoteTarget(self.path, working_ssh_host)
t.put(self.local_file)
self.assertTrue(self._exists(self.path))
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_close(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print('test', file=p)
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_close(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print >> p, 'test'
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
def test_write_cleanup_with_error(self):
t = RemoteTarget(self.path, working_ssh_host)
try:
with t.open('w'):
raise Exception('something broke')
except:
pass
self.assertFalse(t.exists())
def test_write_cleanup_with_error(self):
t = RemoteTarget(self.path, working_ssh_host)
try:
with t.open('w'):
raise Exception('something broke')
except:
pass
self.assertFalse(t.exists())
def test_del(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print >> p, 'test'
tp = p.tmp_path
del p
self.assertFalse(self._exists(tp))
self.assertFalse(self._exists(self.path))
def test_write_cleanup_no_close(self):
t = RemoteTarget(self.path, working_ssh_host)
def context():
f = t.open('w')
f.write('stuff')
context()
gc.collect() # force garbage collection of f variable
self.assertFalse(t.exists())
def test_del(self):
t = RemoteTarget(self.path, working_ssh_host)
p = t.open('w')
print('test', file=p)
tp = p.tmp_path
del p
self.assertFalse(self._exists(tp))
self.assertFalse(self._exists(self.path))
def test_write_cleanup_no_close(self):
t = RemoteTarget(self.path, working_ssh_host)
def context():
f = t.open('w')
f.write('stuff')
context()
gc.collect() # force garbage collection of f variable
self.assertFalse(t.exists())
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
class TestRemoteTarget(unittest.TestCase):
""" These tests assume RemoteContext working
in order for setUp and tearDown to work
"""
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def tearDown(self):
self.ctx.check_output(["rm", "-rf", self.filepath])
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_remove(self):
self.target.remove()
self.assertRaises(
subprocess.CalledProcessError,
self.ctx.check_output,
["cat", self.filepath]
)
def test_open(self):
f = self.target.open('r')
file_content = f.read()
f.close()
self.assertEqual(file_content, "hello")
self.assertTrue(self.target.fs.exists(self.filepath))
self.assertFalse(self.target.fs.isdir(self.filepath))
def test_context_manager(self):
with self.target.open('r') as f:
file_content = f.read()
self.assertEqual(file_content, "hello")
class TestRemoteTarget(unittest.TestCase):
""" These tests assume RemoteContext working
in order for setUp and tearDown to work
"""
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def tearDown(self):
self.ctx.check_output(["rm", "-rf", self.filepath])
def test_exists(self):
self.assertTrue(self.target.exists())
no_file = RemoteTarget(
"/tmp/_file_that_doesnt_exist_",
working_ssh_host,
)
self.assertFalse(no_file.exists())
def test_remove(self):
self.target.remove()
self.assertRaises(
subprocess.CalledProcessError,
self.ctx.check_output,
["cat", self.filepath]
)
def test_open(self):
f = self.target.open('r')
file_content = f.read()
f.close()
self.assertEqual(file_content, "hello")
self.assertTrue(self.target.fs.exists(self.filepath))
self.assertFalse(self.target.fs.isdir(self.filepath))
def test_context_manager(self):
with self.target.open('r') as f:
file_content = f.read()
self.assertEqual(file_content, "hello")
def get_luigi_target(path):
try:
from urlparse import urlparse
# Python3 compatibility
except ImportError:
from urllib.parse import urlparse
from luigi.s3 import S3Target
from luigi.contrib.ssh import RemoteTarget
from luigi.file import LocalTarget
from luigi.format import GzipFormat
file_format = None
if path.endswith(".gz"):
file_format = GzipFormat(compression_level=9)
if path.startswith("s3n://"):
# Try to always store files in S3 in compressed format.
s3_headers = {}
if file_is_compressible(path):
file_format = GzipFormat(compression_level=9)
s3_headers["Content-Encoding"] = "gzip"
return S3Target(path, format=file_format, headers=s3_headers)
elif path.startswith("ssh://"):
ssh_key_file = config.get("ssh", "ssh-key-file", None)
no_host_key_check = config.get("ssh", "no-host-key-check", None)
p = urlparse(path)
return RemoteTarget(p.path, p.hostname, format=format,
username=p.username,
sshpass=p.password,
key_file=ssh_key_file,
no_host_key_check=no_host_key_check)
return LocalTarget(path, format=file_format)
class TestRemoteFilesystem(unittest.TestCase):
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_mkdir(self):
self.assertFalse(self.fs.isdir(self.directory))
self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False)
self.fs.mkdir(self.directory)
self.assertTrue(self.fs.isdir(self.directory))
# Shouldn't throw
self.fs.mkdir(self.directory)
self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True)
def test_list(self):
with self.target.open('w'):
pass
self.assertEquals([self.target.path], list(self.fs.listdir(self.directory)))
class TestRemoteFilesystem(unittest.TestCase):
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_mkdir(self):
self.assertFalse(self.fs.isdir(self.directory))
self.assertRaises(MissingParentDirectory, self.fs.mkdir, self.directory, parents=False)
self.fs.mkdir(self.directory)
self.assertTrue(self.fs.isdir(self.directory))
# Shouldn't throw
self.fs.mkdir(self.directory)
self.assertRaises(FileAlreadyExists, self.fs.mkdir, self.directory, raise_if_exists=True)
def test_list(self):
with self.target.open('w'):
pass
self.assertEqual([self.target.path], list(self.fs.listdir(self.directory)))
def get_luigi_target(path):
try:
from urlparse import urlparse
# Python3 compatibility
except ImportError:
from urllib.parse import urlparse
from luigi.s3 import S3Target
from luigi.contrib.ssh import RemoteTarget
from luigi.file import LocalTarget
from luigi.format import GzipFormat
file_format = None
if path.endswith(".gz"):
file_format = GzipFormat()
if path.startswith("s3n://"):
return S3Target(path, format=file_format)
elif path.startswith("ssh://"):
ssh_key_file = config.get("ssh", "ssh-key-file", None)
no_host_key_check = config.get("ssh", "no-host-key-check", None)
p = urlparse(path)
return RemoteTarget(p.path,
p.hostname,
format=format,
username=p.username,
sshpass=p.password,
key_file=ssh_key_file,
no_host_key_check=no_host_key_check)
return LocalTarget(path, format=file_format)
def complete(self):
rt = RemoteTarget(host=self.host,
path=self.remote_path,
username=self.user)
if not rt.exists():
return False
# Check hashes:
local_target = luigi.LocalTarget(path=self.local_path)
with local_target.open('r') as reader:
local_hash = hashlib.sha512(reader.read()).hexdigest()
logger.info("LOCAL HASH: %s" % local_hash)
# Read from Remote
with rt.open('r') as reader:
remote_hash = hashlib.sha512(reader.read()).hexdigest()
logger.info("REMOTE HASH: %s" % remote_hash)
# If they match, we are good:
return remote_hash == local_hash
def setUp(self):
self.ctx = RemoteContext(working_ssh_host)
self.filepath = "/tmp/luigi_remote_test.dat"
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.ctx.check_output(["rm", "-rf", self.filepath])
self.ctx.check_output(["echo -n 'hello' >", self.filepath])
def output(self):
"""
Returns the target output for this task.
In this case, a successful execution of this task will create a file on a remote server using SSH.
:return: the target output for this task.
:rtype: object (:py:class:`~luigi.target.Target`)
"""
return RemoteTarget("/tmp/stuff", SSH_HOST)
def output(self):
parameters.setParameters(parameters.data)
status = RemoteTarget(
parameters.data['remote_storage'] +
'demux.corrected.merged.aligned.annotated.aclame.json', SSH_HOST)
# if status: stage.pushStatus(parameters.data['id'], stage.AnnotateAlignment)
return status
def setUp(self):
self.fs = RemoteFileSystem(working_ssh_host)
self.root = '/tmp/luigi-remote-test'
self.directory = self.root + '/dir'
self.filepath = self.directory + '/file'
self.target = RemoteTarget(
self.filepath,
working_ssh_host,
)
self.fs.remote_context.check_output(['rm', '-rf', self.root])
self.addCleanup(self.fs.remote_context.check_output, ['rm', '-rf', self.root])
def test_gzip(self):
t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip)
p = t.open('w')
test_data = 'test'
p.write(test_data)
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
# Using gzip module as validation
cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file)
assert os.system(cmd) == 0
f = gzip.open(self.local_file, 'rb')
self.assertTrue(test_data == f.read())
f.close()
# Verifying our own gzip remote reader
f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r')
self.assertTrue(test_data == f.read())
f.close()
def __init__(self,
path,
file_type='regular',
root_dir=None,
format=None,
**kwargs):
self.is_remote = commons().is_remote
if root_dir:
full_path = os.path.join(root_dir, path)
else:
if self.is_remote:
full_path = os.path.join(commons().remote_root, path)
else:
full_path = os.path.join(commons().local_root, path)
self.file_type = file_type
self.format = format
if self.is_remote:
host = commons().SSH_HOST
port = commons().SSH_PORT
kwargs['port'] = port
self._target = RemoteTarget(full_path,
host,
format=format,
**kwargs)
if file_type == 'apk': # create temporary local copy
self.local_path = os.path.join(
tempfile.gettempdir(),
'luigi-{}-{}.apk'.format(os.path.basename(path),
random.randint(0, 999999999)))
self._target.get(self.local_path)
else:
self._target = LocalTarget(full_path, format=format, **kwargs)
if self.is_remote and self.file_type == 'apk':
path = self.local_path
else:
path = self._target.path
super(ExternalFileTarget,
self).__init__(path) # XXX: check if this is right
def test_gzip(self):
t = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip)
p = t.open('w')
test_data = 'test'
p.write(test_data)
self.assertFalse(self._exists(self.path))
p.close()
self.assertTrue(self._exists(self.path))
# Using gzip module as validation
cmd = 'scp -q %s:%s %s' % (working_ssh_host, self.path, self.local_file)
assert os.system(cmd) == 0
f = gzip.open(self.local_file, 'rb')
self.assertTrue(test_data == f.read())
f.close()
# Verifying our own gzip remote reader
f = RemoteTarget(self.path, working_ssh_host, luigi.format.Gzip).open('r')
self.assertTrue(test_data == f.read())
f.close()
def output(self):
parameters.setParameters(parameters.data)
status = RemoteTarget(
parameters.data['remote_storage'] + "barcodes.fasta", SSH_HOST)
return status
def create_target(self, format=None):
return RemoteTarget(self.path, working_ssh_host, format=format)
def output(self):
parameters.setParameters(parameters.data)
status = RemoteTarget(parameters.data['remote_storage'], SSH_HOST)
# if status: stage.pushStatus(parameters.data['id'], stage.CreateEnv)
return status
def output(self):
status = RemoteTarget(
parameters.data['remote_storage'] + "rawreads.fasta", SSH_HOST)
# if status: stage.pushStatus(parameters.data['id'], self.get_params['status'])
return status
def test_recursion_on_delete(self):
target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host)
with self.assertRaises(RemoteCalledProcessError):
with target.open('w') as fh:
fh.write("test")
def output(self):
return RemoteTarget("/tmp/stuff", SSH_HOST)
def output(self):
remote_path = os.path.join(self.remote_dir, 'remote_out.txt')
return RemoteTarget(path=remote_path, host=self.ssh_host)
def output(self):
# Avoid running if the target files already appear to be set up:
return RemoteTarget(host=self.host,
path="/heritrix/jobs/%s/crawler-beans.cxml" %
self.get_job_name(0),
username=self.user)
def output(self):
return RemoteTarget("~/out", SSH_HOST)
def test_write_with_success(self):
t = RemoteTarget(self.path, working_ssh_host)
with t.open('w') as p:
p.write("hello")
self.assertTrue(t.exists())
def run(self):
# Copy the local file over to the remote place
rt = RemoteTarget(host=self.host,
path=self.remote_path,
username=self.user)
rt.put(self.local_path)
def test_write_with_success(self):
t = RemoteTarget(self.path, working_ssh_host)
with t.open('w') as p:
p.write("hello")
self.assertTrue(t.exists())
def output(self):
status = RemoteTarget(
parameters.data['remote_storage'] + 'demux.corrected.merged',
SSH_HOST)
return status
def test_recursion_on_delete(self):
target = RemoteTarget("/etc/this/does/not/exist", working_ssh_host)
with self.assertRaises(RemoteCalledProcessError):
with target.open('w') as fh:
fh.write("test")
