def test_move_across_filesystems(self):
t = LocalTarget(self.path)
with t.open('w') as f:
f.write('test_data')
def rename_across_filesystems(src, dst):
err = OSError()
err.errno = EXDEV
raise err
real_rename = os.rename
def mockrename(src, dst):
if '-across-fs' in src:
real_rename(src, dst)
else:
rename_across_filesystems(src, dst)
copy = '%s-across-fs' % self.copy
with mock.patch('os.rename', mockrename):
t.move(copy)
self.assertFalse(os.path.exists(self.path))
self.assertTrue(os.path.exists(copy))
self.assertEqual('test_data', LocalTarget(copy).open('r').read())
def test_pathlib(self):
"""Test work with pathlib.Path"""
import pathlib
path = pathlib.Path(self.path)
self.assertFalse(path.exists())
target = LocalTarget(path)
self.assertFalse(target.exists())
with path.open('w') as stream:
stream.write('test me')
self.assertTrue(target.exists())
def test_move(self):
t = LocalTarget(self.path)
f = t.open('w')
test_data = 'test'
f.write(test_data)
f.close()
self.assertTrue(os.path.exists(self.path))
self.assertFalse(os.path.exists(self.copy))
t.move(self.copy)
self.assertFalse(os.path.exists(self.path))
self.assertTrue(os.path.exists(self.copy))
def test_format_chain_reverse(self):
t = LocalTarget(self.path, luigi.format.UTF8 >> luigi.format.Gzip)
f = gzip.open(self.path, 'wb')
f.write(b'\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84')
f.close()
with t.open('r') as f:
b = f.read()
self.assertEqual(u'我é\nçф', b)
def test_format_chain_reverse(self):
t = LocalTarget(self.path, luigi.format.UTF8 >> luigi.format.Gzip)
f = gzip.open(self.path, "wb")
f.write(b"\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84")
f.close()
with t.open("r") as f:
b = f.read()
self.assertEqual(u"我é\nçф", b)
def test_copy(self):
t = LocalTarget(self.path)
f = t.open('w')
test_data = 'test'
f.write(test_data)
f.close()
self.assertTrue(os.path.exists(self.path))
self.assertFalse(os.path.exists(self.copy))
t.copy(self.copy)
self.assertTrue(os.path.exists(self.path))
self.assertTrue(os.path.exists(self.copy))
self.assertEqual(t.open('r').read(), LocalTarget(self.copy).open('r').read())
def test_format_chain(self):
UTF8WIN = luigi.format.TextFormat(encoding='utf8', newline='\r\n')
t = LocalTarget(self.path, UTF8WIN >> luigi.format.Gzip)
a = u'我é\nçф'
with t.open('w') as f:
f.write(a)
f = gzip.open(self.path, 'rb')
b = f.read()
f.close()
self.assertEqual(b'\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84', b)
def test_format_chain(self):
UTF8WIN = luigi.format.TextFormat(encoding="utf8", newline="\r\n")
t = LocalTarget(self.path, UTF8WIN >> luigi.format.Gzip)
a = u"我é\nçф"
with t.open("w") as f:
f.write(a)
f = gzip.open(self.path, "rb")
b = f.read()
f.close()
self.assertEqual(b"\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84", b)
def test_format_newline(self):
t = LocalTarget(self.path, luigi.format.SysNewLine)
with t.open('w') as f:
f.write(b'a\rb\nc\r\nd')
with t.open('r') as f:
b = f.read()
with open(self.path, 'rb') as f:
c = f.read()
self.assertEqual(b'a\nb\nc\nd', b)
self.assertEqual(b'a\r\nb\r\nc\r\nd', c)
def test_format_newline(self):
t = LocalTarget(self.path, luigi.format.SysNewLine)
with t.open("w") as f:
f.write(b"a\rb\nc\r\nd")
with t.open("r") as f:
b = f.read()
with open(self.path, "rb") as f:
c = f.read()
self.assertEqual(b"a\nb\nc\nd", b)
self.assertEqual(b"a\r\nb\r\nc\r\nd", c)
def valid_io_modes(self, *a, **kw):
modes = set()
t = LocalTarget(is_tmp=True)
t.open('w').close()
for mode in self.theoretical_io_modes(*a, **kw):
try:
io.FileIO(t.path, mode).close()
except ValueError:
pass
except IOError as err:
if err.errno == EEXIST:
modes.add(mode)
else:
raise
else:
modes.add(mode)
return modes
def test_open_modes(self):
t = LocalTarget(is_tmp=True)
print('Valid write mode:', end=' ')
for mode in self.valid_write_io_modes_for_luigi():
print(mode, end=' ')
p = t.open(mode)
p.close()
print()
print('Valid read mode:', end=' ')
for mode in self.valid_read_io_modes_for_luigi():
print(mode, end=' ')
p = t.open(mode)
p.close()
print()
print('Invalid mode:', end=' ')
for mode in self.invalid_io_modes_for_luigi():
print(mode, end=' ')
self.assertRaises(Exception, t.open, mode)
print()
def test_tmp(self):
t = LocalTarget(is_tmp=True)
self.assertFalse(t.exists())
self.assertFalse(os.path.exists(t.path))
p = t.open('w')
print('test', file=p)
self.assertFalse(t.exists())
self.assertFalse(os.path.exists(t.path))
p.close()
self.assertTrue(t.exists())
self.assertTrue(os.path.exists(t.path))
q = t.open('r')
self.assertEqual(q.readline(), 'test\n')
q.close()
path = t.path
del t # should remove the underlying file
self.assertFalse(os.path.exists(path))
def test_bzip2(self):
t = LocalTarget(self.path, luigi.format.Bzip2)
p = t.open('w')
test_data = b'test'
p.write(test_data)
print(self.path)
self.assertFalse(os.path.exists(self.path))
p.close()
self.assertTrue(os.path.exists(self.path))
# Using bzip module as validation
f = bz2.BZ2File(self.path, 'r')
self.assertTrue(test_data == f.read())
f.close()
# Verifying our own bzip2 reader
f = LocalTarget(self.path, luigi.format.Bzip2).open('r')
self.assertTrue(test_data == f.read())
f.close()
def test_gzip_with_module(self):
t = LocalTarget(self.path, luigi.format.Gzip)
p = t.open('w')
test_data = b'test'
p.write(test_data)
print(self.path)
self.assertFalse(os.path.exists(self.path))
p.close()
self.assertTrue(os.path.exists(self.path))
# Using gzip module as validation
f = gzip.open(self.path, 'r')
self.assertTrue(test_data == f.read())
f.close()
# Verifying our own gzip reader
f = LocalTarget(self.path, luigi.format.Gzip).open('r')
self.assertTrue(test_data == f.read())
f.close()
def output(self):
return LocalTarget(PROTOBOWL_QUESTION_PLAYER_COUNTS)
def output(self):
return [
LocalTarget(QANTA_TRAIN_DATASET_PATH),
LocalTarget(QANTA_DEV_DATASET_PATH),
LocalTarget(QANTA_TEST_DATASET_PATH)
]
def output(self):
return LocalTarget(QANTA_MAPPED_DATASET_PATH), LocalTarget(
QANTA_MAP_REPORT_PATH)
def convert_to_target(path: Path) -> LocalTarget:
return LocalTarget(str(path))
def output(self):
return LocalTarget(self.path)
def get_target(self):
return LocalTarget(self._tag_dir + "/" + self.get_task_name())
def output(self):
shps = shell('ls {}'.format(os.path.join(self.directory, '*.shp')))
for path in shps:
yield LocalTarget(path)
def output(self):
return LocalTarget("configurations/%s.pickle" % self.config_name)
def output(self):
return LocalTarget(
os.path.join('tmp', classpath(self),
unqualified_task_id(self.task_id) + '.dump'))
def output(self):
return LocalTarget("datasets/fruit-images-dataset/%d" % self.dataset_version)
def output(self):
return LocalTarget("/tmp/%s_v%d.%s" % (self.dataset_name,
self.dataset_version,
self.file_fomat))
def output(self):
return LocalTarget("serving/%s/%d" % (self.model_name,
self.model_version))
def output(self):
return LocalTarget("evaluation/%d/%s.json" % (self.model_version, self.model_name))
def output(self):
return LocalTarget("baseline/%s.json" % self.baseline_name)
def output(self):
filenames = shell('ls {}'.format(os.path.join(
self.directory, self.geography, '*.zip'))).split('\n')
for path in filenames:
yield LocalTarget(path)
def output(self):
prefix = self.date.strftime("%m-%d-%Y")
return LocalTarget("datasets/reddit_ds_got/daily/%s/cleaned.csv" %
prefix)
def output(self):
return LocalTarget(os.path.join(dirname, "foo"))
def output(self):
return LocalTarget("model/%d" % self.version)
def output(self):
outputFile = os.path.join(self.paths["state"],
"AddMergedOverviews.json")
return LocalTarget(outputFile)
def output(self):
return LocalTarget('sales.csv')
def output(self):
return [
LocalTarget(QANTA_TORCH_TRAIN_LOCAL_PATH),
LocalTarget(QANTA_TORCH_VAL_LOCAL_PATH),
LocalTarget(QANTA_TORCH_DEV_LOCAL_PATH)
]
def output(self):
return LocalTarget('inventory.csv')
def output(self):
return LocalTarget(QANTA_SQL_DATASET_PATH)
def output(self):
return LocalTarget('orders.csv')
def output(self):
return LocalTarget(QANTA_FOLDED_DATASET_PATH)
def output(self):
return LocalTarget('output/{}.png'.format("whatever"))
def output(self):
return (
LocalTarget(ANSWER_MAP_PATH), LocalTarget(UNBOUND_ANSWER_PATH),
LocalTarget('data/external/answer_mapping/automatic_report.json'))
def output(self):
"""Check."""
return LocalTarget(os.path.abspath(self.path))
def output(self):
return LocalTarget("data/to_upper.txt")
def output(self):
return LocalTarget("model/%d/%s.h5" % (self.model_version, self.model_name))
