示例#1
0
    def test_move_across_filesystems(self):
        t = LocalTarget(self.path)
        with t.open('w') as f:
            f.write('test_data')

        def rename_across_filesystems(src, dst):
            err = OSError()
            err.errno = EXDEV
            raise err

        real_rename = os.rename

        def mockrename(src, dst):
            if '-across-fs' in src:
                real_rename(src, dst)
            else:
                rename_across_filesystems(src, dst)

        copy = '%s-across-fs' % self.copy
        with mock.patch('os.rename', mockrename):
            t.move(copy)

        self.assertFalse(os.path.exists(self.path))
        self.assertTrue(os.path.exists(copy))
        self.assertEqual('test_data', LocalTarget(copy).open('r').read())
示例#2
0
 def test_pathlib(self):
     """Test work with pathlib.Path"""
     import pathlib
     path = pathlib.Path(self.path)
     self.assertFalse(path.exists())
     target = LocalTarget(path)
     self.assertFalse(target.exists())
     with path.open('w') as stream:
         stream.write('test me')
     self.assertTrue(target.exists())
示例#3
0
 def test_move(self):
     t = LocalTarget(self.path)
     f = t.open('w')
     test_data = 'test'
     f.write(test_data)
     f.close()
     self.assertTrue(os.path.exists(self.path))
     self.assertFalse(os.path.exists(self.copy))
     t.move(self.copy)
     self.assertFalse(os.path.exists(self.path))
     self.assertTrue(os.path.exists(self.copy))
示例#4
0
    def test_format_chain_reverse(self):
        t = LocalTarget(self.path, luigi.format.UTF8 >> luigi.format.Gzip)

        f = gzip.open(self.path, 'wb')
        f.write(b'\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84')
        f.close()

        with t.open('r') as f:
            b = f.read()

        self.assertEqual(u'我é\nçф', b)
示例#5
0
文件: file_test.py 项目: jooon/luigi
    def test_format_chain_reverse(self):
        t = LocalTarget(self.path, luigi.format.UTF8 >> luigi.format.Gzip)

        f = gzip.open(self.path, "wb")
        f.write(b"\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84")
        f.close()

        with t.open("r") as f:
            b = f.read()

        self.assertEqual(u"我é\nçф", b)
示例#6
0
 def test_copy(self):
     t = LocalTarget(self.path)
     f = t.open('w')
     test_data = 'test'
     f.write(test_data)
     f.close()
     self.assertTrue(os.path.exists(self.path))
     self.assertFalse(os.path.exists(self.copy))
     t.copy(self.copy)
     self.assertTrue(os.path.exists(self.path))
     self.assertTrue(os.path.exists(self.copy))
     self.assertEqual(t.open('r').read(), LocalTarget(self.copy).open('r').read())
示例#7
0
    def test_format_chain(self):
        UTF8WIN = luigi.format.TextFormat(encoding='utf8', newline='\r\n')
        t = LocalTarget(self.path, UTF8WIN >> luigi.format.Gzip)
        a = u'我é\nçф'

        with t.open('w') as f:
            f.write(a)

        f = gzip.open(self.path, 'rb')
        b = f.read()
        f.close()

        self.assertEqual(b'\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84', b)
示例#8
0
文件: file_test.py 项目: jooon/luigi
    def test_format_chain(self):
        UTF8WIN = luigi.format.TextFormat(encoding="utf8", newline="\r\n")
        t = LocalTarget(self.path, UTF8WIN >> luigi.format.Gzip)
        a = u"我é\nçф"

        with t.open("w") as f:
            f.write(a)

        f = gzip.open(self.path, "rb")
        b = f.read()
        f.close()

        self.assertEqual(b"\xe6\x88\x91\xc3\xa9\r\n\xc3\xa7\xd1\x84", b)
示例#9
0
    def test_format_newline(self):
        t = LocalTarget(self.path, luigi.format.SysNewLine)

        with t.open('w') as f:
            f.write(b'a\rb\nc\r\nd')

        with t.open('r') as f:
            b = f.read()

        with open(self.path, 'rb') as f:
            c = f.read()

        self.assertEqual(b'a\nb\nc\nd', b)
        self.assertEqual(b'a\r\nb\r\nc\r\nd', c)
示例#10
0
文件: file_test.py 项目: jooon/luigi
    def test_format_newline(self):
        t = LocalTarget(self.path, luigi.format.SysNewLine)

        with t.open("w") as f:
            f.write(b"a\rb\nc\r\nd")

        with t.open("r") as f:
            b = f.read()

        with open(self.path, "rb") as f:
            c = f.read()

        self.assertEqual(b"a\nb\nc\nd", b)
        self.assertEqual(b"a\r\nb\r\nc\r\nd", c)
示例#11
0
 def valid_io_modes(self, *a, **kw):
     modes = set()
     t = LocalTarget(is_tmp=True)
     t.open('w').close()
     for mode in self.theoretical_io_modes(*a, **kw):
         try:
             io.FileIO(t.path, mode).close()
         except ValueError:
             pass
         except IOError as err:
             if err.errno == EEXIST:
                 modes.add(mode)
             else:
                 raise
         else:
             modes.add(mode)
     return modes
示例#12
0
 def test_open_modes(self):
     t = LocalTarget(is_tmp=True)
     print('Valid write mode:', end=' ')
     for mode in self.valid_write_io_modes_for_luigi():
         print(mode, end=' ')
         p = t.open(mode)
         p.close()
     print()
     print('Valid read mode:', end=' ')
     for mode in self.valid_read_io_modes_for_luigi():
         print(mode, end=' ')
         p = t.open(mode)
         p.close()
     print()
     print('Invalid mode:', end=' ')
     for mode in self.invalid_io_modes_for_luigi():
         print(mode, end=' ')
         self.assertRaises(Exception, t.open, mode)
     print()
示例#13
0
    def test_tmp(self):
        t = LocalTarget(is_tmp=True)
        self.assertFalse(t.exists())
        self.assertFalse(os.path.exists(t.path))
        p = t.open('w')
        print('test', file=p)
        self.assertFalse(t.exists())
        self.assertFalse(os.path.exists(t.path))
        p.close()
        self.assertTrue(t.exists())
        self.assertTrue(os.path.exists(t.path))

        q = t.open('r')
        self.assertEqual(q.readline(), 'test\n')
        q.close()
        path = t.path
        del t  # should remove the underlying file
        self.assertFalse(os.path.exists(path))
示例#14
0
    def test_bzip2(self):
        t = LocalTarget(self.path, luigi.format.Bzip2)
        p = t.open('w')
        test_data = b'test'
        p.write(test_data)
        print(self.path)
        self.assertFalse(os.path.exists(self.path))
        p.close()
        self.assertTrue(os.path.exists(self.path))

        # Using bzip module as validation
        f = bz2.BZ2File(self.path, 'r')
        self.assertTrue(test_data == f.read())
        f.close()

        # Verifying our own bzip2 reader
        f = LocalTarget(self.path, luigi.format.Bzip2).open('r')
        self.assertTrue(test_data == f.read())
        f.close()
示例#15
0
    def test_gzip_with_module(self):
        t = LocalTarget(self.path, luigi.format.Gzip)
        p = t.open('w')
        test_data = b'test'
        p.write(test_data)
        print(self.path)
        self.assertFalse(os.path.exists(self.path))
        p.close()
        self.assertTrue(os.path.exists(self.path))

        # Using gzip module as validation
        f = gzip.open(self.path, 'r')
        self.assertTrue(test_data == f.read())
        f.close()

        # Verifying our own gzip reader
        f = LocalTarget(self.path, luigi.format.Gzip).open('r')
        self.assertTrue(test_data == f.read())
        f.close()
示例#16
0
 def output(self):
     return LocalTarget(PROTOBOWL_QUESTION_PLAYER_COUNTS)
示例#17
0
 def output(self):
     return [
         LocalTarget(QANTA_TRAIN_DATASET_PATH),
         LocalTarget(QANTA_DEV_DATASET_PATH),
         LocalTarget(QANTA_TEST_DATASET_PATH)
     ]
示例#18
0
 def output(self):
     return LocalTarget(QANTA_MAPPED_DATASET_PATH), LocalTarget(
         QANTA_MAP_REPORT_PATH)
示例#19
0
def convert_to_target(path: Path) -> LocalTarget:
    return LocalTarget(str(path))
示例#20
0
 def output(self):
     return LocalTarget(self.path)
示例#21
0
 def get_target(self):
     return LocalTarget(self._tag_dir + "/" + self.get_task_name())
示例#22
0
 def output(self):
     shps = shell('ls {}'.format(os.path.join(self.directory, '*.shp')))
     for path in shps:
         yield LocalTarget(path)
示例#23
0
 def output(self):
     return LocalTarget("configurations/%s.pickle" % self.config_name)
示例#24
0
 def output(self):
     return LocalTarget(
         os.path.join('tmp', classpath(self),
                      unqualified_task_id(self.task_id) + '.dump'))
示例#25
0
 def output(self):
     return LocalTarget("datasets/fruit-images-dataset/%d" % self.dataset_version)
示例#26
0
 def output(self):
     return LocalTarget("/tmp/%s_v%d.%s" % (self.dataset_name,
                                            self.dataset_version,
                                            self.file_fomat))
示例#27
0
 def output(self):
     return LocalTarget("serving/%s/%d" % (self.model_name,
                                           self.model_version))
示例#28
0
 def output(self):
     return LocalTarget("evaluation/%d/%s.json" % (self.model_version, self.model_name))
示例#29
0
 def output(self):
     return LocalTarget("baseline/%s.json" % self.baseline_name)
示例#30
0
 def output(self):
     filenames = shell('ls {}'.format(os.path.join(
         self.directory, self.geography, '*.zip'))).split('\n')
     for path in filenames:
         yield LocalTarget(path)
示例#31
0
 def output(self):
     prefix = self.date.strftime("%m-%d-%Y")
     return LocalTarget("datasets/reddit_ds_got/daily/%s/cleaned.csv" %
                        prefix)
示例#32
0
 def output(self):
     return LocalTarget(os.path.join(dirname, "foo"))
示例#33
0
 def output(self):
     return LocalTarget("model/%d" % self.version)
 def output(self):
     outputFile = os.path.join(self.paths["state"],
                               "AddMergedOverviews.json")
     return LocalTarget(outputFile)
示例#35
0
 def output(self):
     return LocalTarget('sales.csv')
示例#36
0
 def output(self):
     return [
         LocalTarget(QANTA_TORCH_TRAIN_LOCAL_PATH),
         LocalTarget(QANTA_TORCH_VAL_LOCAL_PATH),
         LocalTarget(QANTA_TORCH_DEV_LOCAL_PATH)
     ]
示例#37
0
 def output(self):
     return LocalTarget('inventory.csv')
示例#38
0
 def output(self):
     return LocalTarget(QANTA_SQL_DATASET_PATH)
示例#39
0
 def output(self):
     return LocalTarget('orders.csv')
示例#40
0
 def output(self):
     return LocalTarget(QANTA_FOLDED_DATASET_PATH)
示例#41
0
 def output(self):
     return LocalTarget('output/{}.png'.format("whatever"))
示例#42
0
 def output(self):
     return (
         LocalTarget(ANSWER_MAP_PATH), LocalTarget(UNBOUND_ANSWER_PATH),
         LocalTarget('data/external/answer_mapping/automatic_report.json'))
示例#43
0
 def output(self):
     """Check."""
     return LocalTarget(os.path.abspath(self.path))
示例#44
0
 def output(self):
     return LocalTarget("data/to_upper.txt")
示例#45
0
 def output(self):
     return LocalTarget("model/%d/%s.h5" % (self.model_version, self.model_name))