Beispiel #1
0
    def test_is_methods(self):
        container = self.test_container
        container = self.test_container
        file_with_prefix = stor.join(container, 'analysis.txt')

        # ensure container is created but empty
        container.post()
        self.assertTrue(stor.isdir(container))
        self.assertFalse(stor.isfile(container))
        self.assertTrue(stor.exists(container))
        self.assertFalse(stor.listdir(container))

        folder = stor.join(container, 'analysis')
        subfolder = stor.join(container, 'analysis', 'alignments')
        file_in_folder = stor.join(container, 'analysis', 'alignments',
                                   'bam.bam')
        self.assertFalse(stor.exists(file_in_folder))
        self.assertFalse(stor.isdir(folder))
        self.assertFalse(stor.isdir(folder + '/'))
        with stor.open(file_with_prefix, 'w') as fp:
            fp.write('data\n')
        self.assertFalse(stor.isdir(folder))
        self.assertTrue(stor.isfile(file_with_prefix))

        with stor.open(file_in_folder, 'w') as fp:
            fp.write('blah.txt\n')

        self.assertTrue(stor.isdir(folder))
        self.assertFalse(stor.isfile(folder))
        self.assertTrue(stor.isdir(subfolder))
Beispiel #2
0
    def test_temp_url(self):
        basic_file = 'test.txt'
        complex_file = 'my test?file=special_chars.txt'
        with NamedTemporaryDirectory(change_dir=True) as tmp_d:
            nested_tmp_dir = stor.join(tmp_d, 'tmp')
            os.mkdir(nested_tmp_dir)
            basic_file_p = stor.join(nested_tmp_dir, basic_file)
            complex_file_p = stor.join(nested_tmp_dir, 'my test?file=special_chars.txt')

            with stor.open(basic_file_p, 'w') as f:
                f.write('basic test')
            with stor.open(complex_file_p, 'w') as f:
                f.write('complex test')

            self.test_container.upload(['.'])

        with NamedTemporaryDirectory(change_dir=True) as tmp_d:
            basic_obj = stor.Path(
                stor.join(self.test_container, 'tmp', basic_file))
            basic_temp_url = basic_obj.temp_url(inline=False, filename=basic_file)
            r = requests.get(basic_temp_url)
            self.assertEquals(r.content, 'basic test')
            self.assertEquals(r.headers['Content-Disposition'],
                              'attachment; filename="test.txt"; filename*=UTF-8\'\'test.txt')

            complex_obj = stor.Path(
                stor.join(self.test_container, 'tmp', complex_file))
            complex_temp_url = complex_obj.temp_url(inline=False, filename=complex_file)
            r = requests.get(complex_temp_url)
            self.assertEquals(r.content, 'complex test')
            self.assertEquals(r.headers['Content-Disposition'],
                              'attachment; filename="my test%3Ffile%3Dspecial_chars.txt"; filename*=UTF-8\'\'my%20test%3Ffile%3Dspecial_chars.txt')  # noqa
Beispiel #3
0
        def test_read_string_from_text(self):
            test_file = self.test_dir / 'test_file.txt'
            with stor.open(test_file, mode='w') as fp:
                fp.write(STRING_STRING)

            with stor.open(test_file, mode='r') as fp:
                result = fp.read()
            assert result == STRING_STRING
Beispiel #4
0
        def test_read_bytes_from_binary(self):
            test_file = self.test_dir / 'test_file.txt'
            with stor.open(test_file, mode='wb') as fp:
                fp.write(BYTE_STRING)

            with stor.open(test_file, mode='rb') as fp:
                result = fp.read()
            assert result == BYTE_STRING
    def test_read_string_from_text(self):
        test_file = self.test_dir / 'test_file.txt'
        with stor.settings.use(self.DX_WAIT_SETTINGS):
            with stor.open(test_file, mode='w') as fp:
                fp.write(test_integration.STRING_STRING)

        with stor.open(test_file, mode='r') as fp:
            result = fp.read()
        assert result == test_integration.STRING_STRING
    def test_read_bytes_from_binary(self):
        test_file = self.test_dir / 'test_file.txt'
        with stor.settings.use(self.DX_WAIT_SETTINGS):
            with stor.open(test_file, mode='wb') as fp:
                fp.write(test_integration.BYTE_STRING)

        with stor.open(test_file, mode='rb') as fp:
            result = fp.read()
        assert result == test_integration.BYTE_STRING
Beispiel #7
0
        def test_custom_encoding_text(self):
            test_file = self.test_dir / 'test_file.txt'
            with stor.open(test_file, mode='w', encoding='utf-16') as fp:
                fp.write(STRING_STRING)

            with stor.open(test_file, mode='r', encoding='utf-16') as fp:
                result = fp.read()
            assert result == STRING_STRING

            with pytest.raises(UnicodeDecodeError):
                with stor.open(test_file, mode='r', encoding='utf-8') as fp:
                    result = fp.read()
Beispiel #8
0
    def test_on_del_no_writes(self, mock_read_object, mock_write_object):
        fileobj = stor.open(stor.join(self.drive, 'B/C/obj'), 'w')
        del fileobj
        gc.collect()

        self.assertFalse(mock_read_object.called)
        self.assertFalse(mock_write_object.called)

        fileobj = stor.open(stor.join(self.drive, 'B/C/obj'), 'r')
        del fileobj
        gc.collect()

        self.assertFalse(mock_read_object.called)
        self.assertFalse(mock_write_object.called)
Beispiel #9
0
    def test_invalid_open(self):
        pth = stor.join(self.drive, 'B/C/D')
        with self.assertRaisesRegexp(ValueError, 'mode'):
            # keep reference here
            f = stor.open(pth, 'invalid')  # noqa
            assert False, 'should error before this'  # pragma: no cover

        with self.assertRaisesRegexp(ValueError, 'mode'):
            with stor.open(pth, 'invalid'):
                assert False, 'should error before this'  # pragma: no cover

        with self.assertRaisesRegexp(ValueError, 'mode'):
            with stor.Path(pth).open('invalid'):
                assert False, 'should error before this'  # pragma: no cover
Beispiel #10
0
def get_data_manifest_contents(manifest_dir):
    """Reads the manifest file and returns a set of expected files"""
    import stor

    manifest = manifest_dir / DATA_MANIFEST_FILE_NAME
    with stor.open(manifest, 'r') as manifest_file:
        return [f.strip() for f in manifest_file.readlines() if f.strip()]
Beispiel #11
0
        def test_file_read_write(self):
            non_with_file = self.test_dir / 'nonwithfile.txt'
            test_file = self.test_dir / 'test_file.txt'
            copy_file = self.test_dir / 'copy_file.txt'

            fp = stor.open(non_with_file, mode='wb')
            fp.write('blah')
            del fp

            self.assertTrue(non_with_file.exists())
            self.assertTrue(non_with_file.isfile())
            self.assertFalse(non_with_file.isdir())

            with test_file.open(mode='wb') as obj:
                obj.write('this is a test\n')
                obj.write('this is another line.\n')

            self.assertTrue(test_file.exists())
            self.assertTrue(test_file.isfile())
            self.assertFalse(test_file.isdir())

            with test_file.open(mode='rb') as obj:
                with copy_file.open(mode='wb') as copy_obj:
                    copy_obj.write(obj.read())

            self.assertTrue(copy_file.exists())
            self.assertTrue(copy_file.isfile())
            self.assertFalse(copy_file.isdir())

            test_contents = test_file.open(mode='rb').read()
            copy_contents = copy_file.open(mode='rb').read()
            self.assertEquals(test_contents,
                              'this is a test\nthis is another line.\n')
            self.assertEquals(test_contents, copy_contents)
Beispiel #12
0
    def test_open_works(self):
        with tempfile.NamedTemporaryFile() as f:
            p = stor.Path(f.name).open()
            p.close()

        with tempfile.NamedTemporaryFile() as f:
            p = stor.open(f.name)
            p.close()
Beispiel #13
0
    def test_metadata_pulling(self):
        file_in_folder = stor.join(self.test_container, 'somefile.svg')
        with stor.open(file_in_folder, 'w') as fp:
            fp.write('12345\n')

        self.assertEqual(stor.getsize(file_in_folder), 6)
        stat_data = stor.Path(file_in_folder).stat()
        self.assertIn('Content-Type', stat_data)
        self.assertEqual(stat_data['Content-Type'], 'image/svg+xml')
Beispiel #14
0
 def test_gzip_on_remote(self):
     local_gzip = os.path.join(os.path.dirname(__file__),
                               'file_data/s_3_2126.bcl.gz')
     remote_gzip = stor.join(self.test_dir, stor.basename(local_gzip))
     stor.copy(local_gzip, remote_gzip)
     with stor.open(remote_gzip) as fp:
         with gzip.GzipFile(fileobj=fp) as remote_gzip_fp:
             with gzip.open(local_gzip) as local_gzip_fp:
                 assert_same_data(remote_gzip_fp, local_gzip_fp)
Beispiel #15
0
 def test_empty_buffer_no_writes(self, mock_read_object, mock_write_object):
     # NOTE: this tests that our current description (only non-empty buffers are uploaded) is
     # enshrined.
     fileobj = stor.open(stor.join(self.drive, 'B/C/obj'), 'w')
     fileobj.flush()
     self.assertFalse(fileobj._buffer)
     fileobj.write('')
     fileobj.flush()
     fileobj.close()
     self.assertFalse(mock_read_object.called)
     self.assertFalse(mock_write_object.called)
Beispiel #16
0
    def test_readable_writable_seekable(self):
        pth = self.normal_path
        read_obj = pth.open(mode='r')
        self.assertTrue(read_obj.readable())
        self.assertFalse(read_obj.writable())
        self.assertTrue(read_obj.seekable())

        write_obj = stor.open(stor.join(self.drive, 'B/C/obj'), 'w')
        self.assertFalse(write_obj.readable())
        self.assertTrue(write_obj.writable())
        self.assertTrue(write_obj.seekable())
Beispiel #17
0
    def test_works_with_gzip(self, mock_read_object):
        gzip_path = stor.join(stor.dirname(__file__), 'file_data',
                              's_3_2126.bcl.gz')
        text = stor.open(gzip_path, 'rb').read()
        mock_read_object.return_value = text
        fileobj = stor.open(stor.join(self.drive, 'A/C/s_3_2126.bcl.gz'), 'rb')

        with fileobj:
            with gzip.GzipFile(fileobj=fileobj) as fp:
                with gzip.open(gzip_path) as gzip_fp:
                    assert_same_data(fp, gzip_fp)

        fileobj = stor.open(stor.join(self.drive, 'A/C/s_3_2126.bcl.gz'), 'rb')

        with fileobj:
            with gzip.GzipFile(fileobj=fileobj) as fp:
                with gzip.open(gzip_path) as gzip_fp:
                    # after seeking should still be same
                    fp.seek(3)
                    gzip_fp.seek(3)
                    assert_same_data(fp, gzip_fp)
Beispiel #18
0
    def test_all_segment_container_types_are_deleted(self):
        segment_containers = [stor.join('swift://' + self.test_container.tenant,
                                        fmt % self.test_container.name)
                              for fmt in ('.segments_%s', '%s+segments', '%s_segments')]
        all_containers = segment_containers + [self.test_container]

        test_files = [stor.join(c, 'test_file_tbdeleted.txt') for c in all_containers]
        for t in test_files:
            with stor.open(t, 'w') as fp:
                fp.write('testtxt\n')
        assert all(t.exists() for t in test_files)
        stor.rmtree(self.test_container)
        for t in test_files:
            assert not t.exists(), 'Did not delete %s' % t
Beispiel #19
0
    def test_gzip_on_remote(self):
        self._skip_if_filesystem_python3(self.test_dir)
        local_gzip = os.path.join(os.path.dirname(__file__),
                                  'file_data/s_3_2126.bcl.gz')
        remote_gzip = stor.join(self.test_dir, stor.basename(local_gzip))
        stor.copy(local_gzip, remote_gzip)
        file_h = dxpy.DXFile(dxid=remote_gzip.canonical_resource,
                             project=remote_gzip.canonical_project)
        file_h.wait_on_close(20)  # wait for file to go to closed state

        with stor.open(remote_gzip, mode='rb') as fp:
            with gzip.GzipFile(fileobj=fp) as remote_gzip_fp:
                with gzip.open(local_gzip) as local_gzip_fp:
                    assert_same_data(remote_gzip_fp, local_gzip_fp)
Beispiel #20
0
def generate_and_save_data_manifest(manifest_dir, data_manifest_contents):
    """Generates a data manifest for a given directory and saves it.

    Args:
        manifest_dir (str): The directory in which the manifest will be saved
        data_manifest_contents (List[str]): The list of all objects that will
            be part of the manifest.
    """
    import stor
    from stor import Path

    manifest_file_name = Path(manifest_dir) / DATA_MANIFEST_FILE_NAME
    with stor.open(manifest_file_name, 'w') as out_file:
        contents = '\n'.join(data_manifest_contents) + '\n'
        out_file.write(contents)
Beispiel #21
0
 def test_w_broken_symlink(self):
     swift_dir = (Path(__file__).expand().abspath().parent / 'swift_upload')
     with utils.NamedTemporaryDirectory(dir=swift_dir) as tmp_dir:
         symlink = tmp_dir / 'broken.symlink'
         symlink_source = tmp_dir / 'nonexistent'
         # put something in symlink source so that Python doesn't complain
         with stor.open(symlink_source, 'w') as fp:
             fp.write('blah')
         os.symlink(symlink_source, symlink)
         uploads = utils.walk_files_and_dirs([swift_dir])
         self.assertEquals(
             set(uploads),
             set([
                 swift_dir / 'file1',
                 swift_dir / 'data_dir' / 'file2',
                 symlink,
                 symlink_source,
             ]))
         # NOW: destroy it with fire and we have empty directory
         os.remove(symlink_source)
         uploads = utils.walk_files_and_dirs([swift_dir])
         self.assertEquals(
             set(uploads),
             set([
                 swift_dir / 'file1',
                 tmp_dir,
                 swift_dir / 'data_dir' / 'file2',
             ]))
         # but put a sub directory, now tmp_dir doesn't show up
         subdir = tmp_dir / 'subdir'
         subdir.makedirs_p()
         uploads = utils.walk_files_and_dirs([swift_dir])
         self.assertEquals(
             set(uploads),
             set([
                 swift_dir / 'file1',
                 subdir,
                 swift_dir / 'data_dir' / 'file2',
             ]))
Beispiel #22
0
    def test_file_read_write(self):
        self._skip_if_filesystem_python3(self.test_dir)
        non_with_file = self.test_dir / 'nonwithfile.txt'
        test_file = self.test_dir / 'test_file.txt'
        copy_file = self.test_dir / 'copy_file.txt'

        fp = stor.open(non_with_file, mode='wb')
        # File opened in wb mode requires: bytes on py3k, str on py27
        fp.write('blah'.encode())
        del fp

        self.assertTrue(non_with_file.exists())
        self.assertTrue(non_with_file.isfile())
        self.assertFalse(non_with_file.isdir())

        with stor.settings.use(self.DX_WAIT_SETTINGS):
            with test_file.open(mode='wb') as obj:
                obj.write('this is a test\n'.encode())
                obj.write('this is another line.\n'.encode())

        self.assertTrue(test_file.exists())
        self.assertTrue(test_file.isfile())
        self.assertFalse(test_file.isdir())

        with stor.settings.use(self.DX_WAIT_SETTINGS):
            with test_file.open(mode='rb') as obj:
                with copy_file.open(mode='wb') as copy_obj:
                    copy_obj.write(obj.read())

        self.assertTrue(copy_file.exists())
        self.assertTrue(copy_file.isfile())
        self.assertFalse(copy_file.isdir())

        test_contents = test_file.open(mode='rb').read()
        copy_contents = copy_file.open(mode='rb').read()
        self.assertEquals(test_contents,
                          'this is a test\nthis is another line.\n'.encode())
        self.assertEquals(test_contents, copy_contents)
Beispiel #23
0
 def test_write_bytes_to_binary(self):
     test_file = self.test_dir / 'test_file.txt'
     with stor.open(test_file, mode='wb') as fp:
         fp.write(BYTE_STRING)
Beispiel #24
0
 def test_functional_open(self):
     with tempfile.NamedTemporaryFile() as f:
         with stor.open(f.name, 'w') as f:
             f.write('blah')
 def test_write_string_to_binary(self):  # pragma: no cover
     test_file = self.test_dir / 'test_file.txt'
     with stor.open(test_file, mode='wb') as fp:
         fp.write(STRING_STRING)
 def test_encoding_typeerror_py2(self):
     test_file = self.test_dir / 'test_file.txt'
     with pytest.raises(TypeError, regex='encoding'):
         stor.open(test_file, mode='r', encoding='utf-8')
     with pytest.raises(TypeError, regex='encoding'):
         stor.Path(test_file).open(mode='r', encoding='utf-8')
Beispiel #27
0
 def test_functional_open(self):
     with tempfile.NamedTemporaryFile() as f:
         with stor.open(f.name, 'w', swift_upload_kwargs={}) as f:
             f.write('blah')
Beispiel #28
0
 def test_write_string_to_text(self):
     test_file = self.test_dir / 'test_file.txt'
     with stor.open(test_file, mode='w') as fp:
         fp.write(STRING_STRING)
Beispiel #29
0
 def test_write_bytes_to_text(self):  # pragma: no cover
     with pytest.raises(TypeError):
         test_file = self.test_dir / 'test_file.txt'
         with stor.open(test_file, mode='w') as fp:
             fp.write(BYTE_STRING)
Beispiel #30
0
 def test_write_string_to_binary(self):
     with pytest.raises(TypeError):
         test_file = self.test_dir / 'test_file.txt'
         with stor.open(test_file, mode='wb') as fp:
             fp.write(STRING_STRING)