def test_is_methods(self):
container = self.test_container
container = self.test_container
file_with_prefix = stor.join(container, 'analysis.txt')
# ensure container is created but empty
container.post()
self.assertTrue(stor.isdir(container))
self.assertFalse(stor.isfile(container))
self.assertTrue(stor.exists(container))
self.assertFalse(stor.listdir(container))
folder = stor.join(container, 'analysis')
subfolder = stor.join(container, 'analysis', 'alignments')
file_in_folder = stor.join(container, 'analysis', 'alignments',
'bam.bam')
self.assertFalse(stor.exists(file_in_folder))
self.assertFalse(stor.isdir(folder))
self.assertFalse(stor.isdir(folder + '/'))
with stor.open(file_with_prefix, 'w') as fp:
fp.write('data\n')
self.assertFalse(stor.isdir(folder))
self.assertTrue(stor.isfile(file_with_prefix))
with stor.open(file_in_folder, 'w') as fp:
fp.write('blah.txt\n')
self.assertTrue(stor.isdir(folder))
self.assertFalse(stor.isfile(folder))
self.assertTrue(stor.isdir(subfolder))
def test_temp_url(self):
basic_file = 'test.txt'
complex_file = 'my test?file=special_chars.txt'
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
nested_tmp_dir = stor.join(tmp_d, 'tmp')
os.mkdir(nested_tmp_dir)
basic_file_p = stor.join(nested_tmp_dir, basic_file)
complex_file_p = stor.join(nested_tmp_dir, 'my test?file=special_chars.txt')
with stor.open(basic_file_p, 'w') as f:
f.write('basic test')
with stor.open(complex_file_p, 'w') as f:
f.write('complex test')
self.test_container.upload(['.'])
with NamedTemporaryDirectory(change_dir=True) as tmp_d:
basic_obj = stor.Path(
stor.join(self.test_container, 'tmp', basic_file))
basic_temp_url = basic_obj.temp_url(inline=False, filename=basic_file)
r = requests.get(basic_temp_url)
self.assertEquals(r.content, 'basic test')
self.assertEquals(r.headers['Content-Disposition'],
'attachment; filename="test.txt"; filename*=UTF-8\'\'test.txt')
complex_obj = stor.Path(
stor.join(self.test_container, 'tmp', complex_file))
complex_temp_url = complex_obj.temp_url(inline=False, filename=complex_file)
r = requests.get(complex_temp_url)
self.assertEquals(r.content, 'complex test')
self.assertEquals(r.headers['Content-Disposition'],
'attachment; filename="my test%3Ffile%3Dspecial_chars.txt"; filename*=UTF-8\'\'my%20test%3Ffile%3Dspecial_chars.txt') # noqa
def test_read_string_from_text(self):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='w') as fp:
fp.write(STRING_STRING)
with stor.open(test_file, mode='r') as fp:
result = fp.read()
assert result == STRING_STRING
def test_read_bytes_from_binary(self):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='wb') as fp:
fp.write(BYTE_STRING)
with stor.open(test_file, mode='rb') as fp:
result = fp.read()
assert result == BYTE_STRING
def test_read_string_from_text(self):
test_file = self.test_dir / 'test_file.txt'
with stor.settings.use(self.DX_WAIT_SETTINGS):
with stor.open(test_file, mode='w') as fp:
fp.write(test_integration.STRING_STRING)
with stor.open(test_file, mode='r') as fp:
result = fp.read()
assert result == test_integration.STRING_STRING
def test_read_bytes_from_binary(self):
test_file = self.test_dir / 'test_file.txt'
with stor.settings.use(self.DX_WAIT_SETTINGS):
with stor.open(test_file, mode='wb') as fp:
fp.write(test_integration.BYTE_STRING)
with stor.open(test_file, mode='rb') as fp:
result = fp.read()
assert result == test_integration.BYTE_STRING
def test_custom_encoding_text(self):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='w', encoding='utf-16') as fp:
fp.write(STRING_STRING)
with stor.open(test_file, mode='r', encoding='utf-16') as fp:
result = fp.read()
assert result == STRING_STRING
with pytest.raises(UnicodeDecodeError):
with stor.open(test_file, mode='r', encoding='utf-8') as fp:
result = fp.read()
def test_on_del_no_writes(self, mock_read_object, mock_write_object):
fileobj = stor.open(stor.join(self.drive, 'B/C/obj'), 'w')
del fileobj
gc.collect()
self.assertFalse(mock_read_object.called)
self.assertFalse(mock_write_object.called)
fileobj = stor.open(stor.join(self.drive, 'B/C/obj'), 'r')
del fileobj
gc.collect()
self.assertFalse(mock_read_object.called)
self.assertFalse(mock_write_object.called)
def test_invalid_open(self):
pth = stor.join(self.drive, 'B/C/D')
with self.assertRaisesRegexp(ValueError, 'mode'):
# keep reference here
f = stor.open(pth, 'invalid') # noqa
assert False, 'should error before this' # pragma: no cover
with self.assertRaisesRegexp(ValueError, 'mode'):
with stor.open(pth, 'invalid'):
assert False, 'should error before this' # pragma: no cover
with self.assertRaisesRegexp(ValueError, 'mode'):
with stor.Path(pth).open('invalid'):
assert False, 'should error before this' # pragma: no cover
def get_data_manifest_contents(manifest_dir):
"""Reads the manifest file and returns a set of expected files"""
import stor
manifest = manifest_dir / DATA_MANIFEST_FILE_NAME
with stor.open(manifest, 'r') as manifest_file:
return [f.strip() for f in manifest_file.readlines() if f.strip()]
def test_file_read_write(self):
non_with_file = self.test_dir / 'nonwithfile.txt'
test_file = self.test_dir / 'test_file.txt'
copy_file = self.test_dir / 'copy_file.txt'
fp = stor.open(non_with_file, mode='wb')
fp.write('blah')
del fp
self.assertTrue(non_with_file.exists())
self.assertTrue(non_with_file.isfile())
self.assertFalse(non_with_file.isdir())
with test_file.open(mode='wb') as obj:
obj.write('this is a test\n')
obj.write('this is another line.\n')
self.assertTrue(test_file.exists())
self.assertTrue(test_file.isfile())
self.assertFalse(test_file.isdir())
with test_file.open(mode='rb') as obj:
with copy_file.open(mode='wb') as copy_obj:
copy_obj.write(obj.read())
self.assertTrue(copy_file.exists())
self.assertTrue(copy_file.isfile())
self.assertFalse(copy_file.isdir())
test_contents = test_file.open(mode='rb').read()
copy_contents = copy_file.open(mode='rb').read()
self.assertEquals(test_contents,
'this is a test\nthis is another line.\n')
self.assertEquals(test_contents, copy_contents)
def test_open_works(self):
with tempfile.NamedTemporaryFile() as f:
p = stor.Path(f.name).open()
p.close()
with tempfile.NamedTemporaryFile() as f:
p = stor.open(f.name)
p.close()
def test_metadata_pulling(self):
file_in_folder = stor.join(self.test_container, 'somefile.svg')
with stor.open(file_in_folder, 'w') as fp:
fp.write('12345\n')
self.assertEqual(stor.getsize(file_in_folder), 6)
stat_data = stor.Path(file_in_folder).stat()
self.assertIn('Content-Type', stat_data)
self.assertEqual(stat_data['Content-Type'], 'image/svg+xml')
def test_gzip_on_remote(self):
local_gzip = os.path.join(os.path.dirname(__file__),
'file_data/s_3_2126.bcl.gz')
remote_gzip = stor.join(self.test_dir, stor.basename(local_gzip))
stor.copy(local_gzip, remote_gzip)
with stor.open(remote_gzip) as fp:
with gzip.GzipFile(fileobj=fp) as remote_gzip_fp:
with gzip.open(local_gzip) as local_gzip_fp:
assert_same_data(remote_gzip_fp, local_gzip_fp)
def test_empty_buffer_no_writes(self, mock_read_object, mock_write_object):
# NOTE: this tests that our current description (only non-empty buffers are uploaded) is
# enshrined.
fileobj = stor.open(stor.join(self.drive, 'B/C/obj'), 'w')
fileobj.flush()
self.assertFalse(fileobj._buffer)
fileobj.write('')
fileobj.flush()
fileobj.close()
self.assertFalse(mock_read_object.called)
self.assertFalse(mock_write_object.called)
def test_readable_writable_seekable(self):
pth = self.normal_path
read_obj = pth.open(mode='r')
self.assertTrue(read_obj.readable())
self.assertFalse(read_obj.writable())
self.assertTrue(read_obj.seekable())
write_obj = stor.open(stor.join(self.drive, 'B/C/obj'), 'w')
self.assertFalse(write_obj.readable())
self.assertTrue(write_obj.writable())
self.assertTrue(write_obj.seekable())
def test_works_with_gzip(self, mock_read_object):
gzip_path = stor.join(stor.dirname(__file__), 'file_data',
's_3_2126.bcl.gz')
text = stor.open(gzip_path, 'rb').read()
mock_read_object.return_value = text
fileobj = stor.open(stor.join(self.drive, 'A/C/s_3_2126.bcl.gz'), 'rb')
with fileobj:
with gzip.GzipFile(fileobj=fileobj) as fp:
with gzip.open(gzip_path) as gzip_fp:
assert_same_data(fp, gzip_fp)
fileobj = stor.open(stor.join(self.drive, 'A/C/s_3_2126.bcl.gz'), 'rb')
with fileobj:
with gzip.GzipFile(fileobj=fileobj) as fp:
with gzip.open(gzip_path) as gzip_fp:
# after seeking should still be same
fp.seek(3)
gzip_fp.seek(3)
assert_same_data(fp, gzip_fp)
def test_all_segment_container_types_are_deleted(self):
segment_containers = [stor.join('swift://' + self.test_container.tenant,
fmt % self.test_container.name)
for fmt in ('.segments_%s', '%s+segments', '%s_segments')]
all_containers = segment_containers + [self.test_container]
test_files = [stor.join(c, 'test_file_tbdeleted.txt') for c in all_containers]
for t in test_files:
with stor.open(t, 'w') as fp:
fp.write('testtxt\n')
assert all(t.exists() for t in test_files)
stor.rmtree(self.test_container)
for t in test_files:
assert not t.exists(), 'Did not delete %s' % t
def test_gzip_on_remote(self):
self._skip_if_filesystem_python3(self.test_dir)
local_gzip = os.path.join(os.path.dirname(__file__),
'file_data/s_3_2126.bcl.gz')
remote_gzip = stor.join(self.test_dir, stor.basename(local_gzip))
stor.copy(local_gzip, remote_gzip)
file_h = dxpy.DXFile(dxid=remote_gzip.canonical_resource,
project=remote_gzip.canonical_project)
file_h.wait_on_close(20) # wait for file to go to closed state
with stor.open(remote_gzip, mode='rb') as fp:
with gzip.GzipFile(fileobj=fp) as remote_gzip_fp:
with gzip.open(local_gzip) as local_gzip_fp:
assert_same_data(remote_gzip_fp, local_gzip_fp)
def generate_and_save_data_manifest(manifest_dir, data_manifest_contents):
"""Generates a data manifest for a given directory and saves it.
Args:
manifest_dir (str): The directory in which the manifest will be saved
data_manifest_contents (List[str]): The list of all objects that will
be part of the manifest.
"""
import stor
from stor import Path
manifest_file_name = Path(manifest_dir) / DATA_MANIFEST_FILE_NAME
with stor.open(manifest_file_name, 'w') as out_file:
contents = '\n'.join(data_manifest_contents) + '\n'
out_file.write(contents)
def test_w_broken_symlink(self):
swift_dir = (Path(__file__).expand().abspath().parent / 'swift_upload')
with utils.NamedTemporaryDirectory(dir=swift_dir) as tmp_dir:
symlink = tmp_dir / 'broken.symlink'
symlink_source = tmp_dir / 'nonexistent'
# put something in symlink source so that Python doesn't complain
with stor.open(symlink_source, 'w') as fp:
fp.write('blah')
os.symlink(symlink_source, symlink)
uploads = utils.walk_files_and_dirs([swift_dir])
self.assertEquals(
set(uploads),
set([
swift_dir / 'file1',
swift_dir / 'data_dir' / 'file2',
symlink,
symlink_source,
]))
# NOW: destroy it with fire and we have empty directory
os.remove(symlink_source)
uploads = utils.walk_files_and_dirs([swift_dir])
self.assertEquals(
set(uploads),
set([
swift_dir / 'file1',
tmp_dir,
swift_dir / 'data_dir' / 'file2',
]))
# but put a sub directory, now tmp_dir doesn't show up
subdir = tmp_dir / 'subdir'
subdir.makedirs_p()
uploads = utils.walk_files_and_dirs([swift_dir])
self.assertEquals(
set(uploads),
set([
swift_dir / 'file1',
subdir,
swift_dir / 'data_dir' / 'file2',
]))
def test_file_read_write(self):
self._skip_if_filesystem_python3(self.test_dir)
non_with_file = self.test_dir / 'nonwithfile.txt'
test_file = self.test_dir / 'test_file.txt'
copy_file = self.test_dir / 'copy_file.txt'
fp = stor.open(non_with_file, mode='wb')
# File opened in wb mode requires: bytes on py3k, str on py27
fp.write('blah'.encode())
del fp
self.assertTrue(non_with_file.exists())
self.assertTrue(non_with_file.isfile())
self.assertFalse(non_with_file.isdir())
with stor.settings.use(self.DX_WAIT_SETTINGS):
with test_file.open(mode='wb') as obj:
obj.write('this is a test\n'.encode())
obj.write('this is another line.\n'.encode())
self.assertTrue(test_file.exists())
self.assertTrue(test_file.isfile())
self.assertFalse(test_file.isdir())
with stor.settings.use(self.DX_WAIT_SETTINGS):
with test_file.open(mode='rb') as obj:
with copy_file.open(mode='wb') as copy_obj:
copy_obj.write(obj.read())
self.assertTrue(copy_file.exists())
self.assertTrue(copy_file.isfile())
self.assertFalse(copy_file.isdir())
test_contents = test_file.open(mode='rb').read()
copy_contents = copy_file.open(mode='rb').read()
self.assertEquals(test_contents,
'this is a test\nthis is another line.\n'.encode())
self.assertEquals(test_contents, copy_contents)
def test_write_bytes_to_binary(self):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='wb') as fp:
fp.write(BYTE_STRING)
def test_functional_open(self):
with tempfile.NamedTemporaryFile() as f:
with stor.open(f.name, 'w') as f:
f.write('blah')
def test_write_string_to_binary(self): # pragma: no cover
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='wb') as fp:
fp.write(STRING_STRING)
def test_encoding_typeerror_py2(self):
test_file = self.test_dir / 'test_file.txt'
with pytest.raises(TypeError, regex='encoding'):
stor.open(test_file, mode='r', encoding='utf-8')
with pytest.raises(TypeError, regex='encoding'):
stor.Path(test_file).open(mode='r', encoding='utf-8')
def test_functional_open(self):
with tempfile.NamedTemporaryFile() as f:
with stor.open(f.name, 'w', swift_upload_kwargs={}) as f:
f.write('blah')
def test_write_string_to_text(self):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='w') as fp:
fp.write(STRING_STRING)
def test_write_bytes_to_text(self): # pragma: no cover
with pytest.raises(TypeError):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='w') as fp:
fp.write(BYTE_STRING)
def test_write_string_to_binary(self):
with pytest.raises(TypeError):
test_file = self.test_dir / 'test_file.txt'
with stor.open(test_file, mode='wb') as fp:
fp.write(STRING_STRING)