def test_broken_symlink_file_name_truncation(self):
with stor.NamedTemporaryDirectory(dir=self.swift_dir) as tmp_dir:
# finally make enough symlinks to trigger log trimming (currently
# untested but we want to make sure we don't error) + hits some
# issues if path to the test file is so long that even 1 file is >
# 50 characters, so we skip coverage check because it's not
# worth the hassle to get full branch coverage for big edge case
super_long_name = tmp_dir / 'abcdefghijklmnopqrstuvwxyzrsjexcasdfawefawefawefawefaef'
for x in range(5):
os.symlink(super_long_name, super_long_name + str(x))
# have no errors! yay!
self.assert_(utils.walk_files_and_dirs([self.swift_dir]))
for x in range(5, 20):
os.symlink(super_long_name, super_long_name + str(x))
# have no errors! yay!
self.assert_(utils.walk_files_and_dirs([self.swift_dir]))
def list(self):
"""
List all files and directories under a path.
Returns:
List[str]: A list of all files and directories.
"""
return list(utils.walk_files_and_dirs([self]).keys())
def upload(self, to_upload, **kwargs):
"""Upload a list of files and directories to a directory.
This is not a batch level operation.
If some file errors, the files uploaded before will remain present.
Args:
to_upload (List[Union[str, OBSUploadObject]]): A list of posix file names,
directory names, or OBSUploadObject objects to upload.
Raises:
ValueError: When source path is not a directory
TargetExistsError: When destination directory already exists
"""
dx_upload_objects = [
name for name in to_upload if isinstance(name, OBSUploadObject)
]
all_files_to_upload = utils.walk_files_and_dirs([
name for name in to_upload
if not isinstance(name, OBSUploadObject)
])
dx_upload_objects.extend([
OBSUploadObject(
f,
object_name=('/' +
self.resource if self.resource else Path('')) /
utils.file_name_to_object_name(f)) for f in all_files_to_upload
])
for upload_obj in dx_upload_objects:
upload_obj.object_name = Path(upload_obj.object_name)
upload_obj.source = Path(upload_obj.source)
dest_file = Path('{drive}{project}:{path}'.format(
drive=self.drive,
project=self.canonical_project,
path=upload_obj.object_name))
if upload_obj.source.isfile():
dest_is_file = dest_file.isfile()
if dest_is_file: # only occurs if upload is called directly with existing objects
logger.warning(
'Destination path ({}) already exists, will not cause '
'duplicate file objects on the platform. Skipping...'.
format(dest_file))
else:
with _wrap_dx_calls():
dxpy.upload_local_file(
filename=upload_obj.source,
project=self.canonical_project,
folder='/' + (dest_file.parent.resource or ''),
parents=True,
name=dest_file.name)
elif upload_obj.source.isdir():
dest_file.makedirs_p()
else:
raise stor_exceptions.NotFoundError(
'Source path ({}) does not exist. Please provide a valid source'
.format(upload_obj.source))
def test_w_dir(self):
# Create an empty directory for this test in ./swift_upload. This
# is because git doesnt allow a truly empty directory to be checked
# in
with utils.NamedTemporaryDirectory(dir=self.swift_dir) as tmp_dir:
uploads = utils.walk_files_and_dirs([self.swift_dir])
self.assertEquals(
set(uploads),
set([
self.swift_dir / 'file1',
tmp_dir,
self.swift_dir / 'data_dir' / 'file2',
]))
def test_w_broken_symlink(self):
swift_dir = (Path(__file__).expand().abspath().parent / 'swift_upload')
with utils.NamedTemporaryDirectory(dir=swift_dir) as tmp_dir:
symlink = tmp_dir / 'broken.symlink'
symlink_source = tmp_dir / 'nonexistent'
# put something in symlink source so that Python doesn't complain
with stor.open(symlink_source, 'w') as fp:
fp.write('blah')
os.symlink(symlink_source, symlink)
uploads = utils.walk_files_and_dirs([swift_dir])
self.assertEquals(
set(uploads),
set([
swift_dir / 'file1',
swift_dir / 'data_dir' / 'file2',
symlink,
symlink_source,
]))
# NOW: destroy it with fire and we have empty directory
os.remove(symlink_source)
uploads = utils.walk_files_and_dirs([swift_dir])
self.assertEquals(
set(uploads),
set([
swift_dir / 'file1',
tmp_dir,
swift_dir / 'data_dir' / 'file2',
]))
# but put a sub directory, now tmp_dir doesn't show up
subdir = tmp_dir / 'subdir'
subdir.makedirs_p()
uploads = utils.walk_files_and_dirs([swift_dir])
self.assertEquals(
set(uploads),
set([
swift_dir / 'file1',
subdir,
swift_dir / 'data_dir' / 'file2',
]))
def upload(self,
source,
condition=None,
use_manifest=False,
headers=None,
**kwargs):
"""Uploads a list of files and directories to s3.
Note that the S3Path is treated as a directory.
Note that for user-provided OBSUploadObjects, an empty directory's destination
must have a trailing slash.
Args:
source (List[str|OBSUploadObject]): A list of source files, directories, and
OBSUploadObjects to upload to S3.
condition (function(results) -> bool): The method will only return
when the results of upload matches the condition.
use_manifest (bool): Generate a data manifest and validate the upload results
are in the manifest.
headers (dict): A dictionary of object headers to apply to the object.
Headers will not be applied to OBSUploadObjects and any headers
specified by an OBSUploadObject will override these headers.
Headers should be specified as key-value pairs,
e.g. {'ContentLanguage': 'en'}
Returns:
List[S3Path]: A list of the uploaded files as S3Paths.
Notes:
- This method uploads to paths relative to the current
directory.
"""
if use_manifest and not (len(source) == 1
and os.path.isdir(source[0])):
raise ValueError(
'can only upload one directory with use_manifest=True')
utils.validate_condition(condition)
files_to_convert = utils.walk_files_and_dirs(
[name for name in source if not isinstance(name, OBSUploadObject)])
files_to_upload = [
obj for obj in source if isinstance(obj, OBSUploadObject)
]
manifest_file_name = (Path(source[0]) / utils.DATA_MANIFEST_FILE_NAME
if use_manifest else None)
resource_base = self.resource or Path('')
files_to_upload.extend([
OBSUploadObject(
name,
resource_base /
(utils.with_trailing_slash(
utils.file_name_to_object_name(name)) if
Path(name).isdir() else utils.file_name_to_object_name(name)),
options={'headers': headers} if headers else None)
for name in files_to_convert if name != manifest_file_name
])
if use_manifest:
# Generate the data manifest and save it remotely
object_names = [o.object_name for o in files_to_upload]
utils.generate_and_save_data_manifest(source[0], object_names)
manifest_obj_name = resource_base / utils.file_name_to_object_name(
manifest_file_name)
manifest_obj = OBSUploadObject(
str(manifest_file_name),
manifest_obj_name,
options={'headers': headers} if headers else None)
self._upload_object(manifest_obj)
# Make a condition for validating the upload
manifest_cond = partial(utils.validate_manifest_list, object_names)
condition = (utils.join_conditions(condition, manifest_cond)
if condition else manifest_cond)
options = settings.get()['s3:upload']
segment_size = utils.str_to_bytes(options.get('segment_size'))
transfer_config = {
'multipart_threshold': segment_size,
'max_concurrency': options.get('segment_threads'),
'multipart_chunksize': segment_size
}
upload_w_config = partial(self._upload_object, config=transfer_config)
uploaded = {'completed': [], 'failed': []}
with S3UploadLogger(len(files_to_upload)) as ul:
pool = ThreadPool(options['object_threads'])
try:
result_iter = pool.imap_unordered(upload_w_config,
files_to_upload)
while True:
try:
result = result_iter.next(0xFFFF)
if result['success']:
ul.add_result(result)
uploaded['completed'].append(result)
else:
uploaded['failed'].append(result)
except StopIteration:
break
pool.close()
except BaseException:
pool.terminate()
raise
finally:
pool.join()
if uploaded['failed']:
raise exceptions.FailedUploadError(
'an error occurred while uploading', uploaded)
utils.check_condition(condition,
[r['dest'] for r in uploaded['completed']])
return uploaded
def test_w_file(self):
name = self.swift_dir / 'file1'
uploads = utils.walk_files_and_dirs([name])
self.assertEquals(set(uploads), set([name]))
def test_w_missing_file(self):
name = self.swift_dir / 'invalid'
with self.assertRaises(ValueError):
utils.walk_files_and_dirs([name])