def stream_tarred_gzipped_directory(self, uuid, path):
"""
Returns a file-like object containing a tarred and gzipped archive
of the given directory.
"""
directory_path = self._get_and_check_target_path(uuid, path)
return file_util.tar_gzip_directory(directory_path)
def stream_tarred_gzipped_directory(self, uuid, path):
"""
Returns a file-like object containing a tarred and gzipped archive
of the given directory.
"""
directory_path = self._get_and_check_target_path(uuid, path)
return file_util.tar_gzip_directory(directory_path)
def test_tar_empty(self):
dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(dir))
output_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(output_dir))
un_tar_gzip_directory(tar_gzip_directory(dir), output_dir)
self.assertEquals(os.listdir(output_dir), [])
def test_tar_empty(self):
dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(dir))
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(temp_dir))
output_dir = os.path.join(temp_dir, 'output')
un_tar_directory(tar_gzip_directory(dir), output_dir, 'gz')
self.assertEquals(os.listdir(output_dir), [])
def test_single_fileobj_tar_gz_no_simplify_archives(self):
source = os.path.join(self.temp_dir, 'source_dir')
os.mkdir(source)
self.write_string_to_file('testing', os.path.join(source, 'filename'))
self.do_upload([('source.tar.gz', tar_gzip_directory(source))],
simplify_archives=False)
self.assertEqual(['filename'], os.listdir(self.bundle_location))
self.check_file_contains_string(
os.path.join(self.bundle_location, 'filename'), 'testing')
def test_single_local_tar_gz_path_simplify_archives(self):
source_dir = os.path.join(self.temp_dir, 'source_dir')
os.mkdir(source_dir)
self.write_string_to_file('testing',
os.path.join(source_dir, 'filename'))
source = os.path.join(self.temp_dir, 'source.tar.gz')
with open(source, 'wb') as f:
f.write(tar_gzip_directory(source_dir).read())
self.do_upload([source], simplify_archives=True)
self.assertTrue(os.path.exists(source))
self.check_file_contains_string(self.bundle_location, 'testing')
def upload_bundle(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet):
"""
See local_bundle_client.py for documentation on the usage.
Strategy:
1) We copy the |sources| to a temporary directory on the server
(streaming either a tar or tar.gz depending on whether compression is
needed).
2) We politely ask the server to finish_upload_bundle (performs a
LocalBundleClient.upload_bundle from the temporary directory).
"""
# URLs can be directly passed to the local client.
if all(path_util.path_is_url(source) for source in sources):
return self.upload_bundle_url(sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet)
remote_file_uuids = []
try:
# 1) Copy sources up to the server (temporary remote zip file)
for source in sources:
if zip_util.path_is_archive(source):
source_handle = open(source)
temp_file_name = os.path.basename(source)
elif os.path.isdir(source):
source_handle = tar_gzip_directory(source, follow_symlinks, exclude_patterns)
temp_file_name = os.path.basename(source) + '.tar.gz'
unpack = True # We packed it, so we have to unpack it
else:
resolved_source = source
if follow_symlinks:
resolved_source = os.path.realpath(source)
if not os.path.exists(resolved_source):
raise UsageError('Broken symlink')
elif os.path.islink(source):
raise UsageError('Not following symlinks.')
source_handle = gzip_file(resolved_source)
temp_file_name = os.path.basename(source) + '.gz'
unpack = True # We packed it, so we have to unpack it
remote_file_uuid = self.open_temp_file(temp_file_name)
remote_file_uuids.append(remote_file_uuid)
with closing(RPCFileHandle(remote_file_uuid, self.proxy)) as dest_handle:
status = 'Uploading %s%s to %s' % (source, ' ('+info['uuid']+')' if 'uuid' in info else '', self.address)
file_util.copy(source_handle, dest_handle, autoflush=False, print_status=status)
# 2) Install upload (this call will be in charge of deleting the temporary file).
return self.finish_upload_bundle(remote_file_uuids, unpack, info, worksheet_uuid, add_to_worksheet)
except:
for remote_file_uuid in remote_file_uuids:
self.finalize_file(remote_file_uuid)
raise
def test_tar_has_files(self):
dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'files')
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: remove_path(temp_dir))
output_dir = os.path.join(temp_dir, 'output')
un_tar_directory(
tar_gzip_directory(dir, False, ['f2'], ['f1', 'b.txt']),
output_dir, 'gz')
output_dir_entries = os.listdir(output_dir)
self.assertIn('dir1', output_dir_entries)
self.assertIn('a.txt', output_dir_entries)
self.assertNotIn('b.txt', output_dir_entries)
self.assertTrue(os.path.exists(os.path.join(output_dir, 'dir1', 'f1')))
self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir1', 'f2')))
self.assertTrue(os.path.islink(os.path.join(output_dir, 'a-symlink.txt')))
def stream_tarred_gzipped_directory(self, uuid, path):
"""
Returns a file-like object containing a tarred and gzipped archive
of the given directory.
"""
if self._is_available_locally(uuid):
directory_path = self._get_target_path(uuid, path)
return file_util.tar_gzip_directory(directory_path)
else:
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'stream_directory',
}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
fileobj = self._get_read_response_stream(response_socket_id)
return Deallocating(fileobj, self._worker_model, response_socket_id)
except:
self._worker_model.deallocate_socket(response_socket_id)
raise
def stream_tarred_gzipped_directory(self, uuid, path):
"""
Returns a file-like object containing a tarred and gzipped archive
of the given directory.
"""
if self._is_available_locally(uuid):
directory_path = self._get_target_path(uuid, path)
return file_util.tar_gzip_directory(directory_path)
else:
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(
worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'stream_directory',
}
self._send_read_message(worker, response_socket_id, uuid, path,
read_args)
fileobj = self._get_read_response_stream(response_socket_id)
return Deallocating(fileobj, self._worker_model,
response_socket_id)
except:
self._worker_model.deallocate_socket(response_socket_id)
raise
def upload_bundle(self, sources, follow_symlinks, exclude_patterns, git,
unpack, remove_sources, info, worksheet_uuid,
add_to_worksheet):
"""
See local_bundle_client.py for documentation on the usage.
Strategy:
1) We copy the |sources| to a temporary directory on the server
(streaming either a tar or tar.gz depending on whether compression is
needed).
2) We politely ask the server to finish_upload_bundle (performs a
LocalBundleClient.upload_bundle from the temporary directory).
"""
# URLs can be directly passed to the local client.
if all(path_util.path_is_url(source) for source in sources):
return self.upload_bundle_url(sources, follow_symlinks,
exclude_patterns, git, unpack,
remove_sources, info, worksheet_uuid,
add_to_worksheet)
remote_file_uuids = []
try:
# 1) Copy sources up to the server (temporary remote zip file)
for source in sources:
if zip_util.path_is_archive(source):
source_handle = open(source)
temp_file_name = os.path.basename(source)
elif os.path.isdir(source):
source_handle = tar_gzip_directory(source, follow_symlinks,
exclude_patterns)
temp_file_name = os.path.basename(source) + '.tar.gz'
unpack = True # We packed it, so we have to unpack it
else:
resolved_source = source
if follow_symlinks:
resolved_source = os.path.realpath(source)
if not os.path.exists(resolved_source):
raise UsageError('Broken symlink')
elif os.path.islink(source):
raise UsageError('Not following symlinks.')
source_handle = gzip_file(resolved_source)
temp_file_name = os.path.basename(source) + '.gz'
unpack = True # We packed it, so we have to unpack it
remote_file_uuid = self.open_temp_file(temp_file_name)
remote_file_uuids.append(remote_file_uuid)
with closing(RPCFileHandle(remote_file_uuid,
self.proxy)) as dest_handle:
status = 'Uploading %s%s to %s' % (
source, ' (' + info['uuid'] +
')' if 'uuid' in info else '', self.address)
file_util.copy(source_handle,
dest_handle,
autoflush=False,
print_status=status)
# 2) Install upload (this call will be in charge of deleting the temporary file).
return self.finish_upload_bundle(remote_file_uuids, unpack, info,
worksheet_uuid, add_to_worksheet)
except:
for remote_file_uuid in remote_file_uuids:
self.finalize_file(remote_file_uuid)
raise
