def stream_tarred_gzipped_directory(self, uuid, path): """ Returns a file-like object containing a tarred and gzipped archive of the given directory. """ directory_path = self._get_and_check_target_path(uuid, path) return file_util.tar_gzip_directory(directory_path)
def test_tar_empty(self): dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(dir)) output_dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(output_dir)) un_tar_gzip_directory(tar_gzip_directory(dir), output_dir) self.assertEquals(os.listdir(output_dir), [])
def test_tar_empty(self): dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(dir)) temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(temp_dir)) output_dir = os.path.join(temp_dir, 'output') un_tar_directory(tar_gzip_directory(dir), output_dir, 'gz') self.assertEquals(os.listdir(output_dir), [])
def test_single_fileobj_tar_gz_no_simplify_archives(self): source = os.path.join(self.temp_dir, 'source_dir') os.mkdir(source) self.write_string_to_file('testing', os.path.join(source, 'filename')) self.do_upload([('source.tar.gz', tar_gzip_directory(source))], simplify_archives=False) self.assertEqual(['filename'], os.listdir(self.bundle_location)) self.check_file_contains_string( os.path.join(self.bundle_location, 'filename'), 'testing')
def test_single_local_tar_gz_path_simplify_archives(self): source_dir = os.path.join(self.temp_dir, 'source_dir') os.mkdir(source_dir) self.write_string_to_file('testing', os.path.join(source_dir, 'filename')) source = os.path.join(self.temp_dir, 'source.tar.gz') with open(source, 'wb') as f: f.write(tar_gzip_directory(source_dir).read()) self.do_upload([source], simplify_archives=True) self.assertTrue(os.path.exists(source)) self.check_file_contains_string(self.bundle_location, 'testing')
def upload_bundle(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet): """ See local_bundle_client.py for documentation on the usage. Strategy: 1) We copy the |sources| to a temporary directory on the server (streaming either a tar or tar.gz depending on whether compression is needed). 2) We politely ask the server to finish_upload_bundle (performs a LocalBundleClient.upload_bundle from the temporary directory). """ # URLs can be directly passed to the local client. if all(path_util.path_is_url(source) for source in sources): return self.upload_bundle_url(sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet) remote_file_uuids = [] try: # 1) Copy sources up to the server (temporary remote zip file) for source in sources: if zip_util.path_is_archive(source): source_handle = open(source) temp_file_name = os.path.basename(source) elif os.path.isdir(source): source_handle = tar_gzip_directory(source, follow_symlinks, exclude_patterns) temp_file_name = os.path.basename(source) + '.tar.gz' unpack = True # We packed it, so we have to unpack it else: resolved_source = source if follow_symlinks: resolved_source = os.path.realpath(source) if not os.path.exists(resolved_source): raise UsageError('Broken symlink') elif os.path.islink(source): raise UsageError('Not following symlinks.') source_handle = gzip_file(resolved_source) temp_file_name = os.path.basename(source) + '.gz' unpack = True # We packed it, so we have to unpack it remote_file_uuid = self.open_temp_file(temp_file_name) remote_file_uuids.append(remote_file_uuid) with closing(RPCFileHandle(remote_file_uuid, self.proxy)) as dest_handle: status = 'Uploading %s%s to %s' % (source, ' ('+info['uuid']+')' if 'uuid' in info else '', self.address) file_util.copy(source_handle, dest_handle, autoflush=False, print_status=status) # 2) Install upload (this call will be in charge of deleting the temporary file). return self.finish_upload_bundle(remote_file_uuids, unpack, info, worksheet_uuid, add_to_worksheet) except: for remote_file_uuid in remote_file_uuids: self.finalize_file(remote_file_uuid) raise
def test_tar_has_files(self): dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'files') temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(temp_dir)) output_dir = os.path.join(temp_dir, 'output') un_tar_directory( tar_gzip_directory(dir, False, ['f2'], ['f1', 'b.txt']), output_dir, 'gz') output_dir_entries = os.listdir(output_dir) self.assertIn('dir1', output_dir_entries) self.assertIn('a.txt', output_dir_entries) self.assertNotIn('b.txt', output_dir_entries) self.assertTrue(os.path.exists(os.path.join(output_dir, 'dir1', 'f1'))) self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir1', 'f2'))) self.assertTrue(os.path.islink(os.path.join(output_dir, 'a-symlink.txt')))
def stream_tarred_gzipped_directory(self, uuid, path): """ Returns a file-like object containing a tarred and gzipped archive of the given directory. """ if self._is_available_locally(uuid): directory_path = self._get_target_path(uuid, path) return file_util.tar_gzip_directory(directory_path) else: worker = self._worker_model.get_bundle_worker(uuid) response_socket_id = self._worker_model.allocate_socket(worker['user_id'], worker['worker_id']) try: read_args = { 'type': 'stream_directory', } self._send_read_message(worker, response_socket_id, uuid, path, read_args) fileobj = self._get_read_response_stream(response_socket_id) return Deallocating(fileobj, self._worker_model, response_socket_id) except: self._worker_model.deallocate_socket(response_socket_id) raise
def stream_tarred_gzipped_directory(self, uuid, path): """ Returns a file-like object containing a tarred and gzipped archive of the given directory. """ if self._is_available_locally(uuid): directory_path = self._get_target_path(uuid, path) return file_util.tar_gzip_directory(directory_path) else: worker = self._worker_model.get_bundle_worker(uuid) response_socket_id = self._worker_model.allocate_socket( worker['user_id'], worker['worker_id']) try: read_args = { 'type': 'stream_directory', } self._send_read_message(worker, response_socket_id, uuid, path, read_args) fileobj = self._get_read_response_stream(response_socket_id) return Deallocating(fileobj, self._worker_model, response_socket_id) except: self._worker_model.deallocate_socket(response_socket_id) raise
def upload_bundle(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet): """ See local_bundle_client.py for documentation on the usage. Strategy: 1) We copy the |sources| to a temporary directory on the server (streaming either a tar or tar.gz depending on whether compression is needed). 2) We politely ask the server to finish_upload_bundle (performs a LocalBundleClient.upload_bundle from the temporary directory). """ # URLs can be directly passed to the local client. if all(path_util.path_is_url(source) for source in sources): return self.upload_bundle_url(sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet) remote_file_uuids = [] try: # 1) Copy sources up to the server (temporary remote zip file) for source in sources: if zip_util.path_is_archive(source): source_handle = open(source) temp_file_name = os.path.basename(source) elif os.path.isdir(source): source_handle = tar_gzip_directory(source, follow_symlinks, exclude_patterns) temp_file_name = os.path.basename(source) + '.tar.gz' unpack = True # We packed it, so we have to unpack it else: resolved_source = source if follow_symlinks: resolved_source = os.path.realpath(source) if not os.path.exists(resolved_source): raise UsageError('Broken symlink') elif os.path.islink(source): raise UsageError('Not following symlinks.') source_handle = gzip_file(resolved_source) temp_file_name = os.path.basename(source) + '.gz' unpack = True # We packed it, so we have to unpack it remote_file_uuid = self.open_temp_file(temp_file_name) remote_file_uuids.append(remote_file_uuid) with closing(RPCFileHandle(remote_file_uuid, self.proxy)) as dest_handle: status = 'Uploading %s%s to %s' % ( source, ' (' + info['uuid'] + ')' if 'uuid' in info else '', self.address) file_util.copy(source_handle, dest_handle, autoflush=False, print_status=status) # 2) Install upload (this call will be in charge of deleting the temporary file). return self.finish_upload_bundle(remote_file_uuids, unpack, info, worksheet_uuid, add_to_worksheet) except: for remote_file_uuid in remote_file_uuids: self.finalize_file(remote_file_uuid) raise