def test_stet_upload_runs_with_binary_from_path_with_correct_settings(
self, mock_execute_external_command):
fake_config_path = self.CreateTempFile()
temporary_path_directory = self.CreateTempDir()
fake_stet_binary_path = self.CreateTempFile(
tmpdir=temporary_path_directory, file_name='stet')
previous_path = os.getenv('PATH')
os.environ['PATH'] += os.path.pathsep + temporary_path_directory
mock_execute_external_command.return_value = ('stdout', 'stderr')
mock_logger = mock.Mock()
source_url = storage_url.StorageUrlFromString('in')
destination_url = storage_url.StorageUrlFromString('gs://bucket/obj')
with util.SetBotoConfigForTest([
('GSUtil', 'stet_binary_path', None),
('GSUtil', 'stet_config_path', fake_config_path),
]):
out_file_url = stet_util.encrypt_upload(source_url,
destination_url,
mock_logger)
self.assertEqual(out_file_url,
storage_url.StorageUrlFromString('in_.stet_tmp'))
mock_execute_external_command.assert_called_once_with([
fake_stet_binary_path,
'encrypt',
'--config-file={}'.format(fake_config_path),
'--blob-id=gs://bucket/obj',
'in',
'in_.stet_tmp',
])
mock_logger.debug.assert_called_once_with('stderr')
os.environ['PATH'] = previous_path
def test_stet_upload_uses_no_config_if_not_provided(
self, mock_execute_external_command):
mock_execute_external_command.return_value = ('stdout', 'stderr')
mock_logger = mock.Mock()
source_url = storage_url.StorageUrlFromString('in')
destination_url = storage_url.StorageUrlFromString('gs://bucket/obj')
with util.SetBotoConfigForTest([
('GSUtil', 'stet_binary_path', 'fake_binary_path'),
('GSUtil', 'stet_config_path', None),
]):
with mock.patch.object(os.path,
'exists',
new=mock.Mock(return_value=True)):
out_file_url = stet_util.encrypt_upload(
source_url, destination_url, mock_logger)
self.assertEqual(out_file_url,
storage_url.StorageUrlFromString('in_.stet_tmp'))
mock_execute_external_command.assert_called_once_with([
'fake_binary_path',
'encrypt',
'--blob-id=gs://bucket/obj',
'in',
'in_.stet_tmp',
])
mock_logger.debug.assert_called_once_with('stderr')
def test_stet_upload_uses_binary_and_config_from_boto(
self, mock_execute_external_command):
fake_config_path = self.CreateTempFile()
mock_execute_external_command.return_value = ('stdout', 'stderr')
mock_logger = mock.Mock()
source_url = storage_url.StorageUrlFromString('in')
destination_url = storage_url.StorageUrlFromString('gs://bucket/obj')
with util.SetBotoConfigForTest([
('GSUtil', 'stet_binary_path', 'fake_binary_path'),
('GSUtil', 'stet_config_path', fake_config_path),
]):
out_file_url = stet_util.encrypt_upload(source_url,
destination_url,
mock_logger)
self.assertEqual(out_file_url,
storage_url.StorageUrlFromString('in_.stet_tmp'))
mock_execute_external_command.assert_called_once_with([
'fake_binary_path',
'encrypt',
'--config-file={}'.format(fake_config_path),
'--blob-id=gs://bucket/obj',
'in',
'in_.stet_tmp',
])
mock_logger.debug.assert_called_once_with('stderr')
def test_stet_util_errors_if_no_binary(self):
fake_config_path = self.CreateTempFile()
source_url = storage_url.StorageUrlFromString('in')
destination_url = storage_url.StorageUrlFromString('gs://bucket/obj')
with util.SetBotoConfigForTest([
('GSUtil', 'stet_binary_path', None),
('GSUtil', 'stet_config_path', fake_config_path),
]):
with self.assertRaises(KeyError):
stet_util.encrypt_upload(source_url, destination_url, None)
def test_stet_util_expands_home_directory_symbol(self, mock_expanduser):
fake_config_path = self.CreateTempFile()
source_url = storage_url.StorageUrlFromString('in')
destination_url = storage_url.StorageUrlFromString('gs://bucket/obj')
with util.SetBotoConfigForTest([
('GSUtil', 'stet_binary_path', 'fake_binary_path'),
('GSUtil', 'stet_config_path', fake_config_path),
]):
stet_util.encrypt_upload(source_url, destination_url, mock.Mock())
mock_expanduser.assert_has_calls(
[mock.call('fake_binary_path'),
mock.call(fake_config_path)])
def test_decrypts_download_if_stet_is_enabled(self):
object_uri = self.CreateObject(contents='abc')
test_file = self.CreateTempFile()
stderr = self.RunGsUtil([
'-o', 'GSUtil:stet_binary_path={}'.format(self.stet_binary_path),
'-o', 'GSUtil:stet_config_path={}'.format(
self.stet_config_path), 'cp', '--stet',
suri(object_uri), test_file
],
return_stderr=True)
# Progress indicator should show transformed file size (variable based on
# random string generation above). 4.0 B is the pre-transform size.
self.assertNotIn('/4.0 B]', stderr)
with open(test_file) as file_reader:
downloaded_text = file_reader.read()
self.assertIn('subcommand: decrypt', downloaded_text)
self.assertIn(
'config file: --config-file={}'.format(self.stet_config_path),
downloaded_text)
self.assertIn('blob id: --blob-id={}'.format(suri(object_uri)),
downloaded_text)
self.assertIn('in file: {}'.format(test_file), downloaded_text)
self.assertIn('out file: {}_.stet_tmp'.format(test_file),
downloaded_text)
self.assertFalse(
os.path.exists(
temporary_file_util.GetStetTempFileName(
storage_url.StorageUrlFromString(test_file))))
def test_storage_url_from_string(self):
url = storage_url.StorageUrlFromString('abc')
self.assertTrue(url.IsFileUrl())
self.assertEquals('abc', url.object_name)
url = storage_url.StorageUrlFromString('file://abc/123')
self.assertTrue(url.IsFileUrl())
self.assertEquals('abc%s123' % os.sep, url.object_name)
url = storage_url.StorageUrlFromString('gs://abc/123/456')
self.assertTrue(url.IsCloudUrl())
self.assertEquals('abc', url.bucket_name)
self.assertEquals('123/456', url.object_name)
url = storage_url.StorageUrlFromString('s3://abc/123/456')
self.assertTrue(url.IsCloudUrl())
self.assertEquals('abc', url.bucket_name)
self.assertEquals('123/456', url.object_name)
def test_stet_download_runs_binary_and_replaces_temp_file(
self, mock_execute_external_command, mock_move):
fake_config_path = self.CreateTempFile()
mock_execute_external_command.return_value = ('stdout', 'stderr')
mock_logger = mock.Mock()
source_url = storage_url.StorageUrlFromString('gs://bucket/obj')
destination_url = storage_url.StorageUrlFromString('out')
temporary_file_name = 'out_.gstmp'
with util.SetBotoConfigForTest([
('GSUtil', 'stet_binary_path', 'fake_binary_path'),
('GSUtil', 'stet_config_path', fake_config_path),
]):
stet_util.decrypt_download(source_url, destination_url,
temporary_file_name, mock_logger)
mock_execute_external_command.assert_called_once_with([
'fake_binary_path', 'decrypt',
'--config-file={}'.format(fake_config_path),
'--blob-id=gs://bucket/obj', 'out_.gstmp', 'out_.stet_tmp'
])
mock_logger.debug.assert_called_once_with('stderr')
mock_move.assert_called_once_with('out_.stet_tmp', 'out_.gstmp')
def test_does_not_warn_if_supported_double_wildcard(self, mock_stderr):
storage_url.StorageUrlFromString('**')
storage_url.StorageUrlFromString('gs://bucket/**')
storage_url.StorageUrlFromString('**' + os.sep)
storage_url.StorageUrlFromString('gs://bucket/**/')
storage_url.StorageUrlFromString(os.sep + '**')
storage_url.StorageUrlFromString('gs://bucket//**')
storage_url.StorageUrlFromString(os.sep + '**' + os.sep)
mock_stderr.assert_not_called()
def encrypt_upload(source_url, destination_url, logger):
"""Encrypts a file with STET binary before upload.
Args:
source_url (StorageUrl): Copy source.
destination_url (StorageUrl): Copy destination.
logger (logging.Logger): For logging STET binary output.
Returns:
stet_temporary_file_url (StorageUrl): Path to STET-encrypted file.
"""
in_file = source_url.object_name
out_file = temporary_file_util.GetStetTempFileName(source_url)
blob_id = destination_url.url_string
_stet_transform(StetSubcommandName.ENCRYPT, blob_id, in_file, out_file,
logger)
return storage_url.StorageUrlFromString(out_file)
def test_raises_error_for_too_many_slashes_after_scheme(self):
with self.assertRaises(InvalidUrlError):
storage_url.StorageUrlFromString('gs:///')
with self.assertRaises(InvalidUrlError):
storage_url.StorageUrlFromString('gs://////')
def testGetsStetTemporaryFileName(self):
self.assertEqual(
temporary_file_util.GetStetTempFileName(
storage_url.StorageUrlFromString('file.txt')),
'file.txt_.stet_tmp')
def testGetsTemporaryZipFileName(self):
self.assertEqual(
temporary_file_util.GetTempZipFileName(
storage_url.StorageUrlFromString('file.txt')),
'file.txt_.gztmp')
def test_warns_if_unsupported_double_wildcard(self, mock_stderr):
storage_url.StorageUrlFromString('abc**')
storage_url.StorageUrlFromString('gs://bucket/object**')
storage_url.StorageUrlFromString('**abc')
storage_url.StorageUrlFromString('gs://bucket/**object')
storage_url.StorageUrlFromString('abc**' + os.sep)
storage_url.StorageUrlFromString('gs://bucket/object**/')
storage_url.StorageUrlFromString(os.sep + '**abc')
storage_url.StorageUrlFromString('gs://bucket//**object')
storage_url.StorageUrlFromString(os.sep + '**' + os.sep + 'abc**')
storage_url.StorageUrlFromString('gs://bucket/**/abc**')
storage_url.StorageUrlFromString('abc**' + os.sep + 'abc')
storage_url.StorageUrlFromString('gs://bucket/abc**/abc')
storage_url.StorageUrlFromString(os.sep + 'abc**' + os.sep + '**')
storage_url.StorageUrlFromString('gs://bucket/abc**/**')
storage_url.StorageUrlFromString('gs://b**')
storage_url.StorageUrlFromString('gs://**b')
mock_stderr.assert_has_calls(
[mock.call(_UNSUPPORTED_DOUBLE_WILDCARD_WARNING_TEXT)] * 14)