def test_translated_boto_config_gets_added(self):
"""Should add translated env vars as well flags."""
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'True',
'hidden_shim_mode': 'no_fallback',
'content_language': 'foo',
'default_project_id': 'fake_project',
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
self.assertTrue(
self._fake_command.translate_to_gcloud_storage_if_requested())
# Verify translation.
expected_gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud')
self.assertEqual(
self._fake_command._translated_gcloud_storage_command, [
expected_gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x',
'arg1', 'arg2', '--content-language=foo'
])
self.assertEqual(self._fake_command._translated_env_variables,
{'CLOUDSDK_CORE_PROJECT': 'fake_project'})
def test_calls_subprocess_with_translated_command_and_env_vars(
self, mock_run, mock_environ_copy):
command_instance = FakeCommandWithGcloudStorageMap(
command_runner=mock.ANY,
args=['-z', 'opt1', '-r', 'arg1', 'arg2'],
headers=mock.ANY,
debug=mock.ANY,
trace_token=mock.ANY,
parallel_operations=mock.ANY,
bucket_storage_uri_class=mock.ANY,
gsutil_api_class_map_factory=mock.MagicMock())
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
('GSUtil', 'hidden_shim_mode',
'no_fallback')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
command_instance._translated_env_variables = {
'new_key': 'new_value',
}
command_instance._translated_gcloud_storage_command = ['gcloud', 'foo']
actual_return_code = command_instance.run_gcloud_storage()
mock_run.assert_called_once_with(['gcloud', 'foo'],
env={
'old_key': 'old_value',
'new_key': 'new_value'
})
mock_environ_copy.assert_called_once_with()
self.assertEqual(actual_return_code, mock_run.return_value.returncode)
def test_translated_headers_get_added_to_final_command(self):
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
fake_command = FakeCommandWithGcloudStorageMap(
command_runner=mock.ANY,
args=['arg1', 'arg2'],
headers={'Content-Type': 'fake_val'},
debug=1,
trace_token=None,
parallel_operations=mock.ANY,
bucket_storage_uri_class=mock.ANY,
gsutil_api_class_map_factory=mock.MagicMock())
self.assertTrue(fake_command.translate_to_gcloud_storage_if_requested())
self.assertEqual(fake_command._translated_gcloud_storage_command, [
os.path.join('fake_dir', 'bin', 'gcloud'), 'objects', 'fake',
'arg1', 'arg2', '--content-type=fake_val'
])
def test_parallel_operations_true_does_not_add_process_count_env_vars(self):
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
('GSUtil', 'hidden_shim_mode',
'no_fallback')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
self._fake_command.parallel_operations = True
self._fake_command.translate_to_gcloud_storage_if_requested()
self.assertNotIn('CLOUDSDK_STORAGE_PROCESS_COUNT',
self._fake_command._translated_env_variables)
self.assertNotIn('CLOUDSDK_STORAGE_THREAD_COUNT',
self._fake_command._translated_env_variables)
def test_raises_error_if_gcloud_storage_map_missing(self):
self._fake_command.gcloud_storage_map = None
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
('GSUtil', 'hidden_shim_mode',
'no_fallback')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
with self.assertRaisesRegex(
exception.CommandException,
'CommandException: Command "fake_shim" cannot be translated to'
' gcloud storage because the translation mapping is missing.'):
self._fake_command.translate_to_gcloud_storage_if_requested()
def test_runs_gcloud_storage_if_use_gcloud_storage_true(self):
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
('GSUtil', 'hidden_shim_mode',
'no_fallback')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': None,
}):
stderr = self.RunGsUtil(['-D', 'ls'],
return_stderr=True,
expected_status=1)
# This is a proxy to ensure that the test attempted to call
# gcloud binary.
self.assertIn('gcloud binary path cannot be found', stderr)
def test_raises_error_if_pass_credentials_to_gsutil_is_missing(self):
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
('GSUtil', 'hidden_shim_mode',
'no_fallback')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_ROOT_DIR': 'fake_dir',
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': None
}):
with self.assertRaisesRegex(
exception.CommandException,
'CommandException: Requested to use "gcloud storage" but gsutil'
' is not using the same credentials as'
' gcloud. You can make gsutil use the same credentials'
' by running:\n'
'fake_dir.bin.gcloud config set pass_credentials_to_gsutil True'):
self._fake_command.translate_to_gcloud_storage_if_requested()
def test_raises_error_if_cloudsdk_root_dir_is_none(self):
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'),
('GSUtil', 'hidden_shim_mode',
'no_fallback')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_ROOT_DIR': None,
}):
with self.assertRaisesRegex(
exception.CommandException,
'CommandException: Requested to use "gcloud storage" but the '
'gcloud binary path cannot be found. This might happen if you'
' attempt to use gsutil that was not installed via Cloud SDK.'
' You can manually set the `CLOUDSDK_ROOT_DIR` environment variable'
' to point to the google-cloud-sdk installation directory to'
' resolve the issue. Alternatively, you can set'
' `use_gcloud_storage=False` to disable running the command'
' using gcloud storage.'):
self._fake_command.translate_to_gcloud_storage_if_requested()
def test_quiet_mode_translation_adds_no_user_output_enabled_flag(self):
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
self._fake_command.quiet_mode = True
self._fake_command.translate_to_gcloud_storage_if_requested()
self.assertEqual(self._fake_command._translated_gcloud_storage_command,
[
os.path.join('fake_dir', 'bin', 'gcloud'),
'objects', 'fake', '--zip', 'opt1', '-x', 'arg1',
'arg2', '--no-user-output-enabled'
])
def test_impersonate_service_account_translation(self):
"""Should add the --impersonate-service-account flag."""
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
self._fake_command.translate_to_gcloud_storage_if_requested()
self.assertEqual(
self._fake_command._translated_gcloud_storage_command, [
os.path.join('fake_dir', 'bin', 'gcloud'), 'objects', 'fake',
'--zip', 'opt1', '-x', 'arg1', 'arg2',
'--impersonate-service-account=fake_service_account'
])
def test_non_dry_mode_logs_translated_command_to_debug_logs(self):
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
with mock.patch.object(self._fake_command, 'logger',
autospec=True) as mock_logger:
self._fake_command.translate_to_gcloud_storage_if_requested()
# Verify translation.
mock_logger.debug.assert_called_once_with(
'Gcloud Storage Command: {} objects'
' fake --zip opt1 -x arg1 arg2'.format(
os.path.join('fake_dir', 'bin', 'gcloud')))
def test_use_gcloud_storage_true_with_hidden_shim_mode_not_set(self):
"""Should not raise error."""
with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
# return_stderr does not work here. Probably because we have
# defined the FakeCommand in the same module.
stdout, mock_log_handler = self.RunCommand('fake_shim',
args=['-i', 'arg1'],
return_stdout=True,
return_log_handler=True)
self.assertIn(
'Cannot translate gsutil command to gcloud storage.'
' Going to run gsutil command. Error: Command option "-i"'
' cannot be translated to gcloud storage',
mock_log_handler.messages['error'])
self.assertIn('FakeCommandWithGcloudStorageMap called', stdout)
def test_dry_run_mode_prints_translated_command(self):
"""Should print the gcloud command and run gsutil."""
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'True',
'hidden_shim_mode': 'dry_run'
}
}):
with util.SetEnvironmentForTest({'CLOUDSDK_ROOT_DIR': 'fake_dir'}):
stdout, mock_log_handler = self.RunCommand('fake_shim',
args=['arg1'],
return_stdout=True,
return_log_handler=True)
self.assertIn(
'Gcloud Storage Command: {} objects fake arg1'.format(
os.path.join('fake_dir', 'bin', 'gcloud')),
mock_log_handler.messages['info'])
self.assertIn(
'FakeCommandWithGcloudStorageMap called'.format(
os.path.join('fake_dir', 'bin', 'gcloud')), stdout)
def test_debug_value_4_adds_log_http_flag(self):
# Debug level 4 represents the -DD option.
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
self._fake_command.debug = 4
self._fake_command.translate_to_gcloud_storage_if_requested()
self.assertEqual(self._fake_command._translated_gcloud_storage_command,
[
os.path.join('fake_dir', 'bin', 'gcloud'),
'objects', 'fake', '--zip', 'opt1', '-x', 'arg1',
'arg2', '--verbosity', 'debug', '--log-http'
])
def test_returns_true_with_valid_gcloud_storage_map(self):
"""Should return True and perform the translation."""
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
self.assertTrue(
self._fake_command.translate_to_gcloud_storage_if_requested())
# Verify translation.
expected_gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud')
self.assertEqual(self._fake_command._translated_gcloud_storage_command,
[
expected_gcloud_path, 'objects', 'fake', '--zip',
'opt1', '-x', 'arg1', 'arg2'
])
def test_with_cloudsdk_root_dir_unset_and_gcloud_binary_path_set(self):
"""Should return True and perform the translation."""
gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud')
with _mock_boto_config({
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': None,
'GCLOUD_BINARY_PATH': gcloud_path,
}):
self.assertTrue(
self._fake_command.translate_to_gcloud_storage_if_requested())
# Verify translation.
self.assertEqual(self._fake_command._translated_gcloud_storage_command,
[
gcloud_path, 'objects', 'fake', '--zip', 'opt1',
'-x', 'arg1', 'arg2'
])
def test_top_level_flags_get_translated(self):
"""Should return True and perform the translation."""
boto_config = {
'GSUtil': {
'use_gcloud_storage': 'always',
'hidden_shim_mode': 'no_fallback'
}
}
with _mock_boto_config(boto_config):
with util.SetEnvironmentForTest({
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True',
'CLOUDSDK_ROOT_DIR': 'fake_dir',
}):
fake_command = FakeCommandWithGcloudStorageMap(
command_runner=mock.ANY,
args=['arg1', 'arg2'],
headers={}, # Headers will be tested separately.
debug=3, # -D option
trace_token='fake_trace_token',
user_project='fake_user_project',
parallel_operations=False, # Without the -m option.
bucket_storage_uri_class=mock.ANY,
gsutil_api_class_map_factory=mock.MagicMock())
self.assertTrue(fake_command.translate_to_gcloud_storage_if_requested())
# Verify translation.
expected_gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud')
self.assertEqual(fake_command._translated_gcloud_storage_command, [
expected_gcloud_path, 'objects', 'fake', 'arg1', 'arg2',
'--verbosity', 'debug', '--billing-project=fake_user_project',
'--trace-token=fake_trace_token'
])
self.assertCountEqual(
fake_command._translated_env_variables, {
'CLOUDSDK_STORAGE_PROCESS_COUNT': '1',
'CLOUDSDK_STORAGE_THREAD_COUNT': '1',
})
