def test_translated_boto_config_gets_added(self): """Should add translated env vars as well flags.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'True', 'hidden_shim_mode': 'no_fallback', 'content_language': 'foo', 'default_project_id': 'fake_project', } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud') self.assertEqual( self._fake_command._translated_gcloud_storage_command, [ expected_gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--content-language=foo' ]) self.assertEqual(self._fake_command._translated_env_variables, {'CLOUDSDK_CORE_PROJECT': 'fake_project'})
def test_calls_subprocess_with_translated_command_and_env_vars( self, mock_run, mock_environ_copy): command_instance = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers=mock.ANY, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): command_instance._translated_env_variables = { 'new_key': 'new_value', } command_instance._translated_gcloud_storage_command = ['gcloud', 'foo'] actual_return_code = command_instance.run_gcloud_storage() mock_run.assert_called_once_with(['gcloud', 'foo'], env={ 'old_key': 'old_value', 'new_key': 'new_value' }) mock_environ_copy.assert_called_once_with() self.assertEqual(actual_return_code, mock_run.return_value.returncode)
def test_translated_headers_get_added_to_final_command(self): with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['arg1', 'arg2'], headers={'Content-Type': 'fake_val'}, debug=1, trace_token=None, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) self.assertTrue(fake_command.translate_to_gcloud_storage_if_requested()) self.assertEqual(fake_command._translated_gcloud_storage_command, [ os.path.join('fake_dir', 'bin', 'gcloud'), 'objects', 'fake', 'arg1', 'arg2', '--content-type=fake_val' ])
def test_parallel_operations_true_does_not_add_process_count_env_vars(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.parallel_operations = True self._fake_command.translate_to_gcloud_storage_if_requested() self.assertNotIn('CLOUDSDK_STORAGE_PROCESS_COUNT', self._fake_command._translated_env_variables) self.assertNotIn('CLOUDSDK_STORAGE_THREAD_COUNT', self._fake_command._translated_env_variables)
def test_raises_error_if_gcloud_storage_map_missing(self): self._fake_command.gcloud_storage_map = None with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex( exception.CommandException, 'CommandException: Command "fake_shim" cannot be translated to' ' gcloud storage because the translation mapping is missing.'): self._fake_command.translate_to_gcloud_storage_if_requested()
def test_runs_gcloud_storage_if_use_gcloud_storage_true(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': None, }): stderr = self.RunGsUtil(['-D', 'ls'], return_stderr=True, expected_status=1) # This is a proxy to ensure that the test attempted to call # gcloud binary. self.assertIn('gcloud binary path cannot be found', stderr)
def test_raises_error_if_pass_credentials_to_gsutil_is_missing(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_ROOT_DIR': 'fake_dir', 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': None }): with self.assertRaisesRegex( exception.CommandException, 'CommandException: Requested to use "gcloud storage" but gsutil' ' is not using the same credentials as' ' gcloud. You can make gsutil use the same credentials' ' by running:\n' 'fake_dir.bin.gcloud config set pass_credentials_to_gsutil True'): self._fake_command.translate_to_gcloud_storage_if_requested()
def test_raises_error_if_cloudsdk_root_dir_is_none(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_ROOT_DIR': None, }): with self.assertRaisesRegex( exception.CommandException, 'CommandException: Requested to use "gcloud storage" but the ' 'gcloud binary path cannot be found. This might happen if you' ' attempt to use gsutil that was not installed via Cloud SDK.' ' You can manually set the `CLOUDSDK_ROOT_DIR` environment variable' ' to point to the google-cloud-sdk installation directory to' ' resolve the issue. Alternatively, you can set' ' `use_gcloud_storage=False` to disable running the command' ' using gcloud storage.'): self._fake_command.translate_to_gcloud_storage_if_requested()
def test_quiet_mode_translation_adds_no_user_output_enabled_flag(self): with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.quiet_mode = True self._fake_command.translate_to_gcloud_storage_if_requested() self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ os.path.join('fake_dir', 'bin', 'gcloud'), 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--no-user-output-enabled' ])
def test_impersonate_service_account_translation(self): """Should add the --impersonate-service-account flag.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.translate_to_gcloud_storage_if_requested() self.assertEqual( self._fake_command._translated_gcloud_storage_command, [ os.path.join('fake_dir', 'bin', 'gcloud'), 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--impersonate-service-account=fake_service_account' ])
def test_non_dry_mode_logs_translated_command_to_debug_logs(self): with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with mock.patch.object(self._fake_command, 'logger', autospec=True) as mock_logger: self._fake_command.translate_to_gcloud_storage_if_requested() # Verify translation. mock_logger.debug.assert_called_once_with( 'Gcloud Storage Command: {} objects' ' fake --zip opt1 -x arg1 arg2'.format( os.path.join('fake_dir', 'bin', 'gcloud')))
def test_use_gcloud_storage_true_with_hidden_shim_mode_not_set(self): """Should not raise error.""" with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # return_stderr does not work here. Probably because we have # defined the FakeCommand in the same module. stdout, mock_log_handler = self.RunCommand('fake_shim', args=['-i', 'arg1'], return_stdout=True, return_log_handler=True) self.assertIn( 'Cannot translate gsutil command to gcloud storage.' ' Going to run gsutil command. Error: Command option "-i"' ' cannot be translated to gcloud storage', mock_log_handler.messages['error']) self.assertIn('FakeCommandWithGcloudStorageMap called', stdout)
def test_dry_run_mode_prints_translated_command(self): """Should print the gcloud command and run gsutil.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'True', 'hidden_shim_mode': 'dry_run' } }): with util.SetEnvironmentForTest({'CLOUDSDK_ROOT_DIR': 'fake_dir'}): stdout, mock_log_handler = self.RunCommand('fake_shim', args=['arg1'], return_stdout=True, return_log_handler=True) self.assertIn( 'Gcloud Storage Command: {} objects fake arg1'.format( os.path.join('fake_dir', 'bin', 'gcloud')), mock_log_handler.messages['info']) self.assertIn( 'FakeCommandWithGcloudStorageMap called'.format( os.path.join('fake_dir', 'bin', 'gcloud')), stdout)
def test_debug_value_4_adds_log_http_flag(self): # Debug level 4 represents the -DD option. with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.debug = 4 self._fake_command.translate_to_gcloud_storage_if_requested() self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ os.path.join('fake_dir', 'bin', 'gcloud'), 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--verbosity', 'debug', '--log-http' ])
def test_returns_true_with_valid_gcloud_storage_map(self): """Should return True and perform the translation.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud') self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ expected_gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2' ])
def test_with_cloudsdk_root_dir_unset_and_gcloud_binary_path_set(self): """Should return True and perform the translation.""" gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud') with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': None, 'GCLOUD_BINARY_PATH': gcloud_path, }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2' ])
def test_top_level_flags_get_translated(self): """Should return True and perform the translation.""" boto_config = { 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } } with _mock_boto_config(boto_config): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['arg1', 'arg2'], headers={}, # Headers will be tested separately. debug=3, # -D option trace_token='fake_trace_token', user_project='fake_user_project', parallel_operations=False, # Without the -m option. bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) self.assertTrue(fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = os.path.join('fake_dir', 'bin', 'gcloud') self.assertEqual(fake_command._translated_gcloud_storage_command, [ expected_gcloud_path, 'objects', 'fake', 'arg1', 'arg2', '--verbosity', 'debug', '--billing-project=fake_user_project', '--trace-token=fake_trace_token' ]) self.assertCountEqual( fake_command._translated_env_variables, { 'CLOUDSDK_STORAGE_PROCESS_COUNT': '1', 'CLOUDSDK_STORAGE_THREAD_COUNT': '1', })