def test_handles_and_logs_unprovisioning_os_error(self, mock_remove): mock_remove.side_effect = OSError('no') context_config.create_context_config(self.mock_logger) context_config._singleton_config.client_cert_path = 'some/path' context_config._singleton_config._unprovision_client_cert() self.mock_logger.error.assert_called_once_with( 'Failed to remove client certificate: no')
def test_context_config_is_a_singleton(self): first = context_config.create_context_config(self.mock_logger) with self.assertRaises( context_config.ContextConfigSingletonAlreadyExistsError): context_config.create_context_config(self.mock_logger) second = context_config.get_context_config() self.assertEqual(first, second)
def test_default_provider_not_found_error(self): with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None)]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( "Failed to provision client certificate: " "Client certificate provider file not found.")
def test_converts_and_logs_provisioning_os_error(self, mock_Popen): mock_Popen.side_effect = OSError('foobar') with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: foobar')
def test_raises_cert_provision_error_on_json_load_error( self, mock_open, mock_json_load): mock_json_load.side_effect = ValueError('valueError') with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None)]): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: valueError')
def test_executes_custom_provider_command_from_boto_config(self, mock_Popen): with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): # Purposely end execution here to avoid writing a file. with self.assertRaises(ValueError): context_config.create_context_config(self.mock_logger) mock_Popen.assert_called_once_with(os.path.realpath('some/path'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def test_default_provider_no_command_error(self, mock_open, mock_json_load): mock_json_load.return_value = DEFAULT_CERT_PROVIDER_FILE_NO_COMMAND with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None)]): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) self.mock_logger.error.assert_called_once_with( "Failed to provision client certificate: " "Client certificate provider command not found.")
def test_converts_and_logs_provisioning_cert_provider_unexpected_exit_error( self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.communicate.return_value = (None, 'oh no') mock_Popen.return_value = mock_command_process with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: oh no')
def testDoesNotAddPasswordFlagToCommandIfAlreadyThere(self, mock_Popen): with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'path --with_passphrase') ]): # Purposely end execution here to avoid writing a file. with self.assertRaises(ValueError): context_config.create_context_config(self.mock_logger) mock_Popen.assert_called_once_with( os.path.realpath('path --with_passphrase'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def test_executes_provider_command_from_default_file(self, mock_open, mock_Popen, mock_json_load): mock_json_load.side_effect = [DEFAULT_CERT_PROVIDER_FILE_CONTENTS] with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True') ]): # Purposely end execution here to avoid writing a file. with self.assertRaises(ValueError): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) mock_Popen.assert_called_once_with( os.path.realpath(os.path.join('some', 'helper')), '--print_certificate', '--with_passphrase')
def test_converts_and_logs_provisioning_key_error(self, mock_Popen): # Mocking f.write would make more sense, but mocking Popen earlier in the # function results in much less code and tests the same error handling. mock_Popen.side_effect = KeyError('foobar') with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) unicode_escaped_error_string = "'foobar'" if six.PY3 else "u'foobar'" self.mock_logger.error.assert_called_once_with( "Failed to provision client certificate:" " Invalid output format from certificate provider, no " + unicode_escaped_error_string)
def test_writes_and_deletes_certificate_file_storing_password_to_memory( self, mock_Popen, mock_remove, mock_open): mock_command_process = mock.Mock() mock_command_process.returncode = 0 mock_command_process.communicate.return_value = (FULL_CERT.encode(), None) mock_Popen.return_value = mock_command_process with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'path --print_certificate') ]): # Mock logger argument to avoid atexit hook writing to stderr. test_config = context_config.create_context_config(mock.Mock()) # Test writes certificate file. # Can't check whole mock_calls list because SetBotoConfigForTest also # uses the mock in Python 3. Should work with any_order=False based on # docs description but does not in current environment. mock_open.assert_has_calls([ mock.call(test_config.client_cert_path, 'w+'), mock.call().write(CERT_SECTION), mock.call().write(KEY_SECTION), ], any_order=True) # Test saves certificate password to memory. self.assertEqual(context_config._singleton_config.client_cert_password, PASSWORD) # Test deletes certificate file. context_config._singleton_config._unprovision_client_cert() mock_remove.assert_called_once_with(test_config.client_cert_path)
def test_does_not_unprovision_if_no_client_certificate(self, mock_remove): context_config.create_context_config(self.mock_logger) context_config._singleton_config._unprovision_client_cert() mock_remove.assert_not_called()
def testRaisesErrorIfCertProviderCommandAbsent(self): with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True')]): with self.assertRaises(context_config.CertProvisionError): context_config.create_context_config(self.mock_logger)
def test_does_not_provision_if_use_client_certificate_not_true( self, mock_Popen): context_config.create_context_config(self.mock_logger) mock_Popen.assert_not_called()
def testDoesNotUnprovisionIfNoClientCertificate(self, mock_remove): context_config.create_context_config(self.mock_logger) context_config._singleton_config._UnprovisionClientCert() mock_remove.assert_not_called()
def main(): InitializeSignalHandling() # Any modules used in initializing multiprocessing variables must be # imported after importing gslib.__main__. # pylint: disable=redefined-outer-name,g-import-not-at-top import gslib.boto_translation import gslib.command import gslib.utils.parallelism_framework_util # pylint: disable=unused-variable from gcs_oauth2_boto_plugin import oauth2_client from apitools.base.py import credentials_lib # pylint: enable=unused-variable if (gslib.utils.parallelism_framework_util. CheckMultiprocessingAvailableAndInit().is_available): # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.command.InitializeMultiprocessingVariables() gslib.boto_translation.InitializeMultiprocessingVariables() else: gslib.command.InitializeThreadingVariables() # This needs to be done after InitializeMultiprocessingVariables(), since # otherwise we can't call CreateLock. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin gsutil_client_id, gsutil_client_secret = ( system_util.GetGsutilClientIdAndSecret()) gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( gsutil_client_id, gsutil_client_secret) gcs_oauth2_boto_plugin.oauth2_helper.SetLock( gslib.utils.parallelism_framework_util.CreateLock()) credentials_lib.SetCredentialsCacheFileLock( gslib.utils.parallelism_framework_util.CreateLock()) except ImportError: pass global debug_level global test_exception_traces supported, err = check_python_version_support() if not supported: raise CommandException(err) sys.exit(1) boto_util.MonkeyPatchBoto() system_util.MonkeyPatchHttp() # In gsutil 4.0 and beyond, we don't use the boto library for the JSON # API. However, we still store gsutil configuration data in the .boto # config file for compatibility with previous versions and user convenience. # Many users have a .boto configuration file from previous versions, and it # is useful to have all of the configuration for gsutil stored in one place. command_runner = CommandRunner() if not boto_util.BOTO_IS_SECURE: raise CommandException('\n'.join( textwrap.wrap( 'Your boto configuration has is_secure = False. Gsutil cannot be ' 'run this way, for security reasons.'))) headers = {} parallel_operations = False quiet = False version = False debug_level = 0 trace_token = None perf_trace_token = None test_exception_traces = False user_project = None # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _CleanupSignalHandler) try: for o, a in opts: if o in ('-d', '--debug'): # Also causes boto to include httplib header output. debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS else: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o == '-u': user_project = a elif o in ('-v', '--version'): version = True elif o in ('-i', '--impersonate-service-account'): constants.IMPERSONATE_SERVICE_ACCOUNT = a elif o == '--perf-trace-token': perf_trace_token = a elif o == '--trace-token': trace_token = a elif o == '--testexceptiontraces': # Hidden flag for integration tests. test_exception_traces = True # Avoid printing extra warnings to stderr regarding long retries by # setting the threshold very high. constants.LONG_RETRY_WARN_SEC = 3600 elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) # Now that any Boto option overrides (via `-o` args) have been parsed, # perform initialization that depends on those options. boto_util.configured_certs_file = (boto_util.ConfigureCertsFile()) metrics.LogCommandParams(global_opts=opts) httplib2.debuglevel = debug_level if trace_token: sys.stderr.write(TRACE_WARNING) if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS: sys.stderr.write(DEBUG_WARNING) _ConfigureRootLogger(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] for config_section in ('Boto', 'GSUtil'): try: config_items.extend(boto.config.items(config_section)) except configparser.NoSectionError: pass for i in range(len(config_items)): config_item_key = config_items[i][0] if config_item_key in CONFIG_KEYS_TO_REDACT: config_items[i] = (config_item_key, 'REDACTED') sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) sys.stderr.write('config_file_list: %s\n' % boto_util.GetFriendlyConfigFilePaths()) sys.stderr.write('config: %s\n' % str(config_items)) else: # Non-debug log level. root_logger_level = logging.WARNING if quiet else logging.INFO # oauth2client uses INFO and WARNING logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing # access tokens), so we bump the threshold one level higher where # appropriate. These log levels work for regular- and quiet-level logging. oa2c_logger_level = logging.WARNING oa2c_multiprocess_file_storage_logger_level = logging.ERROR _ConfigureRootLogger(level=root_logger_level) oauth2client.client.logger.setLevel(oa2c_logger_level) oauth2client.contrib.multiprocess_file_storage.logger.setLevel( oa2c_multiprocess_file_storage_logger_level) # pylint: disable=protected-access oauth2client.transport._LOGGER.setLevel(oa2c_logger_level) reauth_creds._LOGGER.setLevel(oa2c_logger_level) # pylint: enable=protected-access # Initialize context configuration for device mTLS. context_config.create_context_config(logging.getLogger()) # TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3. # Fixes pyu2f v0.1.3 bug. import six # pylint: disable=g-import-not-at-top six.input = six.moves.input if not boto_util.CERTIFICATE_VALIDATION_ENABLED: sys.stderr.write(HTTP_WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] _CheckAndWarnForProxyDifferences() # Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at # what argv[] index the command to match starts. We want it to start at the # value for the path to gsutil, so: # $ gsutil <command> # Should be the 1st argument, so '1' # $ python gsutil <command> # Should be the 2nd argument, so '2' # Both are valid; most users invoke gsutil in the first style, but our # integration and prerelease tests invoke it in the second style, as we need # to specify the Python interpreter used to run gsutil. if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'): return _PerformTabCompletion(command_runner) return _RunNamedCommandAndHandleExceptions( command_runner, command_name, args=args[1:], headers=headers, debug_level=debug_level, trace_token=trace_token, parallel_operations=parallel_operations, perf_trace_token=perf_trace_token, user_project=user_project) finally: _Cleanup()
def testDoesNotProvisionIfUseClientCertificateNotTrue(self, mock_Popen): context_config.create_context_config(self.mock_logger) mock_Popen.assert_not_called()