def test_handles_and_logs_unprovisioning_os_error(self, mock_remove):
mock_remove.side_effect = OSError('no')
context_config.create_context_config(self.mock_logger)
context_config._singleton_config.client_cert_path = 'some/path'
context_config._singleton_config._unprovision_client_cert()
self.mock_logger.error.assert_called_once_with(
'Failed to remove client certificate: no')
def test_context_config_is_a_singleton(self):
first = context_config.create_context_config(self.mock_logger)
with self.assertRaises(
context_config.ContextConfigSingletonAlreadyExistsError):
context_config.create_context_config(self.mock_logger)
second = context_config.get_context_config()
self.assertEqual(first, second)
def test_default_provider_not_found_error(self):
with SetBotoConfigForTest([('Credentials', 'use_client_certificate',
'True'),
('Credentials', 'cert_provider_command', None)]):
context_config.create_context_config(self.mock_logger)
self.mock_logger.error.assert_called_once_with(
"Failed to provision client certificate: "
"Client certificate provider file not found.")
def test_converts_and_logs_provisioning_os_error(self, mock_Popen):
mock_Popen.side_effect = OSError('foobar')
with SetBotoConfigForTest([
('Credentials', 'use_client_certificate', 'True'),
('Credentials', 'cert_provider_command', 'some/path')
]):
context_config.create_context_config(self.mock_logger)
self.mock_logger.error.assert_called_once_with(
'Failed to provision client certificate: foobar')
def test_raises_cert_provision_error_on_json_load_error(
self, mock_open, mock_json_load):
mock_json_load.side_effect = ValueError('valueError')
with SetBotoConfigForTest([('Credentials', 'use_client_certificate',
'True'),
('Credentials', 'cert_provider_command', None)]):
context_config.create_context_config(self.mock_logger)
mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH)
self.mock_logger.error.assert_called_once_with(
'Failed to provision client certificate: valueError')
def test_executes_custom_provider_command_from_boto_config(self, mock_Popen):
with SetBotoConfigForTest([
('Credentials', 'use_client_certificate', 'True'),
('Credentials', 'cert_provider_command', 'some/path')
]):
# Purposely end execution here to avoid writing a file.
with self.assertRaises(ValueError):
context_config.create_context_config(self.mock_logger)
mock_Popen.assert_called_once_with(os.path.realpath('some/path'),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def test_default_provider_no_command_error(self, mock_open, mock_json_load):
mock_json_load.return_value = DEFAULT_CERT_PROVIDER_FILE_NO_COMMAND
with SetBotoConfigForTest([('Credentials', 'use_client_certificate',
'True'),
('Credentials', 'cert_provider_command', None)]):
context_config.create_context_config(self.mock_logger)
mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH)
self.mock_logger.error.assert_called_once_with(
"Failed to provision client certificate: "
"Client certificate provider command not found.")
def test_converts_and_logs_provisioning_cert_provider_unexpected_exit_error(
self, mock_Popen):
mock_command_process = mock.Mock()
mock_command_process.communicate.return_value = (None, 'oh no')
mock_Popen.return_value = mock_command_process
with SetBotoConfigForTest([
('Credentials', 'use_client_certificate', 'True'),
('Credentials', 'cert_provider_command', 'some/path')
]):
context_config.create_context_config(self.mock_logger)
self.mock_logger.error.assert_called_once_with(
'Failed to provision client certificate: oh no')
def testDoesNotAddPasswordFlagToCommandIfAlreadyThere(self, mock_Popen):
with SetBotoConfigForTest([
('Credentials', 'use_client_certificate', 'True'),
('Credentials', 'cert_provider_command', 'path --with_passphrase')
]):
# Purposely end execution here to avoid writing a file.
with self.assertRaises(ValueError):
context_config.create_context_config(self.mock_logger)
mock_Popen.assert_called_once_with(
os.path.realpath('path --with_passphrase'),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def test_executes_provider_command_from_default_file(self, mock_open,
mock_Popen,
mock_json_load):
mock_json_load.side_effect = [DEFAULT_CERT_PROVIDER_FILE_CONTENTS]
with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True')
]):
# Purposely end execution here to avoid writing a file.
with self.assertRaises(ValueError):
context_config.create_context_config(self.mock_logger)
mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH)
mock_Popen.assert_called_once_with(
os.path.realpath(os.path.join('some', 'helper')),
'--print_certificate', '--with_passphrase')
def test_converts_and_logs_provisioning_key_error(self, mock_Popen):
# Mocking f.write would make more sense, but mocking Popen earlier in the
# function results in much less code and tests the same error handling.
mock_Popen.side_effect = KeyError('foobar')
with SetBotoConfigForTest([
('Credentials', 'use_client_certificate', 'True'),
('Credentials', 'cert_provider_command', 'some/path')
]):
context_config.create_context_config(self.mock_logger)
unicode_escaped_error_string = "'foobar'" if six.PY3 else "u'foobar'"
self.mock_logger.error.assert_called_once_with(
"Failed to provision client certificate:"
" Invalid output format from certificate provider, no " +
unicode_escaped_error_string)
def test_writes_and_deletes_certificate_file_storing_password_to_memory(
self, mock_Popen, mock_remove, mock_open):
mock_command_process = mock.Mock()
mock_command_process.returncode = 0
mock_command_process.communicate.return_value = (FULL_CERT.encode(), None)
mock_Popen.return_value = mock_command_process
with SetBotoConfigForTest([
('Credentials', 'use_client_certificate', 'True'),
('Credentials', 'cert_provider_command', 'path --print_certificate')
]):
# Mock logger argument to avoid atexit hook writing to stderr.
test_config = context_config.create_context_config(mock.Mock())
# Test writes certificate file.
# Can't check whole mock_calls list because SetBotoConfigForTest also
# uses the mock in Python 3. Should work with any_order=False based on
# docs description but does not in current environment.
mock_open.assert_has_calls([
mock.call(test_config.client_cert_path, 'w+'),
mock.call().write(CERT_SECTION),
mock.call().write(KEY_SECTION),
],
any_order=True)
# Test saves certificate password to memory.
self.assertEqual(context_config._singleton_config.client_cert_password,
PASSWORD)
# Test deletes certificate file.
context_config._singleton_config._unprovision_client_cert()
mock_remove.assert_called_once_with(test_config.client_cert_path)
def test_does_not_unprovision_if_no_client_certificate(self, mock_remove): context_config.create_context_config(self.mock_logger) context_config._singleton_config._unprovision_client_cert() mock_remove.assert_not_called()
def testRaisesErrorIfCertProviderCommandAbsent(self):
with SetBotoConfigForTest([('Credentials', 'use_client_certificate',
'True')]):
with self.assertRaises(context_config.CertProvisionError):
context_config.create_context_config(self.mock_logger)
def test_does_not_provision_if_use_client_certificate_not_true(
self, mock_Popen):
context_config.create_context_config(self.mock_logger)
mock_Popen.assert_not_called()
def testDoesNotUnprovisionIfNoClientCertificate(self, mock_remove):
context_config.create_context_config(self.mock_logger)
context_config._singleton_config._UnprovisionClientCert()
mock_remove.assert_not_called()
def main():
InitializeSignalHandling()
# Any modules used in initializing multiprocessing variables must be
# imported after importing gslib.__main__.
# pylint: disable=redefined-outer-name,g-import-not-at-top
import gslib.boto_translation
import gslib.command
import gslib.utils.parallelism_framework_util
# pylint: disable=unused-variable
from gcs_oauth2_boto_plugin import oauth2_client
from apitools.base.py import credentials_lib
# pylint: enable=unused-variable
if (gslib.utils.parallelism_framework_util.
CheckMultiprocessingAvailableAndInit().is_available):
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.command.InitializeMultiprocessingVariables()
gslib.boto_translation.InitializeMultiprocessingVariables()
else:
gslib.command.InitializeThreadingVariables()
# This needs to be done after InitializeMultiprocessingVariables(), since
# otherwise we can't call CreateLock.
try:
# pylint: disable=unused-import,g-import-not-at-top
import gcs_oauth2_boto_plugin
gsutil_client_id, gsutil_client_secret = (
system_util.GetGsutilClientIdAndSecret())
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
gsutil_client_id, gsutil_client_secret)
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(
gslib.utils.parallelism_framework_util.CreateLock())
credentials_lib.SetCredentialsCacheFileLock(
gslib.utils.parallelism_framework_util.CreateLock())
except ImportError:
pass
global debug_level
global test_exception_traces
supported, err = check_python_version_support()
if not supported:
raise CommandException(err)
sys.exit(1)
boto_util.MonkeyPatchBoto()
system_util.MonkeyPatchHttp()
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
# API. However, we still store gsutil configuration data in the .boto
# config file for compatibility with previous versions and user convenience.
# Many users have a .boto configuration file from previous versions, and it
# is useful to have all of the configuration for gsutil stored in one place.
command_runner = CommandRunner()
if not boto_util.BOTO_IS_SECURE:
raise CommandException('\n'.join(
textwrap.wrap(
'Your boto configuration has is_secure = False. Gsutil cannot be '
'run this way, for security reasons.')))
headers = {}
parallel_operations = False
quiet = False
version = False
debug_level = 0
trace_token = None
perf_trace_token = None
test_exception_traces = False
user_project = None
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
for signal_num in GetCaughtSignals():
RegisterSignalHandler(signal_num, _CleanupSignalHandler)
try:
for o, a in opts:
if o in ('-d', '--debug'):
# Also causes boto to include httplib header output.
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS:
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS
else:
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o == '-u':
user_project = a
elif o in ('-v', '--version'):
version = True
elif o in ('-i', '--impersonate-service-account'):
constants.IMPERSONATE_SERVICE_ACCOUNT = a
elif o == '--perf-trace-token':
perf_trace_token = a
elif o == '--trace-token':
trace_token = a
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
test_exception_traces = True
# Avoid printing extra warnings to stderr regarding long retries by
# setting the threshold very high.
constants.LONG_RETRY_WARN_SEC = 3600
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
# Now that any Boto option overrides (via `-o` args) have been parsed,
# perform initialization that depends on those options.
boto_util.configured_certs_file = (boto_util.ConfigureCertsFile())
metrics.LogCommandParams(global_opts=opts)
httplib2.debuglevel = debug_level
if trace_token:
sys.stderr.write(TRACE_WARNING)
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS:
sys.stderr.write(DEBUG_WARNING)
_ConfigureRootLogger(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
for config_section in ('Boto', 'GSUtil'):
try:
config_items.extend(boto.config.items(config_section))
except configparser.NoSectionError:
pass
for i in range(len(config_items)):
config_item_key = config_items[i][0]
if config_item_key in CONFIG_KEYS_TO_REDACT:
config_items[i] = (config_item_key, 'REDACTED')
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
sys.stderr.write('config_file_list: %s\n' %
boto_util.GetFriendlyConfigFilePaths())
sys.stderr.write('config: %s\n' % str(config_items))
else: # Non-debug log level.
root_logger_level = logging.WARNING if quiet else logging.INFO
# oauth2client uses INFO and WARNING logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing
# access tokens), so we bump the threshold one level higher where
# appropriate. These log levels work for regular- and quiet-level logging.
oa2c_logger_level = logging.WARNING
oa2c_multiprocess_file_storage_logger_level = logging.ERROR
_ConfigureRootLogger(level=root_logger_level)
oauth2client.client.logger.setLevel(oa2c_logger_level)
oauth2client.contrib.multiprocess_file_storage.logger.setLevel(
oa2c_multiprocess_file_storage_logger_level)
# pylint: disable=protected-access
oauth2client.transport._LOGGER.setLevel(oa2c_logger_level)
reauth_creds._LOGGER.setLevel(oa2c_logger_level)
# pylint: enable=protected-access
# Initialize context configuration for device mTLS.
context_config.create_context_config(logging.getLogger())
# TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3.
# Fixes pyu2f v0.1.3 bug.
import six # pylint: disable=g-import-not-at-top
six.input = six.moves.input
if not boto_util.CERTIFICATE_VALIDATION_ENABLED:
sys.stderr.write(HTTP_WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
_CheckAndWarnForProxyDifferences()
# Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at
# what argv[] index the command to match starts. We want it to start at the
# value for the path to gsutil, so:
# $ gsutil <command> # Should be the 1st argument, so '1'
# $ python gsutil <command> # Should be the 2nd argument, so '2'
# Both are valid; most users invoke gsutil in the first style, but our
# integration and prerelease tests invoke it in the second style, as we need
# to specify the Python interpreter used to run gsutil.
if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'):
return _PerformTabCompletion(command_runner)
return _RunNamedCommandAndHandleExceptions(
command_runner,
command_name,
args=args[1:],
headers=headers,
debug_level=debug_level,
trace_token=trace_token,
parallel_operations=parallel_operations,
perf_trace_token=perf_trace_token,
user_project=user_project)
finally:
_Cleanup()
def testDoesNotProvisionIfUseClientCertificateNotTrue(self, mock_Popen):
context_config.create_context_config(self.mock_logger)
mock_Popen.assert_not_called()
