def test_command_argument_parser_setup_completers(self):
command_map = {
FakeCommandWithCompleters.command_spec.command_name:
FakeCommandWithCompleters()
}
runner = CommandRunner(
bucket_storage_uri_class=self.mock_bucket_storage_uri,
gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory,
command_map=command_map)
subparsers = FakeArgparseSubparsers()
runner.ConfigureCommandArgumentParsers(subparsers)
self.assertEqual(1, len(subparsers.parsers))
parser = subparsers.parsers[0]
self.assertEqual(6, len(parser.arguments))
self.assertEqual(CloudObjectCompleter, type(parser.arguments[0].completer))
self.assertEqual(LocalObjectCompleter, type(parser.arguments[1].completer))
self.assertEqual(
CloudOrLocalObjectCompleter, type(parser.arguments[2].completer))
self.assertEqual(
NoOpCompleter, type(parser.arguments[3].completer))
self.assertEqual(CloudObjectCompleter, type(parser.arguments[4].completer))
self.assertEqual(
LocalObjectOrCannedACLCompleter, type(parser.arguments[5].completer))
def test_command_argument_parser_setup_nested_argparse_arguments(self):
command_map = {
FakeCommandWithNestedArguments.command_spec.command_name:
FakeCommandWithNestedArguments(),
}
runner = CommandRunner(
bucket_storage_uri_class=self.mock_bucket_storage_uri,
gsutil_api_class_map_factory=self.
mock_gsutil_api_class_map_factory,
command_map=command_map)
parser = FakeArgparseParser()
runner.ConfigureCommandArgumentParsers(parser)
# Start at first level of subparsers.
cur_subparser = parser.subparsers.parsers[0]
self.assertEqual(
cur_subparser.name,
FakeCommandWithNestedArguments.command_spec.command_name)
# Go one level further, checking the current subparser's first subparser.
cur_subparser = cur_subparser.subparsers.parsers[0]
self.assertEqual(cur_subparser.name,
FakeCommandWithNestedArguments.subcommand_name)
cur_subparser = cur_subparser.subparsers.parsers[0]
self.assertEqual(cur_subparser.name,
FakeCommandWithNestedArguments.subcommand_subname)
# FakeArgparseParser adds a FakeArgparseArgument in place of any arguments
# passed in via add_argument(), so we can't check for anything beyond the
# presence of the correct number of arguments here.
self.assertEqual(len(cur_subparser.arguments),
FakeCommandWithNestedArguments.num_args)
def test_command_argument_parser_setup_invalid_completer(self):
command_map = {
FakeCommandWithInvalidCompleter.command_spec.command_name:
FakeCommandWithInvalidCompleter()
}
runner = CommandRunner(
bucket_storage_uri_class=self.mock_bucket_storage_uri,
gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory,
command_map=command_map)
subparsers = FakeArgparseSubparsers()
try:
runner.ConfigureCommandArgumentParsers(subparsers)
except RuntimeError as e:
self.assertIn('Unknown completer', e.message)
def setUpClass(cls):
base.GsUtilTestCase.setUpClass()
cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri
cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory
cls.logger = logging.getLogger()
cls.command_runner = CommandRunner(
bucket_storage_uri_class=cls.mock_bucket_storage_uri,
gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory)
def setUpClass(cls):
base.GsUtilTestCase.setUpClass()
cls.mock_bucket_storage_uri = GSMockBucketStorageUri
cls.proj_id_handler = ProjectIdHandler()
config_file_list = boto.pyami.config.BotoConfigLocations
# Use "gsutil_test_commands" as a fake UserAgent. This value will never be
# sent via HTTP because we're using MockStorageService here.
cls.command_runner = CommandRunner(config_file_list,
cls.mock_bucket_storage_uri)
def setUpClass(cls):
base.GsUtilTestCase.setUpClass()
cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri
cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory
cls.logger = logging.getLogger()
cls.command_runner = CommandRunner(
bucket_storage_uri_class=cls.mock_bucket_storage_uri,
gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory)
# Ensure unit tests don't fail if no default_project_id is defined in the
# boto config file.
project_id.UNIT_TEST_PROJECT_ID = 'mock-project-id-for-unit-tests'
def main():
InitializeSignalHandling()
# Any modules used in initializing multiprocessing variables must be
# imported after importing gslib.__main__.
# pylint: disable=redefined-outer-name,g-import-not-at-top
import gslib.boto_translation
import gslib.command
import gslib.utils.parallelism_framework_util
# pylint: disable=unused-variable
from gcs_oauth2_boto_plugin import oauth2_client
from apitools.base.py import credentials_lib
# pylint: enable=unused-variable
if (gslib.utils.parallelism_framework_util.
CheckMultiprocessingAvailableAndInit().is_available):
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.command.InitializeMultiprocessingVariables()
gslib.boto_translation.InitializeMultiprocessingVariables()
else:
gslib.command.InitializeThreadingVariables()
# This needs to be done after InitializeMultiprocessingVariables(), since
# otherwise we can't call CreateLock.
try:
# pylint: disable=unused-import,g-import-not-at-top
import gcs_oauth2_boto_plugin
gsutil_client_id, gsutil_client_secret = (
system_util.GetGsutilClientIdAndSecret())
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
gsutil_client_id, gsutil_client_secret)
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(
gslib.utils.parallelism_framework_util.CreateLock())
credentials_lib.SetCredentialsCacheFileLock(
gslib.utils.parallelism_framework_util.CreateLock())
except ImportError:
pass
global debug_level
global test_exception_traces
supported, err = check_python_version_support()
if not supported:
raise CommandException(err)
sys.exit(1)
boto_util.MonkeyPatchBoto()
system_util.MonkeyPatchHttp()
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
# API. However, we still store gsutil configuration data in the .boto
# config file for compatibility with previous versions and user convenience.
# Many users have a .boto configuration file from previous versions, and it
# is useful to have all of the configuration for gsutil stored in one place.
command_runner = CommandRunner()
if not boto_util.BOTO_IS_SECURE:
raise CommandException('\n'.join(
textwrap.wrap(
'Your boto configuration has is_secure = False. Gsutil cannot be '
'run this way, for security reasons.')))
headers = {}
parallel_operations = False
quiet = False
version = False
debug_level = 0
trace_token = None
perf_trace_token = None
test_exception_traces = False
user_project = None
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
for signal_num in GetCaughtSignals():
RegisterSignalHandler(signal_num, _CleanupSignalHandler)
try:
for o, a in opts:
if o in ('-d', '--debug'):
# Also causes boto to include httplib header output.
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS:
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS
else:
debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o == '-u':
user_project = a
elif o in ('-v', '--version'):
version = True
elif o in ('-i', '--impersonate-service-account'):
constants.IMPERSONATE_SERVICE_ACCOUNT = a
elif o == '--perf-trace-token':
perf_trace_token = a
elif o == '--trace-token':
trace_token = a
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
test_exception_traces = True
# Avoid printing extra warnings to stderr regarding long retries by
# setting the threshold very high.
constants.LONG_RETRY_WARN_SEC = 3600
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
# Now that any Boto option overrides (via `-o` args) have been parsed,
# perform initialization that depends on those options.
boto_util.configured_certs_file = (boto_util.ConfigureCertsFile())
metrics.LogCommandParams(global_opts=opts)
httplib2.debuglevel = debug_level
if trace_token:
sys.stderr.write(TRACE_WARNING)
if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS:
sys.stderr.write(DEBUG_WARNING)
_ConfigureRootLogger(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
for config_section in ('Boto', 'GSUtil'):
try:
config_items.extend(boto.config.items(config_section))
except configparser.NoSectionError:
pass
for i in range(len(config_items)):
config_item_key = config_items[i][0]
if config_item_key in CONFIG_KEYS_TO_REDACT:
config_items[i] = (config_item_key, 'REDACTED')
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
sys.stderr.write('config_file_list: %s\n' %
boto_util.GetFriendlyConfigFilePaths())
sys.stderr.write('config: %s\n' % str(config_items))
else: # Non-debug log level.
root_logger_level = logging.WARNING if quiet else logging.INFO
# oauth2client uses INFO and WARNING logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing
# access tokens), so we bump the threshold one level higher where
# appropriate. These log levels work for regular- and quiet-level logging.
oa2c_logger_level = logging.WARNING
oa2c_multiprocess_file_storage_logger_level = logging.ERROR
_ConfigureRootLogger(level=root_logger_level)
oauth2client.client.logger.setLevel(oa2c_logger_level)
oauth2client.contrib.multiprocess_file_storage.logger.setLevel(
oa2c_multiprocess_file_storage_logger_level)
# pylint: disable=protected-access
oauth2client.transport._LOGGER.setLevel(oa2c_logger_level)
reauth_creds._LOGGER.setLevel(oa2c_logger_level)
# pylint: enable=protected-access
# Initialize context configuration for device mTLS.
context_config.create_context_config(logging.getLogger())
# TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3.
# Fixes pyu2f v0.1.3 bug.
import six # pylint: disable=g-import-not-at-top
six.input = six.moves.input
if not boto_util.CERTIFICATE_VALIDATION_ENABLED:
sys.stderr.write(HTTP_WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
_CheckAndWarnForProxyDifferences()
# Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at
# what argv[] index the command to match starts. We want it to start at the
# value for the path to gsutil, so:
# $ gsutil <command> # Should be the 1st argument, so '1'
# $ python gsutil <command> # Should be the 2nd argument, so '2'
# Both are valid; most users invoke gsutil in the first style, but our
# integration and prerelease tests invoke it in the second style, as we need
# to specify the Python interpreter used to run gsutil.
if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'):
return _PerformTabCompletion(command_runner)
return _RunNamedCommandAndHandleExceptions(
command_runner,
command_name,
args=args[1:],
headers=headers,
debug_level=debug_level,
trace_token=trace_token,
parallel_operations=parallel_operations,
perf_trace_token=perf_trace_token,
user_project=user_project)
finally:
_Cleanup()
def main():
InitializeSignalHandling()
# Any modules used in initializing multiprocessing variables must be
# imported after importing gslib.__main__.
# pylint: disable=redefined-outer-name,g-import-not-at-top
import gslib.boto_translation
import gslib.command
import gslib.util
from gslib.util import BOTO_IS_SECURE
from gslib.util import CERTIFICATE_VALIDATION_ENABLED
# pylint: disable=unused-variable
from gcs_oauth2_boto_plugin import oauth2_client
from apitools.base.py import credentials_lib
# pylint: enable=unused-variable
from gslib.util import CheckMultiprocessingAvailableAndInit
if CheckMultiprocessingAvailableAndInit().is_available:
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.command.InitializeMultiprocessingVariables()
gslib.boto_translation.InitializeMultiprocessingVariables()
else:
gslib.command.InitializeThreadingVariables()
# This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
# since otherwise we can't call gslib.util.CreateLock.
try:
# pylint: disable=unused-import,g-import-not-at-top
import gcs_oauth2_boto_plugin
gsutil_client_id, gsutil_client_secret = GetGsutilClientIdAndSecret()
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
gsutil_client_id, gsutil_client_secret)
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
credentials_lib.SetCredentialsCacheFileLock(CreateLock())
except ImportError:
pass
global debug
global test_exception_traces
if not (2, 7) <= sys.version_info[:3] < (3,):
raise CommandException('gsutil requires python 2.7.')
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
# API. However, we still store gsutil configuration data in the .boto
# config file for compatibility with previous versions and user convenience.
# Many users have a .boto configuration file from previous versions, and it
# is useful to have all of the configuration for gsutil stored in one place.
command_runner = CommandRunner()
if not BOTO_IS_SECURE:
raise CommandException('\n'.join(textwrap.wrap(
'Your boto configuration has is_secure = False. Gsutil cannot be '
'run this way, for security reasons.')))
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
trace_token = None
perf_trace_token = None
test_exception_traces = False
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
gslib.util.configured_certs_file = gslib.util.ConfigureCertsFile()
for signal_num in GetCaughtSignals():
RegisterSignalHandler(signal_num, _CleanupSignalHandler)
GetCertsFile()
try:
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
['debug', 'detailedDebug', 'version', 'option',
'help', 'header', 'multithreaded', 'quiet',
'testexceptiontraces', 'trace-token=',
'perf-trace-token='])
except getopt.GetoptError as e:
_HandleCommandException(CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Also causes boto to include httplib header output.
debug = DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == DEBUGLEVEL_DUMP_REQUESTS:
debug = DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS
else:
debug = DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
elif o == '--perf-trace-token':
perf_trace_token = a
elif o == '--trace-token':
trace_token = a
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
test_exception_traces = True
# Avoid printing extra warnings to stderr regarding long retries by
# setting the threshold very high.
gslib.util.LONG_RETRY_WARN_SEC = 3600
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
metrics.LogCommandParams(global_opts=opts)
httplib2.debuglevel = debug
if trace_token:
sys.stderr.write(TRACE_WARNING)
if debug >= DEBUGLEVEL_DUMP_REQUESTS:
sys.stderr.write(DEBUG_WARNING)
_ConfigureLogging(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
for i in xrange(len(config_items)):
config_item_key = config_items[i][0]
if config_item_key in CONFIG_KEYS_TO_REDACT:
config_items[i] = (config_item_key, 'REDACTED')
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList())
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
_ConfigureLogging(level=logging.WARNING)
else:
_ConfigureLogging(level=logging.INFO)
# oauth2client uses info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing
# access tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
if not CERTIFICATE_VALIDATION_ENABLED:
sys.stderr.write(HTTP_WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
_CheckAndWarnForProxyDifferences()
if os.environ.get('_ARGCOMPLETE', '0') == '1':
return _PerformTabCompletion(command_runner)
return _RunNamedCommandAndHandleExceptions(
command_runner, command_name, args=args[1:], headers=headers,
debug_level=debug, trace_token=trace_token,
parallel_operations=parallel_operations,
perf_trace_token=perf_trace_token)
finally:
_Cleanup()
def main():
# These modules must be imported after importing gslib.__main__.
import gslib.command
import gslib.util
from gslib.third_party.oauth2_plugin import oauth2_client
from gslib.util import MultiprocessingIsAvailable
if MultiprocessingIsAvailable()[0]:
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.util.InitializeMultiprocessingVariables()
gslib.command.InitializeMultiprocessingVariables()
oauth2_client.InitializeMultiprocessingVariables()
global debug
if not (2, 6) <= sys.version_info[:3] < (3, ):
raise gslib.exception.CommandException(
'gsutil requires python 2.6 or 2.7.')
config_file_list = GetBotoConfigFileList()
command_runner = CommandRunner(config_file_list)
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
# If ca_certificates_file is configured use it; otherwise configure boto to
# use the cert roots distributed with gsutil.
if not boto.config.get_value('Boto', 'ca_certificates_file', None):
disk_certs_file = os.path.abspath(
os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt'))
if not os.path.exists(disk_certs_file):
# If the file is not present on disk, this means the gslib module doesn't
# actually exist on disk anywhere. This can happen if it's being imported
# from a zip file. Unfortunately, we have to copy the certs file to a
# local temp file on disk because the underlying SSL socket requires it
# to be a filesystem path.
certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt')
if not certs_data:
raise gslib.exception.CommandException(
'Certificates file not found. Please reinstall gsutil from scratch'
)
fd, fname = tempfile.mkstemp(suffix='.txt',
prefix='gsutil-cacerts')
f = os.fdopen(fd, 'w')
f.write(certs_data)
f.close()
disk_certs_file = fname
cleanup_files.append(disk_certs_file)
boto.config.set('Boto', 'ca_certificates_file', disk_certs_file)
try:
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', [
'debug', 'detailedDebug', 'version', 'option', 'help',
'header', 'multithreaded', 'quiet'
])
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Passing debug=2 causes boto to include httplib header output.
debug = 2
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == 3:
debug = 4
else:
debug = 3
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
httplib2.debuglevel = debug
if debug > 1:
sys.stderr.write(DEBUG_WARNING)
if debug == 2:
_ConfigureLogging(level=logging.DEBUG)
elif debug > 2:
_ConfigureLogging(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
sys.stderr.write('config_file_list: %s\n' % config_file_list)
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
_ConfigureLogging(level=logging.WARNING)
else:
_ConfigureLogging(level=logging.INFO)
# apiclient and oauth2client use info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing access
# tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
apiclient.discovery.logger.setLevel(logging.WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
# Unset http_proxy environment variable if it's set, because it confuses
# boto. (Proxies should instead be configured via the boto config file.)
if 'http_proxy' in os.environ:
if debug > 1:
sys.stderr.write(
'Unsetting http_proxy environment variable within gsutil run.\n'
)
del os.environ['http_proxy']
return _RunNamedCommandAndHandleExceptions(command_runner,
command_name, args[1:],
headers, debug,
parallel_operations)
finally:
_Cleanup()
def _RunCp(self, args):
command_runner = CommandRunner(GetBotoConfigFileList)
command_runner.RunNamedCommand("cp", args=args, headers=self.headers,
parallel_operations=False, skip_update_check=True,
logging_filters=[self._CpFilter()])
def main():
# Any modules used in initializing multiprocessing variables must be
# imported after importing gslib.__main__.
# pylint: disable=redefined-outer-name,g-import-not-at-top
import gslib.boto_translation
import gslib.command
import gslib.util
from gslib.util import BOTO_IS_SECURE
from gslib.util import CERTIFICATE_VALIDATION_ENABLED
from gcs_oauth2_boto_plugin import oauth2_client
from gslib.util import MultiprocessingIsAvailable
if MultiprocessingIsAvailable()[0]:
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.util.InitializeMultiprocessingVariables()
gslib.command.InitializeMultiprocessingVariables()
gslib.boto_translation.InitializeMultiprocessingVariables()
# This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
# since otherwise we can't call gslib.util.CreateLock.
try:
# pylint: disable=unused-import,g-import-not-at-top
import gcs_oauth2_boto_plugin
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET)
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
except ImportError:
pass
global debug
global test_exception_traces
if not (2, 6) <= sys.version_info[:3] < (3,):
raise gslib.exception.CommandException(
'gsutil requires python 2.6 or 2.7.')
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
# API. However, we still store gsutil configuration data in the .boto
# config file for compatibility with previous versions and user convenience.
# Many users have a .boto configuration file from previous versions, and it
# is useful to have all of the configuration for gsutil stored in one place.
command_runner = CommandRunner()
if not BOTO_IS_SECURE:
raise CommandException('\n'.join(textwrap.wrap(
'Your boto configuration has is_secure = False. Gsutil cannot be '
'run this way, for security reasons.')))
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
test_exception_traces = False
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
GetCertsFile()
try:
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
['debug', 'detailedDebug', 'version', 'option',
'help', 'header', 'multithreaded', 'quiet',
'testexceptiontraces'])
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Passing debug=2 causes boto to include httplib header output.
debug = 3
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == 3:
debug = 4
else:
debug = 3
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
test_exception_traces = True
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
httplib2.debuglevel = debug
if debug > 1:
sys.stderr.write(DEBUG_WARNING)
if debug >= 2:
_ConfigureLogging(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
for i in xrange(len(config_items)):
config_item_key = config_items[i][0]
if config_item_key in CONFIG_KEYS_TO_REDACT:
config_items[i] = (config_item_key, 'REDACTED')
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList())
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
_ConfigureLogging(level=logging.WARNING)
else:
_ConfigureLogging(level=logging.INFO)
# oauth2client uses info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing
# access tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
if not CERTIFICATE_VALIDATION_ENABLED:
sys.stderr.write(HTTP_WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
# Unset http_proxy environment variable if it's set, because it confuses
# boto. (Proxies should instead be configured via the boto config file.)
if 'http_proxy' in os.environ:
if debug > 1:
sys.stderr.write(
'Unsetting http_proxy environment variable within gsutil run.\n')
del os.environ['http_proxy']
if os.environ.get('_ARGCOMPLETE', '0') == '1':
return _PerformTabCompletion(command_runner)
return _RunNamedCommandAndHandleExceptions(
command_runner, command_name, args=args[1:], headers=headers,
debug_level=debug, parallel_operations=parallel_operations)
finally:
_Cleanup()
def main():
global debug
if not (2, 6) <= sys.version_info[:3] < (3,):
raise gslib.exception.CommandException(
'gsutil requires python 2.6 or 2.7.')
# Load the gsutil version number and append it to boto.UserAgent so the value
# is set before anything instantiates boto. (If parts of boto were
# instantiated first those parts would have the old value of boto.UserAgent,
# so we wouldn't be guaranteed that all code paths send the correct user
# agent.)
boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform)
config_file_list = GetBotoConfigFileList()
command_runner = CommandRunner(config_file_list)
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
# If ca_certificates_file is configured use it; otherwise configure boto to
# use the cert roots distributed with gsutil.
if not boto.config.get_value('Boto', 'ca_certificates_file', None):
boto.config.set('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mq',
['debug', 'detailedDebug', 'version', 'help',
'header', 'multithreaded', 'quiet'])
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Passing debug=2 causes boto to include httplib header output.
debug = 2
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == 3:
debug = 4
else:
debug = 3
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, unused_ptn, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
httplib2.debuglevel = debug
if debug > 1:
sys.stderr.write(
'***************************** WARNING *****************************\n'
'*** You are running gsutil with debug output enabled.\n'
'*** Be aware that debug output includes authentication '
'credentials.\n'
'*** Do not share (e.g., post to support forums) debug output\n'
'*** unless you have sanitized authentication tokens in the\n'
'*** output, or have revoked your credentials.\n'
'***************************** WARNING *****************************\n')
if debug == 2:
logging.basicConfig(level=logging.DEBUG)
elif debug > 2:
logging.basicConfig(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
sys.stderr.write('config_file_list: %s\n' % config_file_list)
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
logging.basicConfig(level=logging.WARNING)
else:
logging.basicConfig(level=logging.INFO)
# apiclient and oauth2client use info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing access
# tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
apiclient.discovery.logger.setLevel(logging.WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
# Unset http_proxy environment variable if it's set, because it confuses
# boto. (Proxies should instead be configured via the boto config file.)
if 'http_proxy' in os.environ:
if debug > 1:
sys.stderr.write(
'Unsetting http_proxy environment variable within gsutil run.\n')
del os.environ['http_proxy']
return _RunNamedCommandAndHandleExceptions(command_runner, command_name,
args[1:], headers, debug,
parallel_operations)
