def RunCommand(self):
"""Command entry point for the version command."""
long_form = False
if self.sub_opts:
for o, _ in self.sub_opts:
if o == '-l':
long_form = True
if GetConfigFilePaths():
config_paths = ', '.join(GetConfigFilePaths())
else:
config_paths = 'no config found'
shipped_checksum = gslib.CHECKSUM
try:
cur_checksum = self._ComputeCodeChecksum()
except IOError:
cur_checksum = 'MISSING FILES'
if shipped_checksum == cur_checksum:
checksum_ok_str = 'OK'
else:
checksum_ok_str = '!= %s' % shipped_checksum
sys.stdout.write('gsutil version: %s\n' % gslib.VERSION)
if long_form:
long_form_output = (
'checksum: {checksum} ({checksum_ok})\n'
'boto version: {boto_version}\n'
'python version: {python_version}\n'
'OS: {os_version}\n'
'multiprocessing available: {multiprocessing_available}\n'
'using cloud sdk: {cloud_sdk}\n'
'config path(s): {config_paths}\n'
'gsutil path: {gsutil_path}\n'
'compiled crcmod: {compiled_crcmod}\n'
'installed via package manager: {is_package_install}\n'
'editable install: {is_editable_install}\n')
sys.stdout.write(
long_form_output.format(
checksum=cur_checksum,
checksum_ok=checksum_ok_str,
boto_version=boto.__version__,
python_version=sys.version.replace('\n', ''),
os_version='%s %s' %
(platform.system(), platform.release()),
multiprocessing_available=(
CheckMultiprocessingAvailableAndInit().is_available),
cloud_sdk=(os.environ.get('CLOUDSDK_WRAPPER') == '1'),
config_paths=config_paths,
gsutil_path=gslib.GSUTIL_PATH,
compiled_crcmod=UsingCrcmodExtension(crcmod),
is_package_install=gslib.IS_PACKAGE_INSTALL,
is_editable_install=gslib.IS_EDITABLE_INSTALL,
))
return 0
def __init__(self, do_parallel):
self.bucket_storage_uri_class = BucketStorageUri
support_map = {'gs': ['JSON'], 's3': ['XML']}
default_map = {'gs': 'JSON', 's3': 'XML'}
self.gsutil_api_map = cs_api_map.GsutilApiMapFactory.GetApiMap(
cs_api_map.GsutilApiClassMapFactory, support_map, default_map)
self.logger = CreateGsutilLogger('FakeCommand')
self.parallel_operations = do_parallel
self.failure_count = 0
self.multiprocessing_is_available = (
CheckMultiprocessingAvailableAndInit().is_available)
self.debug = 0
def main():
InitializeSignalHandling()
# Any modules used in initializing multiprocessing variables must be
# imported after importing gslib.__main__.
# pylint: disable=redefined-outer-name,g-import-not-at-top
import gslib.boto_translation
import gslib.command
import gslib.util
from gslib.util import BOTO_IS_SECURE
from gslib.util import CERTIFICATE_VALIDATION_ENABLED
# pylint: disable=unused-variable
from gcs_oauth2_boto_plugin import oauth2_client
from apitools.base.py import credentials_lib
# pylint: enable=unused-variable
from gslib.util import CheckMultiprocessingAvailableAndInit
if CheckMultiprocessingAvailableAndInit().is_available:
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.command.InitializeMultiprocessingVariables()
gslib.boto_translation.InitializeMultiprocessingVariables()
else:
gslib.command.InitializeThreadingVariables()
# This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
# since otherwise we can't call gslib.util.CreateLock.
try:
# pylint: disable=unused-import,g-import-not-at-top
import gcs_oauth2_boto_plugin
gsutil_client_id, gsutil_client_secret = GetGsutilClientIdAndSecret()
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
gsutil_client_id, gsutil_client_secret)
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
credentials_lib.SetCredentialsCacheFileLock(CreateLock())
except ImportError:
pass
global debug
global test_exception_traces
if not (2, 7) <= sys.version_info[:3] < (3,):
raise CommandException('gsutil requires python 2.7.')
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
# API. However, we still store gsutil configuration data in the .boto
# config file for compatibility with previous versions and user convenience.
# Many users have a .boto configuration file from previous versions, and it
# is useful to have all of the configuration for gsutil stored in one place.
command_runner = CommandRunner()
if not BOTO_IS_SECURE:
raise CommandException('\n'.join(textwrap.wrap(
'Your boto configuration has is_secure = False. Gsutil cannot be '
'run this way, for security reasons.')))
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
trace_token = None
perf_trace_token = None
test_exception_traces = False
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
gslib.util.configured_certs_file = gslib.util.ConfigureCertsFile()
for signal_num in GetCaughtSignals():
RegisterSignalHandler(signal_num, _CleanupSignalHandler)
GetCertsFile()
try:
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
['debug', 'detailedDebug', 'version', 'option',
'help', 'header', 'multithreaded', 'quiet',
'testexceptiontraces', 'trace-token=',
'perf-trace-token='])
except getopt.GetoptError as e:
_HandleCommandException(CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Also causes boto to include httplib header output.
debug = DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == DEBUGLEVEL_DUMP_REQUESTS:
debug = DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS
else:
debug = DEBUGLEVEL_DUMP_REQUESTS
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
elif o == '--perf-trace-token':
perf_trace_token = a
elif o == '--trace-token':
trace_token = a
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
test_exception_traces = True
# Avoid printing extra warnings to stderr regarding long retries by
# setting the threshold very high.
gslib.util.LONG_RETRY_WARN_SEC = 3600
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
metrics.LogCommandParams(global_opts=opts)
httplib2.debuglevel = debug
if trace_token:
sys.stderr.write(TRACE_WARNING)
if debug >= DEBUGLEVEL_DUMP_REQUESTS:
sys.stderr.write(DEBUG_WARNING)
_ConfigureLogging(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
for i in xrange(len(config_items)):
config_item_key = config_items[i][0]
if config_item_key in CONFIG_KEYS_TO_REDACT:
config_items[i] = (config_item_key, 'REDACTED')
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList())
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
_ConfigureLogging(level=logging.WARNING)
else:
_ConfigureLogging(level=logging.INFO)
# oauth2client uses info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing
# access tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
if not CERTIFICATE_VALIDATION_ENABLED:
sys.stderr.write(HTTP_WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
_CheckAndWarnForProxyDifferences()
if os.environ.get('_ARGCOMPLETE', '0') == '1':
return _PerformTabCompletion(command_runner)
return _RunNamedCommandAndHandleExceptions(
command_runner, command_name, args=args[1:], headers=headers,
debug_level=debug, trace_token=trace_token,
parallel_operations=parallel_operations,
perf_trace_token=perf_trace_token)
finally:
_Cleanup()
def RunNamedCommand(self,
command_name,
args=None,
headers=None,
debug=0,
trace_token=None,
parallel_operations=False,
skip_update_check=False,
logging_filters=None,
do_shutdown=True):
"""Runs the named command.
Used by gsutil main, commands built atop other commands, and tests.
Args:
command_name: The name of the command being run.
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
trace_token: Trace token to pass to the underlying API.
parallel_operations: Should command operations be executed in parallel?
skip_update_check: Set to True to disable checking for gsutil updates.
logging_filters: Optional list of logging.Filters to apply to this
command's logger.
do_shutdown: Stop all parallelism framework workers iff this is True.
Raises:
CommandException: if errors encountered.
Returns:
Return value(s) from Command that was run.
"""
command_changed_to_update = False
if (not skip_update_check and self.MaybeCheckForAndOfferSoftwareUpdate(
command_name, debug)):
command_name = 'update'
command_changed_to_update = True
args = ['-n']
if not args:
args = []
# Include api_version header in all commands.
api_version = boto.config.get_value('GSUtil', 'default_api_version',
'1')
if not headers:
headers = {}
headers['x-goog-api-version'] = api_version
if command_name not in self.command_map:
close_matches = difflib.get_close_matches(command_name,
self.command_map.keys(),
n=1)
if close_matches:
# Instead of suggesting a deprecated command alias, suggest the new
# name for that command.
translated_command_name = (OLD_ALIAS_MAP.get(
close_matches[0], close_matches)[0])
print >> sys.stderr, 'Did you mean this?'
print >> sys.stderr, '\t%s' % translated_command_name
elif command_name == 'update' and gslib.IS_PACKAGE_INSTALL:
sys.stderr.write(
'Update command is not supported for package installs; '
'please instead update using your package manager.')
raise CommandException('Invalid command "%s".' % command_name)
if '--help' in args:
new_args = [command_name]
original_command_class = self.command_map[command_name]
subcommands = original_command_class.help_spec.subcommand_help_text.keys(
)
for arg in args:
if arg in subcommands:
new_args.append(arg)
break # Take the first match and throw away the rest.
args = new_args
command_name = 'help'
args = HandleArgCoding(args)
command_class = self.command_map[command_name]
command_inst = command_class(self,
args,
headers,
debug,
trace_token,
parallel_operations,
self.bucket_storage_uri_class,
self.gsutil_api_class_map_factory,
logging_filters,
command_alias_used=command_name)
return_code = command_inst.RunCommand()
if CheckMultiprocessingAvailableAndInit().is_available and do_shutdown:
ShutDownGsutil()
if GetFailureCount() > 0:
return_code = 1
if command_changed_to_update:
# If the command changed to update, the user's original command was
# not executed.
return_code = 1
print '\n'.join(
textwrap.wrap(
'Update was successful. Exiting with code 1 as the original command '
'issued prior to the update was not executed and should be re-run.'
))
return return_code