def RunCommand(self):
"""Command entry point for the version command."""
long_form = False
if self.sub_opts:
for o, _ in self.sub_opts:
if o == '-l':
long_form = True
config_path = GetConfigFilePath()
shipped_checksum = gslib.CHECKSUM
try:
cur_checksum = self._ComputeCodeChecksum()
except IOError:
cur_checksum = 'MISSING FILES'
if shipped_checksum == cur_checksum:
checksum_ok_str = 'OK'
else:
checksum_ok_str = '!= %s' % shipped_checksum
sys.stdout.write('gsutil version: %s\n' % gslib.VERSION)
if long_form:
long_form_output = (
'checksum: {checksum} ({checksum_ok})\n'
'boto version: {boto_version}\n'
'python version: {python_version}\n'
'OS: {os_version}\n'
'multiprocessing available: {multiprocessing_available}\n'
'using cloud sdk: {cloud_sdk}\n'
'config path: {config_path}\n'
'gsutil path: {gsutil_path}\n'
'compiled crcmod: {compiled_crcmod}\n'
'installed via package manager: {is_package_install}\n'
'editable install: {is_editable_install}\n')
sys.stdout.write(
long_form_output.format(
checksum=cur_checksum,
checksum_ok=checksum_ok_str,
boto_version=boto.__version__,
python_version=sys.version.replace('\n', ''),
os_version='%s %s' %
(platform.system(), platform.release()),
multiprocessing_available=MultiprocessingIsAvailable()[0],
cloud_sdk=(os.environ.get('CLOUDSDK_WRAPPER') == '1'),
config_path=config_path,
gsutil_path=gslib.GSUTIL_PATH,
compiled_crcmod=UsingCrcmodExtension(crcmod),
is_package_install=gslib.IS_PACKAGE_INSTALL,
is_editable_install=gslib.IS_EDITABLE_INSTALL,
))
return 0
def __init__(self, do_parallel):
self.bucket_storage_uri_class = BucketStorageUri
support_map = {'gs': ['JSON'], 's3': ['XML']}
default_map = {'gs': 'JSON', 's3': 'XML'}
self.gsutil_api_map = cs_api_map.GsutilApiMapFactory.GetApiMap(
cs_api_map.GsutilApiClassMapFactory, support_map, default_map)
self.logger = CreateGsutilLogger('FakeCommand')
self.parallel_operations = do_parallel
self.failure_count = 0
self.multiprocessing_is_available = MultiprocessingIsAvailable()[0]
self.debug = 0
def main():
InitializeSignalHandling()
# Any modules used in initializing multiprocessing variables must be
# imported after importing gslib.__main__.
# pylint: disable=redefined-outer-name,g-import-not-at-top
import gslib.boto_translation
import gslib.command
import gslib.util
from gslib.util import BOTO_IS_SECURE
from gslib.util import CERTIFICATE_VALIDATION_ENABLED
# pylint: disable=unused-variable
from gcs_oauth2_boto_plugin import oauth2_client
# pylint: enable=unused-variable
from gslib.util import MultiprocessingIsAvailable
if MultiprocessingIsAvailable()[0]:
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.util.InitializeMultiprocessingVariables()
gslib.command.InitializeMultiprocessingVariables()
gslib.boto_translation.InitializeMultiprocessingVariables()
# This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
# since otherwise we can't call gslib.util.CreateLock.
try:
# pylint: disable=unused-import,g-import-not-at-top
import gcs_oauth2_boto_plugin
gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET)
gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
except ImportError:
pass
global debug
global test_exception_traces
if not (2, 6) <= sys.version_info[:3] < (3,):
raise gslib.exception.CommandException(
'gsutil requires python 2.6 or 2.7.')
# In gsutil 4.0 and beyond, we don't use the boto library for the JSON
# API. However, we still store gsutil configuration data in the .boto
# config file for compatibility with previous versions and user convenience.
# Many users have a .boto configuration file from previous versions, and it
# is useful to have all of the configuration for gsutil stored in one place.
command_runner = CommandRunner()
if not BOTO_IS_SECURE:
raise CommandException('\n'.join(textwrap.wrap(
'Your boto configuration has is_secure = False. Gsutil cannot be '
'run this way, for security reasons.')))
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
test_exception_traces = False
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
gslib.util.certs_file_lock = CreateLock()
for signal_num in GetCaughtSignals():
RegisterSignalHandler(signal_num, _CleanupSignalHandler)
GetCertsFile()
try:
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
['debug', 'detailedDebug', 'version', 'option',
'help', 'header', 'multithreaded', 'quiet',
'testexceptiontraces'])
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Passing debug=2 causes boto to include httplib header output.
debug = 3
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == 3:
debug = 4
else:
debug = 3
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
elif o == '--testexceptiontraces': # Hidden flag for integration tests.
test_exception_traces = True
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
httplib2.debuglevel = debug
if debug > 1:
sys.stderr.write(DEBUG_WARNING)
if debug >= 2:
_ConfigureLogging(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
for i in xrange(len(config_items)):
config_item_key = config_items[i][0]
if config_item_key in CONFIG_KEYS_TO_REDACT:
config_items[i] = (config_item_key, 'REDACTED')
sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList())
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
_ConfigureLogging(level=logging.WARNING)
else:
_ConfigureLogging(level=logging.INFO)
# oauth2client uses info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing
# access tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
if not CERTIFICATE_VALIDATION_ENABLED:
sys.stderr.write(HTTP_WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
_CheckAndWarnForProxyDifferences()
if os.environ.get('_ARGCOMPLETE', '0') == '1':
return _PerformTabCompletion(command_runner)
return _RunNamedCommandAndHandleExceptions(
command_runner, command_name, args=args[1:], headers=headers,
debug_level=debug, parallel_operations=parallel_operations)
finally:
_Cleanup()
def __init__(self, do_parallel):
self.logger = CreateGsutilLogger('FakeCommand')
self.parallel_operations = do_parallel
self.failure_count = 0
self.multiprocessing_is_available = MultiprocessingIsAvailable()[0]
def RunNamedCommand(self, command_name, args=None, headers=None, debug=0,
parallel_operations=False, test_method=None,
skip_update_check=False, logging_filters=None,
do_shutdown=True):
"""Runs the named command.
Used by gsutil main, commands built atop other commands, and tests.
Args:
command_name: The name of the command being run.
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
parallel_operations: Should command operations be executed in parallel?
test_method: Optional general purpose method for testing purposes.
Application and semantics of this method will vary by
command and test type.
skip_update_check: Set to True to disable checking for gsutil updates.
logging_filters: Optional list of logging.Filters to apply to this
command's logger.
do_shutdown: Stop all parallelism framework workers iff this is True.
Raises:
CommandException: if errors encountered.
Returns:
Return value(s) from Command that was run.
"""
if (not skip_update_check and
self.MaybeCheckForAndOfferSoftwareUpdate(command_name, debug)):
command_name = 'update'
args = ['-n']
if not args:
args = []
# Include api_version header in all commands.
api_version = boto.config.get_value('GSUtil', 'default_api_version', '1')
if not headers:
headers = {}
headers['x-goog-api-version'] = api_version
if command_name not in self.command_map:
close_matches = difflib.get_close_matches(
command_name, self.command_map.keys(), n=1)
if close_matches:
# Instead of suggesting a deprecated command alias, suggest the new
# name for that command.
translated_command_name = (
OLD_ALIAS_MAP.get(close_matches[0], close_matches)[0])
print >> sys.stderr, 'Did you mean this?'
print >> sys.stderr, '\t%s' % translated_command_name
raise CommandException('Invalid command "%s".' % command_name)
if '--help' in args:
new_args = [command_name]
original_command_class = self.command_map[command_name]
subcommands = original_command_class.help_spec.subcommand_help_text.keys()
for arg in args:
if arg in subcommands:
new_args.append(arg)
break # Take the first match and throw away the rest.
args = new_args
command_name = 'help'
args = HandleArgCoding(args)
command_class = self.command_map[command_name]
command_inst = command_class(
self, args, headers, debug, parallel_operations,
self.bucket_storage_uri_class, self.gsutil_api_class_map_factory,
test_method, logging_filters, command_alias_used=command_name)
return_code = command_inst.RunCommand()
if MultiprocessingIsAvailable()[0] and do_shutdown:
ShutDownGsutil()
if GetFailureCount() > 0:
return_code = 1
return return_code
def main():
# These modules must be imported after importing gslib.__main__.
import gslib.command
import gslib.util
from gslib.third_party.oauth2_plugin import oauth2_client
from gslib.util import MultiprocessingIsAvailable
if MultiprocessingIsAvailable()[0]:
# These setup methods must be called, and, on Windows, they can only be
# called from within an "if __name__ == '__main__':" block.
gslib.util.InitializeMultiprocessingVariables()
gslib.command.InitializeMultiprocessingVariables()
oauth2_client.InitializeMultiprocessingVariables()
global debug
if not (2, 6) <= sys.version_info[:3] < (3, ):
raise gslib.exception.CommandException(
'gsutil requires python 2.6 or 2.7.')
config_file_list = GetBotoConfigFileList()
command_runner = CommandRunner(config_file_list)
headers = {}
parallel_operations = False
quiet = False
version = False
debug = 0
# If user enters no commands just print the usage info.
if len(sys.argv) == 1:
sys.argv.append('help')
# Change the default of the 'https_validate_certificates' boto option to
# True (it is currently False in boto).
if not boto.config.has_option('Boto', 'https_validate_certificates'):
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
boto.config.setbool('Boto', 'https_validate_certificates', True)
# If ca_certificates_file is configured use it; otherwise configure boto to
# use the cert roots distributed with gsutil.
if not boto.config.get_value('Boto', 'ca_certificates_file', None):
disk_certs_file = os.path.abspath(
os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt'))
if not os.path.exists(disk_certs_file):
# If the file is not present on disk, this means the gslib module doesn't
# actually exist on disk anywhere. This can happen if it's being imported
# from a zip file. Unfortunately, we have to copy the certs file to a
# local temp file on disk because the underlying SSL socket requires it
# to be a filesystem path.
certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt')
if not certs_data:
raise gslib.exception.CommandException(
'Certificates file not found. Please reinstall gsutil from scratch'
)
fd, fname = tempfile.mkstemp(suffix='.txt',
prefix='gsutil-cacerts')
f = os.fdopen(fd, 'w')
f.write(certs_data)
f.close()
disk_certs_file = fname
cleanup_files.append(disk_certs_file)
boto.config.set('Boto', 'ca_certificates_file', disk_certs_file)
try:
try:
opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', [
'debug', 'detailedDebug', 'version', 'option', 'help',
'header', 'multithreaded', 'quiet'
])
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
for o, a in opts:
if o in ('-d', '--debug'):
# Passing debug=2 causes boto to include httplib header output.
debug = 2
elif o in ('-D', '--detailedDebug'):
# We use debug level 3 to ask gsutil code to output more detailed
# debug output. This is a bit of a hack since it overloads the same
# flag that was originally implemented for boto use. And we use -DD
# to ask for really detailed debugging (i.e., including HTTP payload).
if debug == 3:
debug = 4
else:
debug = 3
elif o in ('-?', '--help'):
_OutputUsageAndExit(command_runner)
elif o in ('-h', '--header'):
(hdr_name, _, hdr_val) = a.partition(':')
if not hdr_name:
_OutputUsageAndExit(command_runner)
headers[hdr_name.lower()] = hdr_val
elif o in ('-m', '--multithreaded'):
parallel_operations = True
elif o in ('-q', '--quiet'):
quiet = True
elif o in ('-v', '--version'):
version = True
elif o in ('-o', '--option'):
(opt_section_name, _, opt_value) = a.partition('=')
if not opt_section_name:
_OutputUsageAndExit(command_runner)
(opt_section, _, opt_name) = opt_section_name.partition(':')
if not opt_section or not opt_name:
_OutputUsageAndExit(command_runner)
if not boto.config.has_section(opt_section):
boto.config.add_section(opt_section)
boto.config.set(opt_section, opt_name, opt_value)
httplib2.debuglevel = debug
if debug > 1:
sys.stderr.write(DEBUG_WARNING)
if debug == 2:
_ConfigureLogging(level=logging.DEBUG)
elif debug > 2:
_ConfigureLogging(level=logging.DEBUG)
command_runner.RunNamedCommand('ver', ['-l'])
config_items = []
try:
config_items.extend(boto.config.items('Boto'))
config_items.extend(boto.config.items('GSUtil'))
except ConfigParser.NoSectionError:
pass
sys.stderr.write('config_file_list: %s\n' % config_file_list)
sys.stderr.write('config: %s\n' % str(config_items))
elif quiet:
_ConfigureLogging(level=logging.WARNING)
else:
_ConfigureLogging(level=logging.INFO)
# apiclient and oauth2client use info logging in places that would better
# correspond to gsutil's debug logging (e.g., when refreshing access
# tokens).
oauth2client.client.logger.setLevel(logging.WARNING)
apiclient.discovery.logger.setLevel(logging.WARNING)
if version:
command_name = 'version'
elif not args:
command_name = 'help'
else:
command_name = args[0]
# Unset http_proxy environment variable if it's set, because it confuses
# boto. (Proxies should instead be configured via the boto config file.)
if 'http_proxy' in os.environ:
if debug > 1:
sys.stderr.write(
'Unsetting http_proxy environment variable within gsutil run.\n'
)
del os.environ['http_proxy']
return _RunNamedCommandAndHandleExceptions(command_runner,
command_name, args[1:],
headers, debug,
parallel_operations)
finally:
_Cleanup()
