def RunCommand(self):
"""Command entry point for the version command."""
long_form = False
if self.sub_opts:
for o, _ in self.sub_opts:
if o == '-l':
long_form = True
config_path = GetConfigFilePath()
shipped_checksum = gslib.CHECKSUM
try:
cur_checksum = self._ComputeCodeChecksum()
except IOError:
cur_checksum = 'MISSING FILES'
if shipped_checksum == cur_checksum:
checksum_ok_str = 'OK'
else:
checksum_ok_str = '!= %s' % shipped_checksum
sys.stdout.write('gsutil version: %s\n' % gslib.VERSION)
if long_form:
long_form_output = (
'checksum: {checksum} ({checksum_ok})\n'
'boto version: {boto_version}\n'
'python version: {python_version}\n'
'OS: {os_version}\n'
'multiprocessing available: {multiprocessing_available}\n'
'using cloud sdk: {cloud_sdk}\n'
'config path: {config_path}\n'
'gsutil path: {gsutil_path}\n'
'compiled crcmod: {compiled_crcmod}\n'
'installed via package manager: {is_package_install}\n'
'editable install: {is_editable_install}\n')
sys.stdout.write(
long_form_output.format(
checksum=cur_checksum,
checksum_ok=checksum_ok_str,
boto_version=boto.__version__,
python_version=sys.version.replace('\n', ''),
os_version='%s %s' %
(platform.system(), platform.release()),
multiprocessing_available=(
CheckMultiprocessingAvailableAndInit().is_available),
cloud_sdk=(os.environ.get('CLOUDSDK_WRAPPER') == '1'),
config_path=config_path,
gsutil_path=gslib.GSUTIL_PATH,
compiled_crcmod=UsingCrcmodExtension(crcmod),
is_package_install=gslib.IS_PACKAGE_INSTALL,
is_editable_install=gslib.IS_EDITABLE_INSTALL,
))
return 0
def RunCommand(self):
long_form = False
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-l':
long_form = True
config_path = GetConfigFilePath()
shipped_checksum = gslib.CHECKSUM
try:
cur_checksum = self._ComputeCodeChecksum()
except IOError:
cur_checksum = 'MISSING FILES'
if shipped_checksum == cur_checksum:
checksum_ok_str = 'OK'
else:
checksum_ok_str = '!= %s' % shipped_checksum
sys.stdout.write('gsutil version %s\n' % gslib.VERSION)
if long_form:
long_form_output = (
'checksum {checksum} ({checksum_ok})\n'
'boto version {boto_version}\n'
'python version {python_version}\n'
'config path: {config_path}\n'
'gsutil path: {gsutil_path}\n'
'compiled crcmod: {compiled_crcmod}\n'
'installed via package manager: {is_package_install}\n'
'editable install: {is_editable_install}\n')
sys.stdout.write(
long_form_output.format(
checksum=cur_checksum,
checksum_ok=checksum_ok_str,
boto_version=boto.__version__,
python_version=sys.version,
config_path=config_path,
gsutil_path=gslib.GSUTIL_PATH,
compiled_crcmod=UsingCrcmodExtension(crcmod),
is_package_install=gslib.IS_PACKAGE_INSTALL,
is_editable_install=gslib.IS_EDITABLE_INSTALL,
))
return 0
def _RunNamedCommandAndHandleExceptions(
command_runner, command_name, args=None, headers=None, debug_level=0,
trace_token=None, parallel_operations=False, perf_trace_token=None):
"""Runs the command and handles common exceptions."""
# pylint: disable=g-import-not-at-top
from gslib.util import GetConfigFilePath
from gslib.util import IS_WINDOWS
from gslib.util import IsRunningInteractively
try:
# Catch ^C so we can print a brief message instead of the normal Python
# stack trace. Register as a final signal handler because this handler kills
# the main gsutil process (so it must run last).
RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True)
# Catch ^\ so we can force a breakpoint in a running gsutil.
if not IS_WINDOWS:
RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit)
return command_runner.RunNamedCommand(
command_name, args, headers, debug_level, trace_token,
parallel_operations, perf_trace_token=perf_trace_token)
except AttributeError as e:
if str(e).find('secret_access_key') != -1:
_OutputAndExit('Missing credentials for the given URI(s). Does your '
'boto config file contain all needed credentials?')
else:
_OutputAndExit(str(e))
except gslib.exception.CommandException as e:
_HandleCommandException(e)
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
except boto.exception.InvalidUriError as e:
_OutputAndExit('InvalidUriError: %s.' % e.message)
except gslib.exception.InvalidUrlError as e:
_OutputAndExit('InvalidUrlError: %s.' % e.message)
except boto.auth_handler.NotReadyToAuthenticate:
_OutputAndExit('NotReadyToAuthenticate')
except OSError as e:
_OutputAndExit('OSError: %s.' % e.strerror)
except IOError as e:
if (e.errno == errno.EPIPE or (IS_WINDOWS and e.errno == errno.EINVAL)
and not IsRunningInteractively()):
# If we get a pipe error, this just means that the pipe to stdout or
# stderr is broken. This can happen if the user pipes gsutil to a command
# that doesn't use the entire output stream. Instead of raising an error,
# just swallow it up and exit cleanly.
sys.exit(0)
else:
raise
except wildcard_iterator.WildcardException as e:
_OutputAndExit(e.reason)
except ProjectIdException as e:
_OutputAndExit(
'You are attempting to perform an operation that requires a '
'project id, with none configured. Please re-run '
'gsutil config and make sure to follow the instructions for '
'finding and entering your default project id.')
except BadRequestException as e:
if e.reason == 'MissingSecurityHeader':
_CheckAndHandleCredentialException(e, args)
_OutputAndExit(e)
except AccessDeniedException as e:
_CheckAndHandleCredentialException(e, args)
_OutputAndExit(e)
except ArgumentException as e:
_OutputAndExit(e)
except ServiceException as e:
_OutputAndExit(e)
except apitools_exceptions.HttpError as e:
# These should usually be retried by the underlying implementation or
# wrapped by CloudApi ServiceExceptions, but if we do get them,
# print something useful.
_OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''),
e.content or ''))
except socket.error as e:
if e.args[0] == errno.EPIPE:
# Retrying with a smaller file (per suggestion below) works because
# the library code send loop (in boto/s3/key.py) can get through the
# entire file and then request the HTTP response before the socket
# gets closed and the response lost.
_OutputAndExit(
'Got a "Broken pipe" error. This can happen to clients using Python '
'2.x, when the server sends an error response and then closes the '
'socket (see http://bugs.python.org/issue5542). If you are trying to '
'upload a large object you might retry with a small (say 200k) '
'object, and see if you get a more specific error code.'
)
elif e.args[0] == errno.ECONNRESET and ' '.join(args).contains('s3://'):
_OutputAndExit('\n'.join(textwrap.wrap(
'Got a "Connection reset by peer" error. One way this can happen is '
'when copying data to/from an S3 regional bucket. If you are using a '
'regional S3 bucket you could try re-running this command using the '
'regional S3 endpoint, for example '
's3://s3-<region>.amazonaws.com/your-bucket. For details about this '
'problem see https://github.com/boto/boto/issues/2207')))
else:
_HandleUnknownFailure(e)
except Exception as e: # pylint: disable=broad-except
# Check for two types of errors related to service accounts. These errors
# appear to be the same except for their messages, but they are caused by
# different problems and both have unhelpful error messages. Moreover,
# the error type belongs to PyOpenSSL, which is not necessarily installed.
if 'mac verify failure' in str(e):
_OutputAndExit(
'Encountered an error while refreshing access token. '
'If you are using a service account,\nplease verify that the '
'gs_service_key_file_password field in your config file,'
'\n%s, is correct.' % GetConfigFilePath())
elif 'asn1 encoding routines' in str(e):
_OutputAndExit(
'Encountered an error while refreshing access token. '
'If you are using a service account,\nplease verify that the '
'gs_service_key_file field in your config file,\n%s, is correct.'
% GetConfigFilePath())
_HandleUnknownFailure(e)
def _RunNamedCommandAndHandleExceptions(command_runner,
command_name,
args=None,
headers=None,
debug=0,
parallel_operations=False):
try:
# Catch ^C so we can print a brief message instead of the normal Python
# stack trace.
signal.signal(signal.SIGINT, _HandleControlC)
# Catch ^\ so we can force a breakpoint in a running gsutil.
if not util.IS_WINDOWS:
signal.signal(signal.SIGQUIT, _HandleSigQuit)
return command_runner.RunNamedCommand(command_name, args, headers,
debug, parallel_operations)
except AttributeError as e:
if str(e).find('secret_access_key') != -1:
_OutputAndExit(
'Missing credentials for the given URI(s). Does your '
'boto config file contain all needed credentials?')
else:
_OutputAndExit(str(e))
except boto.exception.StorageDataError as e:
_OutputAndExit('StorageDataError: %s.' % e.reason)
except boto.exception.BotoClientError as e:
_OutputAndExit('BotoClientError: %s.' % e.reason)
except gslib.exception.CommandException as e:
_HandleCommandException(e)
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
except boto.exception.InvalidAclError as e:
_OutputAndExit('InvalidAclError: %s.' % str(e))
except boto.exception.InvalidUriError as e:
_OutputAndExit('InvalidUriError: %s.' % e.message)
except gslib.exception.ProjectIdException as e:
_OutputAndExit('ProjectIdException: %s.' % e.reason)
except boto.auth_handler.NotReadyToAuthenticate:
_OutputAndExit('NotReadyToAuthenticate')
except OSError as e:
_OutputAndExit('OSError: %s.' % e.strerror)
except IOError as e:
if e.errno == errno.EPIPE and not IsRunningInteractively():
# If we get a pipe error, this just means that the pipe to stdout or
# stderr is broken. This can happen if the user pipes gsutil to a command
# that doesn't use the entire output stream. Instead of raising an error,
# just swallow it up and exit cleanly.
sys.exit(0)
else:
raise
except wildcard_iterator.WildcardException as e:
_OutputAndExit(e.reason)
except boto.exception.StorageResponseError as e:
# Check for access denied, and provide detail to users who have no boto
# config file (who might previously have been using gsutil only for
# accessing publicly readable buckets and objects).
if (e.status == 403
or (e.status == 400 and e.code == 'MissingSecurityHeader')):
_, _, detail = util.ParseErrorDetail(e)
if detail and detail.find(
'x-goog-project-id header is required') != -1:
_OutputAndExit('\n'.join(
textwrap.wrap(
'You are attempting to perform an operation that requires an '
'x-goog-project-id header, with none configured. Please re-run '
'gsutil config and make sure to follow the instructions for '
'finding and entering your default project id.')))
if (not HasConfiguredCredentials() and not boto.config.get_value(
'Tests', 'bypass_anonymous_access_warning', False)):
# The check above allows tests to assert that we get a particular,
# expected failure, rather than always encountering this error message
# when there are no configured credentials. This allows tests to
# simulate a second user without permissions, without actually requiring
# two separate configured users.
_OutputAndExit('\n'.join(
textwrap.wrap(
'You are attempting to access protected data with no configured '
'credentials. Please visit '
'https://cloud.google.com/console#/project and sign up for an '
'account, and then run the "gsutil config" command to configure '
'gsutil to use these credentials.')))
elif (e.error_code == 'AccountProblem'
and ','.join(args).find('gs://') != -1):
default_project_id = boto.config.get_value(
'GSUtil', 'default_project_id')
(acct_help_part_1, acct_help_part_2,
acct_help_part_3) = (_ConstructAclHelp(default_project_id))
if default_project_id:
_OutputAndExit(acct_help_part_1 + acct_help_part_2 +
'3. ' + acct_help_part_3)
else:
_OutputAndExit(acct_help_part_1 + '2. ' + acct_help_part_3)
exc_name, message, detail = util.ParseErrorDetail(e)
_OutputAndExit(
util.FormatErrorMessage(exc_name, e.status, e.code, e.reason,
message, detail))
except boto.exception.ResumableUploadException as e:
_OutputAndExit('ResumableUploadException: %s.' % e.message)
except socket.error as e:
if e.args[0] == errno.EPIPE:
# Retrying with a smaller file (per suggestion below) works because
# the library code send loop (in boto/s3/key.py) can get through the
# entire file and then request the HTTP response before the socket
# gets closed and the response lost.
message = ("""
Got a "Broken pipe" error. This can happen to clients using Python 2.x,
when the server sends an error response and then closes the socket (see
http://bugs.python.org/issue5542). If you are trying to upload a large
object you might retry with a small (say 200k) object, and see if you get
a more specific error code.
""")
_OutputAndExit(message)
else:
_HandleUnknownFailure(e)
except Exception as e:
# Check for two types of errors related to service accounts. These errors
# appear to be the same except for their messages, but they are caused by
# different problems and both have unhelpful error messages. Moreover,
# the error type belongs to PyOpenSSL, which is not necessarily installed.
if 'mac verify failure' in str(e):
_OutputAndExit(
"Encountered an error while refreshing access token." +
" If you are using a service account,\nplease verify that the "
+ "gs_service_key_file_password field in your config file," +
"\n%s, is correct." % GetConfigFilePath())
elif 'asn1 encoding routines' in str(e):
_OutputAndExit(
"Encountered an error while refreshing access token." +
" If you are using a service account,\nplease verify that the "
+
"gs_service_key_file field in your config file,\n%s, is correct."
% GetConfigFilePath())
_HandleUnknownFailure(e)
def _RunNamedCommandAndHandleExceptions(command_runner,
command_name,
args=None,
headers=None,
debug=0,
parallel_operations=False):
try:
# Catch ^C so we can print a brief message instead of the normal Python
# stack trace.
signal.signal(signal.SIGINT, _HandleControlC)
# Catch ^\ so we can force a breakpoint in a running gsutil.
if not util.IS_WINDOWS:
signal.signal(signal.SIGQUIT, _HandleSigQuit)
return command_runner.RunNamedCommand(command_name, args, headers,
debug, parallel_operations)
except AttributeError as e:
if str(e).find('secret_access_key') != -1:
_OutputAndExit(
'Missing credentials for the given URI(s). Does your '
'boto config file contain all needed credentials?')
else:
_OutputAndExit(str(e))
except boto.exception.StorageDataError as e:
_OutputAndExit('StorageDataError: %s.' % e.reason)
except boto.exception.BotoClientError as e:
_OutputAndExit('BotoClientError: %s.' % e.reason)
except gslib.exception.CommandException as e:
_HandleCommandException(e)
except getopt.GetoptError as e:
_HandleCommandException(gslib.exception.CommandException(e.msg))
except boto.exception.InvalidAclError as e:
_OutputAndExit('InvalidAclError: %s.' % str(e))
except boto.exception.InvalidUriError as e:
_OutputAndExit('InvalidUriError: %s.' % e.message)
except gslib.exception.ProjectIdException as e:
_OutputAndExit('ProjectIdException: %s.' % e.reason)
except boto.auth_handler.NotReadyToAuthenticate:
_OutputAndExit('NotReadyToAuthenticate')
except OSError as e:
_OutputAndExit('OSError: %s.' % e.strerror)
except wildcard_iterator.WildcardException as e:
_OutputAndExit(e.reason)
except boto.exception.StorageResponseError as e:
# Check for access denied, and provide detail to users who have no boto
# config file (who might previously have been using gsutil only for
# accessing publicly readable buckets and objects).
if (e.status == 403
or (e.status == 400 and e.code == 'MissingSecurityHeader')):
if not util.HasConfiguredCredentials():
_OutputAndExit(
'You are attempting to access protected data with no configured '
'credentials.\nPlease see '
'http://code.google.com/apis/storage/docs/signup.html for\ndetails '
'about activating the Google Cloud Storage service and then run '
'the\n"gsutil config" command to configure gsutil to use these '
'credentials.')
elif (e.error_code == 'AccountProblem'
and ','.join(args).find('gs://') != -1):
default_project_id = boto.config.get_value(
'GSUtil', 'default_project_id')
acct_help_part_1 = (
"""Your request resulted in an AccountProblem (403) error. Usually this happens
if you attempt to create a bucket or upload an object without having first
enabled billing for the project you are using. To remedy this problem, please do
the following:
1. Navigate to the Google APIs console (https://code.google.com/apis/console),
and ensure the drop-down selector beneath "Google APIs" shows the project
you're attempting to use.
""")
acct_help_part_2 = '\n'
if default_project_id:
acct_help_part_2 = (
"""2. Click "Google Cloud Storage" on the left hand pane, and then check that
the value listed for "x-goog-project-id" on this page matches the project ID
(%s) from your boto config file.
""" % default_project_id)
acct_help_part_3 = (
"""Check whether there's an "!" next to Billing. If so, click Billing and then
enable billing for this project. Note that it can take up to one hour after
enabling billing for the project to become activated for creating buckets and
uploading objects.
If the above doesn't resolve your AccountProblem, please send mail to
[email protected] requesting assistance, noting the exact command you ran, the
fact that you received a 403 AccountProblem error, and your project ID. Please
do not post your project ID on StackOverflow.
Note: It's possible to use Google Cloud Storage without enabling billing if
you're only listing or reading objects for which you're authorized, or if
you're uploading objects to a bucket billed to a project that has billing
enabled. But if you're attempting to create buckets or upload objects to a
bucket owned by your own project, you must first enable billing for that
project.""")
if default_project_id:
_OutputAndExit(acct_help_part_1 + acct_help_part_2 +
'3. ' + acct_help_part_3)
else:
_OutputAndExit(acct_help_part_1 + '2. ' + acct_help_part_3)
if not e.body:
e.body = ''
exc_name, error_detail = util.ExtractErrorDetail(e)
if error_detail:
_OutputAndExit(
'%s: status=%d, code=%s, reason=%s, detail=%s.' %
(exc_name, e.status, e.code, e.reason, error_detail))
else:
_OutputAndExit('%s: status=%d, code=%s, reason=%s.' %
(exc_name, e.status, e.code, e.reason))
except boto.exception.ResumableUploadException as e:
_OutputAndExit('ResumableUploadException: %s.' % e.message)
except socket.error as e:
if e.args[0] == errno.EPIPE:
# Retrying with a smaller file (per suggestion below) works because
# the library code send loop (in boto/s3/key.py) can get through the
# entire file and then request the HTTP response before the socket
# gets closed and the response lost.
message = ("""
Got a "Broken pipe" error. This can happen to clients using Python 2.x,
when the server sends an error response and then closes the socket (see
http://bugs.python.org/issue5542). If you are trying to upload a large
object you might retry with a small (say 200k) object, and see if you get
a more specific error code.
""")
_OutputAndExit(message)
else:
_HandleUnknownFailure(e)
except Exception as e:
# Check for two types of errors related to service accounts. These errors
# appear to be the same except for their messages, but they are caused by
# different problems and both have unhelpful error messages. Moreover,
# the error type belongs to PyOpenSSL, which is not necessarily installed.
if 'mac verify failure' in str(e):
_OutputAndExit(
"Encountered an error while refreshing access token." +
" If you are using a service account,\nplease verify that the "
+ "gs_service_key_file_password field in your config file," +
"\n%s, is correct." % GetConfigFilePath())
elif 'asn1 encoding routines' in str(e):
_OutputAndExit(
"Encountered an error while refreshing access token." +
" If you are using a service account,\nplease verify that the "
+
"gs_service_key_file field in your config file,\n%s, is correct."
% GetConfigFilePath())
_HandleUnknownFailure(e)