def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
"""Checks the last time we checked for an update and offers one if needed.
Offer is made if the time since the last update check is longer
than the configured threshold offers the user to update gsutil.
Args:
command_name: The name of the command being run.
debug: Debug level to pass in to boto connection (range 0..3).
Returns:
True if the user decides to update.
"""
# Don't try to interact with user if:
# - gsutil is not connected to a tty (e.g., if being run from cron);
# - user is running gsutil -q
# - user is running the config command (which could otherwise attempt to
# check for an update for a user running behind a proxy, who has not yet
# configured gsutil to go through the proxy; for such users we need the
# first connection attempt to be made by the gsutil config command).
# - user is running the version command (which gets run when using
# gsutil -D, which would prevent users with proxy config problems from
# sending us gsutil -D output).
# - user is running the update command (which could otherwise cause an
# additional note that an update is available when user is already trying
# to perform an update);
# - user specified gs_host (which could be a non-production different
# service instance, in which case credentials won't work for checking
# gsutil tarball).
# - user is using a Cloud SDK install (which should only be updated via
# gcloud components update)
logger = logging.getLogger()
gs_host = boto.config.get('Credentials', 'gs_host', None)
gs_host_is_not_default = (gs_host !=
boto.gs.connection.GSConnection.DefaultHost)
if (not system_util.IsRunningInteractively()
or command_name in ('config', 'update', 'ver', 'version')
or not logger.isEnabledFor(logging.INFO)
or gs_host_is_not_default or system_util.InvokedViaCloudSdk()):
return False
software_update_check_period = boto.config.getint(
'GSUtil', 'software_update_check_period', 30)
# Setting software_update_check_period to 0 means periodic software
# update checking is disabled.
if software_update_check_period == 0:
return False
last_checked_for_gsutil_update_timestamp_file = (
boto_util.GetLastCheckedForGsutilUpdateTimestampFile())
cur_ts = int(time.time())
if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file):
# Set last_checked_ts from date of VERSION file, so if the user installed
# an old copy of gsutil it will get noticed (and an update offered) the
# first time they try to run it.
last_checked_ts = gslib.GetGsutilVersionModifiedTime()
with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
f.write(str(last_checked_ts))
else:
try:
with open(last_checked_for_gsutil_update_timestamp_file,
'r') as f:
last_checked_ts = int(f.readline())
except (TypeError, ValueError):
return False
if (cur_ts - last_checked_ts >
software_update_check_period * SECONDS_PER_DAY):
# Create a credential-less gsutil API to check for the public
# update tarball.
gsutil_api = GcsJsonApi(self.bucket_storage_uri_class,
logger,
DiscardMessagesQueue(),
credentials=NoOpCredentials(),
debug=debug)
cur_ver = LookUpGsutilVersion(gsutil_api, GSUTIL_PUB_TARBALL)
with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f:
f.write(str(cur_ts))
(g, m) = CompareVersions(cur_ver, gslib.VERSION)
if m:
print '\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). NOTE: This is a major new version, so it is '
'strongly recommended that you review the release note details at '
'%s before updating to this version, especially if you use gsutil '
'in scripts.' %
(cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))
if gslib.IS_PACKAGE_INSTALL:
return False
print
answer = raw_input('Would you like to update [y/N]? ')
return answer and answer.lower()[0] == 'y'
elif g:
print '\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). A detailed log of gsutil release changes is '
'available at %s if you would like to read them before updating.'
% (cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))
if gslib.IS_PACKAGE_INSTALL:
return False
print
answer = raw_input('Would you like to update [Y/n]? ')
return not answer or answer.lower()[0] != 'n'
return False
def _RunNamedCommandAndHandleExceptions(command_runner,
command_name,
args=None,
headers=None,
debug_level=0,
trace_token=None,
parallel_operations=False,
perf_trace_token=None,
user_project=None):
"""Runs the command and handles common exceptions."""
# Note that this method is run at the end of main() and thus has access to
# all of the modules imported there.
# pylint: disable=g-import-not-at-top
try:
# Catch ^C so we can print a brief message instead of the normal Python
# stack trace. Register as a final signal handler because this handler kills
# the main gsutil process (so it must run last).
RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True)
# Catch ^\ so we can force a breakpoint in a running gsutil.
if not system_util.IS_WINDOWS:
RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit)
return command_runner.RunNamedCommand(command_name,
args,
headers,
debug_level,
trace_token,
parallel_operations,
perf_trace_token=perf_trace_token,
collect_analytics=True,
user_project=user_project)
except AttributeError as e:
if str(e).find('secret_access_key') != -1:
_OutputAndExit(
'Missing credentials for the given URI(s). Does your '
'boto config file contain all needed credentials?',
exception=e)
else:
_OutputAndExit(message=str(e), exception=e)
except CommandException as e:
_HandleCommandException(e)
except getopt.GetoptError as e:
_HandleCommandException(CommandException(e.msg))
except boto.exception.InvalidUriError as e:
_OutputAndExit(message='InvalidUriError: %s.' % e.message, exception=e)
except gslib.exception.InvalidUrlError as e:
_OutputAndExit(message='InvalidUrlError: %s.' % e.message, exception=e)
except boto.auth_handler.NotReadyToAuthenticate:
_OutputAndExit(message='NotReadyToAuthenticate', exception=e)
except OSError as e:
# In Python 3, IOError (next except) is an alias for OSError
# Sooo... we need the same logic here
if (e.errno == errno.EPIPE or
(system_util.IS_WINDOWS and e.errno == errno.EINVAL) and
not system_util.IsRunningInteractively()):
# If we get a pipe error, this just means that the pipe to stdout or
# stderr is broken. This can happen if the user pipes gsutil to a command
# that doesn't use the entire output stream. Instead of raising an error,
# just swallow it up and exit cleanly.
sys.exit(0)
else:
_OutputAndExit(message='OSError: %s.' % e.strerror, exception=e)
except IOError as e:
if (e.errno == errno.EPIPE or
(system_util.IS_WINDOWS and e.errno == errno.EINVAL) and
not system_util.IsRunningInteractively()):
# If we get a pipe error, this just means that the pipe to stdout or
# stderr is broken. This can happen if the user pipes gsutil to a command
# that doesn't use the entire output stream. Instead of raising an error,
# just swallow it up and exit cleanly.
sys.exit(0)
else:
raise
except wildcard_iterator.WildcardException as e:
_OutputAndExit(message=e.reason, exception=e)
except ProjectIdException as e:
_OutputAndExit(
'You are attempting to perform an operation that requires a '
'project id, with none configured. Please re-run '
'gsutil config and make sure to follow the instructions for '
'finding and entering your default project id.',
exception=e)
except BadRequestException as e:
if e.reason == 'MissingSecurityHeader':
_CheckAndHandleCredentialException(e, args)
_OutputAndExit(message=e, exception=e)
except AccessDeniedException as e:
_CheckAndHandleCredentialException(e, args)
_OutputAndExit(message=e, exception=e)
except ArgumentException as e:
_OutputAndExit(message=e, exception=e)
except ServiceException as e:
_OutputAndExit(message=e, exception=e)
except oauth2client.client.HttpAccessTokenRefreshError as e:
if system_util.InvokedViaCloudSdk():
_OutputAndExit(
'Your credentials are invalid. '
'Please run\n$ gcloud auth login',
exception=e)
else:
_OutputAndExit(
'Your credentials are invalid. For more help, see '
'"gsutil help creds", or re-run the gsutil config command (see '
'"gsutil help config").',
exception=e)
except apitools_exceptions.HttpError as e:
# These should usually be retried by the underlying implementation or
# wrapped by CloudApi ServiceExceptions, but if we do get them,
# print something useful.
_OutputAndExit('HttpError: %s, %s' %
(getattr(e.response, 'status', ''), e.content or ''),
exception=e)
except socket.error as e:
if e.args[0] == errno.EPIPE:
# Retrying with a smaller file (per suggestion below) works because
# the library code send loop (in boto/s3/key.py) can get through the
# entire file and then request the HTTP response before the socket
# gets closed and the response lost.
_OutputAndExit(
'Got a "Broken pipe" error. This can happen to clients using Python '
'2.x, when the server sends an error response and then closes the '
'socket (see http://bugs.python.org/issue5542). If you are trying to '
'upload a large object you might retry with a small (say 200k) '
'object, and see if you get a more specific error code.',
exception=e)
elif e.args[0] == errno.ECONNRESET and ' '.join(args).contains('s3://'):
_OutputAndExit('\n'.join(
textwrap.wrap(
'Got a "Connection reset by peer" error. One way this can happen is '
'when copying data to/from an S3 regional bucket. If you are using a '
'regional S3 bucket you could try re-running this command using the '
'regional S3 endpoint, for example '
's3://s3-<region>.amazonaws.com/your-bucket. For details about this '
'problem see https://github.com/boto/boto/issues/2207')),
exception=e)
else:
_HandleUnknownFailure(e)
except oauth2client.client.FlowExchangeError as e:
_OutputAndExit('\n%s\n\n' % '\n'.join(
textwrap.wrap(
'Failed to retrieve valid credentials (%s). Make sure you selected and '
'pasted the ENTIRE authorization code (including any numeric prefix '
"e.g. '4/')." % e)),
exception=e)
except reauth_errors.ReauthSamlLoginRequiredError:
if system_util.InvokedViaCloudSdk():
_OutputAndExit('You must re-authenticate with your SAML IdP. '
'Please run\n$ gcloud auth login')
else:
_OutputAndExit('You must re-authenticate with your SAML IdP. '
'Please run\n$ gsutil config')
except Exception as e: # pylint: disable=broad-except
config_paths = ', '.join(boto_util.GetFriendlyConfigFilePaths())
# Check for two types of errors related to service accounts. These errors
# appear to be the same except for their messages, but they are caused by
# different problems and both have unhelpful error messages. Moreover,
# the error type belongs to PyOpenSSL, which is not necessarily installed.
if 'mac verify failure' in str(e):
_OutputAndExit(
'Encountered an error while refreshing access token. '
'If you are using a service account,\nplease verify that the '
'gs_service_key_file_password field in your config file(s),'
'\n%s, is correct.' % config_paths,
exception=e)
elif 'asn1 encoding routines' in str(e):
_OutputAndExit(
'Encountered an error while refreshing access token. '
'If you are using a service account,\nplease verify that the '
'gs_service_key_file field in your config file(s),\n%s, is correct.' %
config_paths,
exception=e)
_HandleUnknownFailure(e)
def RunNamedCommand(self,
command_name,
args=None,
headers=None,
debug=0,
trace_token=None,
parallel_operations=False,
skip_update_check=False,
logging_filters=None,
do_shutdown=True,
perf_trace_token=None,
user_project=None,
collect_analytics=False):
"""Runs the named command.
Used by gsutil main, commands built atop other commands, and tests.
Args:
command_name: The name of the command being run.
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
trace_token: Trace token to pass to the underlying API.
parallel_operations: Should command operations be executed in parallel?
skip_update_check: Set to True to disable checking for gsutil updates.
logging_filters: Optional list of logging.Filters to apply to this
command's logger.
do_shutdown: Stop all parallelism framework workers iff this is True.
perf_trace_token: Performance measurement trace token to pass to the
underlying API.
user_project: The project to bill this request to.
collect_analytics: Set to True to collect an analytics metric logging this
command.
Raises:
CommandException: if errors encountered.
Returns:
Return value(s) from Command that was run.
"""
command_changed_to_update = False
if (not skip_update_check and self.MaybeCheckForAndOfferSoftwareUpdate(
command_name, debug)):
command_name = 'update'
command_changed_to_update = True
args = ['-n']
# Check for opt-in analytics.
if system_util.IsRunningInteractively() and collect_analytics:
metrics.CheckAndMaybePromptForAnalyticsEnabling()
if not args:
args = []
# Include api_version header in all commands.
api_version = boto.config.get_value('GSUtil', 'default_api_version',
'1')
if not headers:
headers = {}
headers['x-goog-api-version'] = api_version
if command_name not in self.command_map:
close_matches = difflib.get_close_matches(command_name,
self.command_map.keys(),
n=1)
if close_matches:
# Instead of suggesting a deprecated command alias, suggest the new
# name for that command.
translated_command_name = (OLD_ALIAS_MAP.get(
close_matches[0], close_matches)[0])
print >> sys.stderr, 'Did you mean this?'
print >> sys.stderr, '\t%s' % translated_command_name
elif command_name == 'update' and gslib.IS_PACKAGE_INSTALL:
sys.stderr.write(
'Update command is not supported for package installs; '
'please instead update using your package manager.')
raise CommandException('Invalid command "%s".' % command_name)
if '--help' in args:
new_args = [command_name]
original_command_class = self.command_map[command_name]
subcommands = original_command_class.help_spec.subcommand_help_text.keys(
)
for arg in args:
if arg in subcommands:
new_args.append(arg)
break # Take the first match and throw away the rest.
args = new_args
command_name = 'help'
HandleArgCoding(args)
HandleHeaderCoding(headers)
command_class = self.command_map[command_name]
command_inst = command_class(self,
args,
headers,
debug,
trace_token,
parallel_operations,
self.bucket_storage_uri_class,
self.gsutil_api_class_map_factory,
logging_filters,
command_alias_used=command_name,
perf_trace_token=perf_trace_token,
user_project=user_project)
# Log the command name, command alias, and sub-options after being parsed by
# RunCommand and the command constructor. For commands with subcommands and
# suboptions, we need to log the suboptions again within the command itself
# because the command constructor will not parse the suboptions fully.
if collect_analytics:
metrics.LogCommandParams(command_name=command_inst.command_name,
sub_opts=command_inst.sub_opts,
command_alias=command_name)
return_code = command_inst.RunCommand()
if CheckMultiprocessingAvailableAndInit().is_available and do_shutdown:
ShutDownGsutil()
if GetFailureCount() > 0:
return_code = 1
if command_changed_to_update:
# If the command changed to update, the user's original command was
# not executed.
return_code = 1
print '\n'.join(
textwrap.wrap(
'Update was successful. Exiting with code 1 as the original command '
'issued prior to the update was not executed and should be re-run.'
))
return return_code
