def testIsInteractiveErr(self):
self.StartObjectPatch(sys.stdin, 'isatty').return_value = True
stderr_mock = self.StartObjectPatch(sys.stderr, 'isatty')
stderr_mock.return_value = False
self.assertFalse(console_io.IsInteractive(output=True))
stderr_mock.return_value = True
self.assertTrue(console_io.IsInteractive(error=True))
def MakeUserAgentString(cmd_path=None):
"""Return a user-agent string for this request.
Contains 'gcloud' in addition to several other product IDs used for tracing in
metrics reporting.
Args:
cmd_path: str representing the current command for tracing.
Returns:
str, User Agent string.
"""
return ('gcloud/{0}'
' command/{1}'
' invocation-id/{2}'
' environment/{3}'
' environment-version/{4}'
' interactive/{5}'
' python/{6}'
' {7}').format(
config.CLOUD_SDK_VERSION.replace(' ', '_'), cmd_path
or properties.VALUES.metrics.command_name.Get(),
uuid.uuid4().hex, properties.GetMetricsEnvironment(),
properties.VALUES.metrics.environment_version.Get(),
console_io.IsInteractive(error=True, heuristic=True),
platform.python_version(),
platforms.Platform.Current().UserAgentFragment())
def ProgressTracker(
message=None, autotick=True, detail_message_callback=None, tick_delay=1,
interruptable=True,
aborted_message=console_io.OperationCancelledError.DEFAULT_MESSAGE):
"""A context manager for telling the user about long-running progress.
Args:
message: str, The message to show next to the spinner.
autotick: bool, True to have the spinner tick on its own. Otherwise, you
need to call Tick() explicitly to move the spinner.
detail_message_callback: func, A no argument function that will be called
and the result appended to message each time it needs to be printed.
tick_delay: float, The amount of time to wait between ticks, in second.
interruptable: boolean, True if the user can ctrl-c the operation. If so,
it will stop and will report as aborted. If False, a message will be
displayed saying that it cannot be cancelled.
aborted_message: str, A custom message to put in the exception when it is
cancelled by the user.
Returns:
The progress tracker.
"""
style = properties.VALUES.core.interactive_ux_style.Get()
if style == properties.VALUES.core.InteractiveUXStyles.OFF.name:
return _NoOpProgressTracker(interruptable, aborted_message)
elif style == properties.VALUES.core.InteractiveUXStyles.TESTING.name:
return _StubProgressTracker(message, interruptable, aborted_message)
else:
is_tty = console_io.IsInteractive(error=True)
tracker_cls = (_NormalProgressTracker if is_tty
else _NonInteractiveProgressTracker)
return tracker_cls(
message, autotick, detail_message_callback, tick_delay, interruptable,
aborted_message)
def __init__(self,
message=None,
autotick=True,
detail_message_callback=None,
tick_delay=1):
if message is None:
self._spinner_only = True
self._message = ''
self._prefix = ''
self._suffix = ''
else:
self._spinner_only = False
self._message = message
self._prefix = message + '...'
self._suffix = 'done.'
self._ticks = 0
self._done = False
self._lock = threading.Lock()
self._detail_message_callback = detail_message_callback
self._multi_line = False
self._last_display_message = ''
self._tick_delay = tick_delay
self._is_tty = console_io.IsInteractive(error=True)
self._ticker = None
self.__autotick = autotick
def __call__(self, parser, namespace, values, option_string=None):
"""Render a help document according to the style in values.
Args:
parser: The ArgParse object.
namespace: The ArgParse namespace.
values: The --document flag ArgDict() value:
style=STYLE
The output style. Must be specified.
title=DOCUMENT TITLE
The document title.
notes=SENTENCES
Inserts SENTENCES into the document NOTES section.
option_string: The ArgParse flag string.
Raises:
parser_errors.ArgumentError: For unknown flag value attribute name.
"""
base.LogCommand(parser.prog, namespace)
if default_style:
# --help
metrics.Loaded()
style = default_style
notes = None
title = None
for attributes in values:
for name, value in six.iteritems(attributes):
if name == 'notes':
notes = value
elif name == 'style':
style = value
elif name == 'title':
title = value
else:
raise parser_errors.ArgumentError(
'Unknown document attribute [{0}]'.format(name))
if title is None:
title = command.dotted_name
metrics.Help(command.dotted_name, style)
# '--help' is set by the --help flag, the others by gcloud <style> ... .
if style in ('--help', 'help', 'topic'):
style = 'text'
md = io.StringIO(markdown.Markdown(command))
out = (io.StringIO() if console_io.IsInteractive(output=True)
else None)
if style == 'linter':
meta_data = GetCommandMetaData(command)
else:
meta_data = None
render_document.RenderDocument(style, md, out=out or log.out, notes=notes,
title=title, command_metadata=meta_data)
metrics.Ran()
if out:
console_io.More(out.getvalue())
sys.exit(0)
def __init__(self, project, args, history_id, gcs_results_root,
testing_client, testing_messages):
"""Construct a TestingApiHelper to be used with a single test invocation.
Args:
project: string containing the GCE project id.
args: an argparse namespace. All the arguments that were provided to this
command invocation (i.e. group and command arguments combined).
history_id: A history ID string to publish Tool Results to.
gcs_results_root: the root dir for a matrix within the GCS results bucket.
testing_client: Testing API client lib generated by Apitools.
testing_messages: Testing API messages lib generated by Apitools.
"""
self._project = project
self._args = args
self._history_id = history_id
self._gcs_results_root = gcs_results_root
self._client = testing_client
self._messages = testing_messages
self._max_status_length = 0
self.status_interval_sec = (
properties.VALUES.test.matrix_status_interval.GetInt()
or DEFAULT_STATUS_INTERVAL_SECS)
# Poll the matrix status half as fast if not running in interactive mode.
if not console_io.IsInteractive(error=True):
self.status_interval_sec *= 2
log.info('Matrix status interval: {0} sec'.format(
self.status_interval_sec))
exec_states = testing_messages.TestExecution.StateValueValuesEnum
self.state_names = {
exec_states.VALIDATING: 'Validating',
exec_states.PENDING: 'Pending',
exec_states.RUNNING: 'Running',
exec_states.FINISHED: 'Finished',
exec_states.ERROR: 'Error',
exec_states.UNSUPPORTED_ENVIRONMENT: 'Unsupported',
exec_states.INCOMPATIBLE_ENVIRONMENT: 'Incompatible Environment',
exec_states.INCOMPATIBLE_ARCHITECTURE: 'Incompatible Architecture',
exec_states.CANCELLED: 'Cancelled',
exec_states.INVALID: 'Invalid',
exec_states.TEST_STATE_UNSPECIFIED: '*Unspecified*',
}
self.completed_execution_states = set([
exec_states.FINISHED,
exec_states.ERROR,
exec_states.UNSUPPORTED_ENVIRONMENT,
exec_states.INCOMPATIBLE_ENVIRONMENT,
exec_states.INCOMPATIBLE_ARCHITECTURE,
exec_states.CANCELLED,
exec_states.INVALID,
])
matrix_states = testing_messages.TestMatrix.StateValueValuesEnum
self.completed_matrix_states = set([
matrix_states.FINISHED,
matrix_states.ERROR,
matrix_states.CANCELLED,
matrix_states.INVALID,
])
def WaitForOperation(self, operation, message, timeout_s, poll_period_s=5):
"""Poll dataproc Operation until its status is done or timeout reached.
Args:
operation: Operation, message of the operation to be polled.
message: str, message to display to user while polling.
timeout_s: number, seconds to poll with retries before timing out.
poll_period_s: number, delay in seconds between requests.
Returns:
Operation: the return value of the last successful operations.get
request.
Raises:
OperationError: if the operation times out or finishes with an error.
"""
request = self.messages.DataprocProjectsRegionsOperationsGetRequest(
name=operation.name)
log.status.Print('Waiting on operation [{0}].'.format(operation.name))
start_time = time.time()
warnings_so_far = 0
is_tty = console_io.IsInteractive(error=True)
tracker_separator = '\n' if is_tty else ''
def _LogWarnings(warnings):
new_warnings = warnings[warnings_so_far:]
if new_warnings:
# Drop a line to print nicely with the progress tracker.
log.err.write(tracker_separator)
for warning in new_warnings:
log.warn(warning)
with progress_tracker.ProgressTracker(message, autotick=True):
while timeout_s > (time.time() - start_time):
try:
operation = self.client.projects_regions_operations.Get(
request)
metadata = self.ParseOperationJsonMetadata(
operation.metadata)
_LogWarnings(metadata.warnings)
warnings_so_far = len(metadata.warnings)
if operation.done:
break
except apitools_exceptions.HttpError:
# Keep trying until we timeout in case error is transient.
pass
time.sleep(poll_period_s)
metadata = self.ParseOperationJsonMetadata(operation.metadata)
_LogWarnings(metadata.warnings)
if not operation.done:
raise exceptions.OperationTimeoutError(
'Operation [{0}] timed out.'.format(operation.name))
elif operation.error:
raise exceptions.OperationError(
'Operation [{0}] failed: {1}.'.format(
operation.name, util.FormatRpcError(operation.error)))
log.info('Operation [%s] finished after %.3f seconds', operation.name,
(time.time() - start_time))
return operation
def __init__(self, message, autotick, detail_message_callback,
done_message_callback, tick_delay, interruptable,
aborted_message, spinner_override_message, no_spacing):
self._stream = sys.stderr
if message is None:
self._spinner_only = True
self._message = ''
self._prefix = ''
else:
self._spinner_only = False
self._message = message
self._prefix = message + ('' if no_spacing else '...')
self._detail_message_callback = detail_message_callback
self.spinner_override_message = spinner_override_message
self._done_message_callback = done_message_callback
self._ticks = 0
self._done = False
self._lock = threading.Lock()
self._tick_delay = tick_delay
self._ticker = None
console_width = console_attr.ConsoleAttr().GetTermSize()[0]
if console_width < 0:
# This can happen if we're on a pseudo-TTY. Set it to 0 and also
# turn off output to prevent it from stopping responding.
console_width = 0
self._output_enabled = log.IsUserOutputEnabled() and console_width != 0
# Don't bother autoticking if we aren't going to print anything.
self.__autotick = autotick and self._output_enabled
self._interruptable = interruptable
self._aborted_message = aborted_message
self._old_signal_handler = None
self._symbols = console_attr.GetConsoleAttr().GetProgressTrackerSymbols()
self._no_spacing = no_spacing
self._is_tty = console_io.IsInteractive(error=True)
def MakeUserAgentString(cmd_path=None):
"""Return a user-agent string for this request.
Contains 'gcloud' in addition to several other product IDs used for tracing in
metrics reporting.
Args:
cmd_path: str representing the current command for tracing.
Returns:
str, User Agent string.
"""
return (
'gcloud/{version}'
' command/{cmd}'
' invocation-id/{inv_id}'
' environment/{environment}'
' environment-version/{env_version}'
' interactive/{is_interactive}'
' from-script/{from_script}'
' python/{py_version}'
' term/{term}'
' {ua_fragment}').format(
version=config.CLOUD_SDK_VERSION.replace(' ', '_'),
cmd=(cmd_path or properties.VALUES.metrics.command_name.Get()),
inv_id=uuid.uuid4().hex,
environment=properties.GetMetricsEnvironment(),
env_version=properties.VALUES.metrics.environment_version.Get(),
is_interactive=console_io.IsInteractive(error=True,
heuristic=True),
py_version=platform.python_version(),
ua_fragment=platforms.Platform.Current().UserAgentFragment(),
from_script=console_io.IsRunFromShellScript(),
term=console_attr.GetConsoleAttr().GetTermIdentifier())
def refresh(self, request):
"""Refreshes the access token and handles reauth request when it is asked.
Args:
request: google.auth.transport.Request, a callable used to make HTTP
requests.
"""
try:
return self._Refresh(request)
except ReauthRequiredError:
# reauth.GetRaptToken is implemented in oauth2client and it is built on
# httplib2. GetRaptToken does not work with
# google.auth.transport.Request.
if not console_io.IsInteractive():
log.info(
'Reauthentication not performed as we cannot prompt during '
'non-interactive execution.')
return
response_encoding = None if six.PY2 else 'utf-8'
http_request = http.Http(
response_encoding=response_encoding).request
self._rapt_token = reauth.GetRaptToken(http_request,
self._client_id,
self._client_secret,
self._refresh_token,
self._token_uri,
list(self.scopes or []))
return self._Refresh(request)
def __init__(self, message, autotick, detail_message_callback, tick_delay,
interruptable, aborted_message):
if message is None:
self._spinner_only = True
self._message = ''
self._prefix = ''
self._suffix = ''
else:
self._spinner_only = False
self._message = message
self._prefix = message + '...'
self._suffix = 'done.'
self._stream = sys.stderr
self._ticks = 0
self._done = False
self._lock = threading.Lock()
self._detail_message_callback = detail_message_callback
self._multi_line = False
self._last_display_message = ''
self._tick_delay = tick_delay
self._is_tty = console_io.IsInteractive(error=True)
self._ticker = None
self._output_enabled = log.IsUserOutputEnabled()
# Don't bother autoticking if we aren't going to print anything.
self.__autotick = autotick and self._output_enabled
self._interruptable = interruptable
self._aborted_message = aborted_message
self._old_signal_handler = None
def __init__(self, ga_tid=_GA_TID):
"""Initialize a new MetricsCollector.
This should only be invoked through the static GetCollector() function or
the static ResetCollectorInstance() function.
Args:
ga_tid: The Google Analytics tracking ID to use for metrics collection.
Defaults to _GA_TID.
"""
current_platform = platforms.Platform.Current()
self._user_agent = 'CloudSDK/{version} {fragment}'.format(
version=config.CLOUD_SDK_VERSION,
fragment=current_platform.UserAgentFragment())
self._async_popen_args = current_platform.AsyncPopenArgs()
self._project_ids = {}
hostname = socket.getfqdn()
install_type = 'Google' if hostname.endswith(
'.google.com') else 'External'
cid = _MetricsCollector._GetCID()
# Table of common params to send to both GA and CSI.
# First column is GA name, second column is CSI name, third is the value.
common_params = [
('cd1', 'release_channel',
config.INSTALLATION_CONFIG.release_channel),
('cd2', 'install_type', install_type),
('cd3', 'environment', properties.GetMetricsEnvironment()),
('cd4', 'interactive',
console_io.IsInteractive(error=True, heuristic=True)),
('cd5', 'python_version', platform.python_version()),
# cd6 passed as argument to _GAEvent - cd6 = Flag Names
('cd7', 'environment_version',
properties.VALUES.metrics.environment_version.Get()),
# cd8 passed as argument to _GAEvent - cd8 = Error
# cd9 passed as argument to _GAEvent - cd9 = Error Extra Info
]
self._ga_params = [('v', '1'), ('tid', ga_tid), ('cid', cid),
('t', 'event')]
self._ga_params.extend([(param[0], param[2])
for param in common_params])
self._csi_params = [('s', _CSI_ID), ('v', '2'),
('rls', config.CLOUD_SDK_VERSION), ('c', cid)]
self._csi_params.extend([(param[1], param[2])
for param in common_params])
self.StartTimer(_GetTimeMillis())
self._metrics = []
# Tracking the level so we can only report metrics for the top level action
# (and not other actions executed within an action). Zero is the top level.
self._action_level = 0
log.debug('Metrics collector initialized...')
def Http(cmd_path=None,
trace_token=None,
auth=True,
creds=None,
timeout=None,
log_http=False):
"""Get an httplib2.Http object for working with the Google API.
Args:
cmd_path: str, Path of command that will use the httplib2.Http object.
trace_token: str, Token to be used to route service request traces.
auth: bool, True if the http object returned should be authorized.
creds: oauth2client.client.Credentials, If auth is True and creds is not
None, use those credentials to authorize the httplib2.Http object.
timeout: double, The timeout in seconds to pass to httplib2. This is the
socket level timeout. If timeout is None, timeout is infinite.
log_http: bool, Enable/disable client side logging of service requests.
Returns:
An authorized httplib2.Http object, or a regular httplib2.Http object if no
credentials are available.
Raises:
c_store.Error: If an error loading the credentials occurs.
"""
# TODO(user): Have retry-once-if-denied logic, to allow client tools to not
# worry about refreshing credentials.
http = c_store._Http(timeout=timeout) # pylint:disable=protected-access
# Wrap first to dump any data added by other wrappers.
if log_http:
http = _WrapRequestForLogging(http)
# Wrap the request method to put in our own user-agent, and trace reporting.
gcloud_ua = ('gcloud/{0}'
' command/{1}'
' invocation-id/{2}'
' environment/{3}'
' interactive/{4}'
' {5}').format(
config.CLOUD_SDK_VERSION, cmd_path,
uuid.uuid4().hex,
properties.VALUES.metrics.environment.Get(),
console_io.IsInteractive(error=True, heuristic=True),
platforms.Platform.Current().UserAgentFragment())
http = _WrapRequestForUserAgentAndTracing(http, trace_token, gcloud_ua)
if auth:
if not creds:
creds = c_store.Load()
http = creds.authorize(http)
# Wrap the request method to put in our own error handling.
http = _WrapRequestForAuthErrHandling(http)
return http
def ProgressTracker(
message=None,
autotick=True,
detail_message_callback=None,
done_message_callback=None,
tick_delay=0.2,
interruptable=True,
screen_reader=False,
aborted_message=console_io.OperationCancelledError.DEFAULT_MESSAGE,
no_spacing=False):
"""A context manager for telling the user about long-running progress.
Args:
message: str, The message to show next to the spinner.
autotick: bool, True to have the spinner tick on its own. Otherwise, you
need to call Tick() explicitly to move the spinner.
detail_message_callback: func, A no argument function that will be called
and the result appended to message each time it needs to be printed.
done_message_callback: func, A no argument function whose result will be
appended to message if the progress tracker successfully exits.
tick_delay: float, The amount of time to wait between ticks, in second.
interruptable: boolean, True if the user can ctrl-c the operation. If so,
it will stop and will report as aborted. If False, a message will be
displayed saying that it cannot be cancelled.
screen_reader: boolean, override for screen reader accessibility property
toggle.
aborted_message: str, A custom message to put in the exception when it is
cancelled by the user.
no_spacing: boolean, Removes ellipses and other spacing between text.
Returns:
The progress tracker.
"""
style = properties.VALUES.core.interactive_ux_style.Get()
if style == properties.VALUES.core.InteractiveUXStyles.OFF.name:
return NoOpProgressTracker(interruptable, aborted_message)
elif style == properties.VALUES.core.InteractiveUXStyles.TESTING.name:
return _StubProgressTracker(message, interruptable, aborted_message)
else:
is_tty = console_io.IsInteractive(error=True)
tracker_cls = (_NormalProgressTracker if is_tty
else _NonInteractiveProgressTracker)
screen_reader = (screen_reader or
properties.VALUES.accessibility.screen_reader.GetBool())
spinner_override_message = None
if screen_reader:
tick_delay = 1
spinner_override_message = 'working'
return tracker_cls(message, autotick, detail_message_callback,
done_message_callback, tick_delay, interruptable,
aborted_message, spinner_override_message, no_spacing)
def __init__(self,
message,
autotick=True,
detail_message_callback=None,
tick_delay=1):
self._message = message
self._prefix = message + '...'
self._ticks = 0
self._autotick = autotick
self._done = False
self._lock = threading.Lock()
self._detail_message_callback = detail_message_callback
self._multi_line = False
self._last_display_message = ''
self._tick_delay = tick_delay
self._is_tty = console_io.IsInteractive(output=True, error=True)
def __init__(self):
hostname = socket.gethostname()
install_type = 'Google' if hostname.endswith('.google.com') else 'External'
current_platform = platforms.Platform.Current()
self.client_id = config.GetCID()
self.current_platform = current_platform
self.user_agent = GetUserAgent(current_platform)
self.release_channel = config.INSTALLATION_CONFIG.release_channel
self.install_type = install_type
self.metrics_environment = properties.GetMetricsEnvironment()
self.is_interactive = console_io.IsInteractive(error=True, heuristic=True)
self.python_version = platform.python_version()
self.metrics_environment_version = (properties.VALUES
.metrics.environment_version.Get())
self.is_run_from_shell_script = console_io.IsRunFromShellScript()
self.term_identifier = console_attr.GetConsoleAttr().GetTermIdentifier()
def _ChooseEntrypoint(default_entrypoint, appinfo):
"""Prompt the user for an entrypoint.
Args:
default_entrypoint: (str) Default entrypoint determined from the app.
appinfo: (apphosting.api.appinfo.AppInfoExternal or None) The parsed
app.yaml file for the module if it exists.
Returns:
(str) The actual entrypoint to use.
"""
if console_io.IsInteractive():
prompt = (
'Please enter the command to run this ruby app in production:\n'
'[ {0} ] ')
entrypoint = console_io.PromptResponse(
prompt.format(default_entrypoint))
entrypoint = entrypoint.strip()
if not entrypoint:
entrypoint = default_entrypoint
if appinfo:
# We've got an entrypoint and the user had an app.yaml that didn't
# specify it.
# TODO(user): Offer to edit the user's app.yaml
msg = (
'To avoid being asked for an entrypoint in the future, please '
'add it to your app.yaml. e.g.\n'
' entrypoint: {0}')
log.status.Print(msg.format(entrypoint))
return entrypoint
else:
msg = (
'Using a default entrypoint of [{0}]. If this is incorrect, please '
'create an app.yaml file with an "entrypoint" field specifying the '
'command to run the app in production.')
log.info(msg.format(default_entrypoint))
return default_entrypoint
def __init__(self, ga_tid=_GA_TID):
"""Initialize a new MetricsCollector.
This should only be invoked through the static GetCollector() function or
the static ResetCollectorInstance() function.
Args:
ga_tid: The Google Analytics tracking ID to use for metrics collection.
Defaults to _GA_TID.
"""
current_platform = platforms.Platform.Current()
self._user_agent = 'CloudSDK/{version} {fragment}'.format(
version=config.CLOUD_SDK_VERSION,
fragment=current_platform.UserAgentFragment())
self._async_popen_args = current_platform.AsycPopenArgs()
self._project_ids = {}
hostname = socket.getfqdn()
install_type = 'Google' if hostname.endswith(
'.google.com') else 'External'
self._ga_params = [
('v', '1'), ('tid', ga_tid), ('cid', _MetricsCollector._GetCID()),
('t', 'event'),
('cd1', config.INSTALLATION_CONFIG.release_channel),
('cd2', install_type),
('cd3', properties.VALUES.metrics.environment.Get()),
('cd4', console_io.IsInteractive(error=True, heuristic=True))
]
self._csi_params = [('s', _CSI_ID), ('v', '2'),
('rls', config.CLOUD_SDK_VERSION)]
self.StartTimer(time.time())
self._metrics = []
log.debug('Metrics collector initialized...')
def __init__(self,
matrix_id,
test_type,
context,
clock=datetime.datetime.now,
status_interval_secs=None):
"""Construct a MatrixMonitor to monitor a single test matrix instance.
Args:
matrix_id: {str} the unique ID of the matrix being monitored.
test_type: {str} the type of matrix test being run (e.g. 'robo')
context: {str:obj} dict containing the gcloud command context, which
includes the Testing API client & messages libs generated by Apitools.
clock: injected function which returns a current datetime object when
called. Used to generate time-stamps on progress messages.
status_interval_secs: {float} how long to sleep between status checks.
"""
self.matrix_id = matrix_id
self._test_type = test_type
self._client = context['testing_client']
self._messages = context['testing_messages']
self._clock = clock
self._project = util.GetProject()
self._max_status_length = 0
if status_interval_secs is not None:
self._status_interval_secs = status_interval_secs
else:
self._status_interval_secs = (
properties.VALUES.test.matrix_status_interval.GetInt()
or _DEFAULT_STATUS_INTERVAL_SECS)
# Poll for matrix status half as fast if the end user is not running in
# interactive mode (i.e. either sys.stdin or sys.stderr is not a terminal
# i/o stream) such as when gcloud is called by a CI system like Jenkins).
# This reduces Testing service load and API quota usage.
if not console_io.IsInteractive(error=True):
self._status_interval_secs *= 2
exec_states = self._messages.TestExecution.StateValueValuesEnum
self._state_names = {
exec_states.VALIDATING: 'Validating',
exec_states.PENDING: 'Pending',
exec_states.RUNNING: 'Running',
exec_states.FINISHED: 'Finished',
exec_states.ERROR: 'Error',
exec_states.UNSUPPORTED_ENVIRONMENT: 'Unsupported',
exec_states.INCOMPATIBLE_ENVIRONMENT: 'Incompatible Environment',
exec_states.INCOMPATIBLE_ARCHITECTURE: 'Incompatible Architecture',
exec_states.CANCELLED: 'Cancelled',
exec_states.INVALID: 'Invalid',
exec_states.TEST_STATE_UNSPECIFIED: '*Unspecified*',
}
self._completed_execution_states = set([
exec_states.FINISHED,
exec_states.ERROR,
exec_states.UNSUPPORTED_ENVIRONMENT,
exec_states.INCOMPATIBLE_ENVIRONMENT,
exec_states.INCOMPATIBLE_ARCHITECTURE,
exec_states.CANCELLED,
exec_states.INVALID,
])
matrix_states = self._messages.TestMatrix.StateValueValuesEnum
self.completed_matrix_states = set([
matrix_states.FINISHED,
matrix_states.ERROR,
matrix_states.CANCELLED,
matrix_states.INVALID,
])
def __init__(self, ga_tid=_GA_TID):
"""Initialize a new MetricsCollector.
This should only be invoked through the static GetCollector() function or
the static ResetCollectorInstance() function.
Args:
ga_tid: The Google Analytics tracking ID to use for metrics collection.
Defaults to _GA_TID.
"""
current_platform = platforms.Platform.Current()
self._user_agent = _MetricsCollector._GetUserAgent(current_platform)
self._async_popen_args = current_platform.AsyncPopenArgs()
self._project_ids = {}
hostname = socket.gethostname()
install_type = 'Google' if hostname.endswith('.google.com') else 'External'
cid = _MetricsCollector._GetCID()
# Table of common params to send to both GA and CSI.
# First column is GA name, second column is CSI name, third is the value.
common_params = [
('cd1', 'release_channel', config.INSTALLATION_CONFIG.release_channel),
('cd2', 'install_type', install_type),
('cd3', 'environment', properties.GetMetricsEnvironment()),
('cd4', 'interactive', console_io.IsInteractive(error=True,
heuristic=True)),
('cd5', 'python_version', platform.python_version()),
# cd6 passed as argument to _GAEvent - cd6 = Flag Names
('cd7', 'environment_version',
properties.VALUES.metrics.environment_version.Get()),
('cd12', 'from_script', console_io.IsRunFromShellScript()),
# cd8 passed as argument to _GAEvent - cd8 = Error
# cd9 passed as argument to _GAEvent - cd9 = Error Extra Info
]
self._ga_event_params = [
('v', '1'),
('tid', ga_tid),
('cid', cid),
('t', 'event')]
self._ga_event_params.extend(
[(param[0], param[2]) for param in common_params])
self._ga_events = []
self._ga_timing_params = [
('v', '1'),
('tid', ga_tid),
('cid', cid),
('t', 'timing')]
self._ga_timing_params.extend(
[(param[0], param[2]) for param in common_params])
cloud_sdk_version = config.CLOUD_SDK_VERSION
self._csi_params = [('s', _CSI_ID),
('v', '2'),
('rls', cloud_sdk_version),
('c', cid)]
self._csi_params.extend([(param[1], param[2]) for param in common_params])
self._timer = _CommandTimer()
self._clearcut_request_params = {
'client_info': {
'client_type': 'DESKTOP',
'desktop_client_info': {
'os': current_platform.operating_system.id
}
},
'log_source_name': 'CONCORD',
'zwieback_cookie': cid,
}
self._clearcut_concord_event_params = {
'release_version': cloud_sdk_version,
'console_type': 'CloudSDK',
}
self._clearcut_concord_event_metadata = [
{'key': param[1], 'value': str(param[2])} for param in common_params]
self._clearcut_concord_timed_events = []
self._metrics = []
# Tracking the level so we can only report metrics for the top level action
# (and not other actions executed within an action). Zero is the top level.
self._action_level = 0
log.debug('Metrics collector initialized...')
def _RunIsInteractive(self, args):
sys.exit(int(not console_io.IsInteractive(heuristic=True)))
def StagedProgressTracker(
message, stages, tracker_id=None, autotick=True, tick_delay=0.1,
interruptable=True, done_message_callback=None, success_message=None,
warning_message=None, failure_message=None,
aborted_message=console_io.OperationCancelledError.DEFAULT_MESSAGE,
suppress_output=False):
"""A progress tracker for performing actions with multiple stages.
The progress tracker is a context manager. To start displaying information
about a running stage, call StartStage within the staged progress tracker
context. To update the message of a stage, use UpdateStage. When a stage is
completed/failed there are CompleteStage and FailStage methods on the
tracker as well.
Note that stages do not need to be started/completed in order. In
non-multiline (the only supported mode) output mode, the displayed stage will
be the earliest started stage that has not been completed.
Example Usage:
stages = [
Stage('Getting bread...', key='bread'),
Stage('Getting peanut butter...', key='pb'),
Stage('Making sandwich...', key='make')]
with StagedProgressTracker(
'Making sandwich...',
stages,
success_message='Time to eat!',
failure_message='Time to order delivery..!',
tracker_id='meta.make_sandwich') as tracker:
tracker.StartStage('bread')
# Go to pantry
tracker.UpdateStage('bread', 'Looking for bread in the pantry')
# Get bread
tracker.CompleteStage('bread', 'Got some whole wheat bread!')
tracker.StartStage('pb')
# Look for peanut butter
if pb_not_found:
error = exceptions.NoPeanutButterError('So sad!')
tracker.FailStage('pb', error)
elif pb_not_organic:
tracker.CompleteStageWithWarning('pb', 'The pb is not organic!')
else:
tracker.CompleteStage('bread', 'Got some organic pb!')
Args:
message: str, The message to show next to the spinner.
stages: list[Stage], A list of stages for the progress tracker to run. Once
you pass the stages to a StagedProgressTracker, they're owned by the
tracker and you should not mutate them.
tracker_id: str The ID of this tracker that will be used for metrics.
autotick: bool, True to have the spinner tick on its own. Otherwise, you
need to call Tick() explicitly to move the spinner.
tick_delay: float, The amount of time to wait between ticks, in second.
interruptable: boolean, True if the user can ctrl-c the operation. If so,
it will stop and will report as aborted. If False,
done_message_callback: func, A callback to get a more detailed done message.
success_message: str, A message to display on success of all tasks.
warning_message: str, A message to display when no task fails but one or
more tasks complete with a warning and none fail.
failure_message: str, A message to display on failure of a task.
aborted_message: str, A custom message to put in the exception when it is
cancelled by the user.
suppress_output: bool, True to suppress output from the tracker.
Returns:
The progress tracker.
"""
style = properties.VALUES.core.interactive_ux_style.Get()
if (suppress_output
or style == properties.VALUES.core.InteractiveUXStyles.OFF.name):
return NoOpStagedProgressTracker(stages, interruptable, aborted_message)
elif style == properties.VALUES.core.InteractiveUXStyles.TESTING.name:
return _StubStagedProgressTracker(
message, stages, interruptable, aborted_message)
else:
is_tty = console_io.IsInteractive(error=True)
if is_tty:
if console_attr.ConsoleAttr().SupportsAnsi():
tracker_cls = _MultilineStagedProgressTracker
else:
tracker_cls = _NormalStagedProgressTracker
else:
tracker_cls = _NonInteractiveStagedProgressTracker
return tracker_cls(
message, stages, success_message, warning_message, failure_message,
autotick, tick_delay, interruptable, aborted_message, tracker_id,
done_message_callback)
def Fingerprint(path, params):
"""Check for a Python app.
Args:
path: (str) Application path.
params: (fingerprinting.Params) Parameters passed through to the
fingerprinters.
Returns:
(PythonConfigurator or None) Returns a module if the path contains a
python app.
"""
entrypoint = None
appinfo = params.appinfo
if appinfo:
# We only want to do fingerprinting for 'python', for 'python-compat' we
# want to fall through to plain dockerfiles.
if appinfo.GetEffectiveRuntime() != 'python':
return None
if appinfo.entrypoint:
entrypoint = appinfo.entrypoint
log.info('Checking for Python.')
requirements_txt = os.path.join(path, 'requirements.txt')
got_requirements_txt = os.path.isfile(requirements_txt)
got_py_files = False
# check for any python files.
for _, _, files in os.walk(path):
for filename in files:
if filename.endswith('.py'):
got_py_files = True
if not got_requirements_txt and not got_py_files:
return None
# Query the user for the WSGI entrypoint:
if not entrypoint:
if console_io.IsInteractive():
entrypoint = console_io.PromptResponse(
'This looks like a Python app. If so, please enter the command to '
"run to run the app in production (enter nothing if it's not a "
'python app): ').strip()
if not entrypoint:
log.info(
'No entrypoint specified. Assuming this is not a python app.'
)
elif appinfo:
# We've got an entrypoint and the user had an app.yaml that didn't
# specify it.
# TODO(user): offer to edit the user's app.yaml.
log.status.Print(
'To avoid being asked for an entrypoint in the '
'future, please add the entrypoint to your app.yaml:\n'
' entrypoint: %s' % entrypoint)
else:
log.warn(
"This appears to be a python app. You'll need to provide the "
'command to run the app in production. Please either run this '
'interactively or create an app.yaml with "runtime: python" and '
'an "entrypoint" field defining the full command.')
return None
return PythonConfigurator(path, params, got_requirements_txt, entrypoint)
def Execute(self, args=None, call_arg_complete=True):
"""Execute the CLI tool with the given arguments.
Args:
args: [str], The arguments from the command line or None to use sys.argv
call_arg_complete: Call the _ArgComplete function if True
Returns:
The result of executing the command determined by the command
implementation.
Raises:
ValueError: for ill-typed arguments.
"""
if isinstance(args, basestring):
raise ValueError(
'Execute expects an iterable of strings, not a string.')
# The argparse module does not handle unicode args when run in Python 2
# because it uses str(x) even when type(x) is unicode. This sets itself up
# for failure because it converts unicode strings back to byte strings which
# will trigger ASCII codec exceptions. It works in Python 3 because str() is
# equivalent to unicode() in Python 3. The next Pythonically magic and dirty
# statement coaxes the Python 3 behavior out of argparse running in
# Python 2. Doing it here ensures that the workaround is in place for
# calliope argparse use cases.
argparse.str = unicode
if call_arg_complete:
_ArgComplete(self.__top_element.ai)
if not args:
args = sys.argv[1:]
# help ... is the same as ... --help or ... --document=style=help. We use
# --document=style=help to signal the metrics.Help() 'help' label in
# actions.RenderDocumentAction().Action(). It doesn't matter if we append
# to a command that already has --help or --document=style=... because the
# first --help/--document from the left takes effect. Note that
# `help ... --help` produces help for the help command itself.
if args and args[0] == 'help' and '--help' not in args:
args = args[1:] + ['--document=style=help']
# Look for a --configuration flag and update property state based on
# that before proceeding to the main argparse parse step.
named_configs.FLAG_OVERRIDE_STACK.PushFromArgs(args)
properties.VALUES.PushInvocationValues()
# Set the command name in case an exception happens before the command name
# is finished parsing.
command_path_string = self.__name
specified_arg_names = None
argv = self._ConvertNonAsciiArgsToUnicode(args)
old_user_output_enabled = None
old_verbosity = None
try:
args = self.__parser.parse_args(argv)
command_path_string = '.'.join(args.command_path)
if not args.calliope_command.IsUnicodeSupported():
self._EnforceAsciiArgs(argv)
specified_arg_names = args.GetSpecifiedArgNames()
# -h|--help|--document are dispatched by parse_args and never get here.
# Now that we have parsed the args, reload the settings so the flags will
# take effect. These will use the values from the properties.
old_user_output_enabled = log.SetUserOutputEnabled(None)
old_verbosity = log.SetVerbosity(None)
# Set the command_name property so it is persisted until the process ends.
# Only do this for the top level command that can be detected by looking
# at the stack. It will have one initial level, and another level added by
# the PushInvocationValues earlier in this method.
if len(properties.VALUES.GetInvocationStack()) == 2:
properties.VALUES.metrics.command_name.Set(command_path_string)
# Set the invocation value for all commands, this is lost when popped
properties.VALUES.SetInvocationValue(
properties.VALUES.metrics.command_name, command_path_string,
None)
if properties.VALUES.core.capture_session_file.Get() is not None:
capturer = session_capturer.SessionCapturer()
capturer.CaptureArgs(args)
capturer.CaptureState(
interactive_console=console_io.IsInteractive())
capturer.CaptureProperties(properties.VALUES.AllValues())
session_capturer.SessionCapturer.capturer = capturer
for hook in self.__pre_run_hooks:
hook.Run(command_path_string)
resources = args.calliope_command.Run(cli=self, args=args)
for hook in self.__post_run_hooks:
hook.Run(command_path_string)
# Preserve generator or static list resources.
if isinstance(resources, types.GeneratorType):
def _Yield():
"""Activates generator exceptions."""
try:
for resource in resources:
yield resource
except Exception as exc: # pylint: disable=broad-except
self._HandleAllErrors(exc, command_path_string,
specified_arg_names)
return _Yield()
# Do this last. If there is an error, the error handler will log the
# command execution along with the error.
metrics.Commands(command_path_string, config.CLOUD_SDK_VERSION,
specified_arg_names)
return resources
except Exception as exc: # pylint: disable=broad-except
self._HandleAllErrors(exc, command_path_string,
specified_arg_names)
finally:
if session_capturer.SessionCapturer.capturer is not None:
with open(properties.VALUES.core.capture_session_file.Get(),
'w') as f:
session_capturer.SessionCapturer.capturer.Print(f)
properties.VALUES.PopInvocationValues()
named_configs.FLAG_OVERRIDE_STACK.Pop()
# Reset these values to their previous state now that we popped the flag
# values.
if old_user_output_enabled is not None:
log.SetUserOutputEnabled(old_user_output_enabled)
if old_verbosity is not None:
log.SetVerbosity(old_verbosity)