def __init__(self, bindings, agent=None):
"""Constructor
Args:
bindings: [dict] The parameter bindings for overriding the test
scenario configuration.
agent: [SpinnakerAgent] The Spinnaker agent to bind to the scenario.
"""
super(SpinnakerTestScenario, self).__init__(bindings, agent)
agent = self.agent
self.__update_bindings_with_subsystem_configuration(agent)
JournalLogger.begin_context('Configure Cloud Bindings')
try:
self.__init_google_bindings()
self.__init_aws_bindings()
self.__init_kubernetes_bindings()
self.__init_appengine_bindings()
self.__init_openstack_bindings()
self._do_init_bindings()
except:
logger = logging.getLogger(__name__)
logger.exception('Failed to initialize spinnaker agent.')
raise
finally:
JournalLogger.end_context()
def wait(self, poll_every_secs=None, max_secs=None):
"""Wait until the status reaches a final state.
Args:
poll_every_secs: [float] Interval to refresh() from the proxy.
This could also be a function taking an attempt number and
returning number of seconds for that attempt.
The default is default_wait_time_func.
max_secs: [float] Most seconds to wait before giving up.
0 is a poll, None is unbounded. Otherwise, number of seconds.
"""
if self.finished:
return
if max_secs is None:
max_secs = self.operation.max_wait_secs
if max_secs is not None and max_secs < 0:
raise ValueError()
message = 'Wait on id={0}, max_secs={1}'.format(self.id, max_secs)
JournalLogger.begin_context(message)
context_relation = 'ERROR'
try:
self.refresh()
self.__wait_helper(poll_every_secs, max_secs)
context_relation = 'VALID' if self.finished_ok else 'INVALID'
finally:
JournalLogger.end_context(relation=context_relation)
def verify_quota(title, gcp_agent, project_quota, regions):
"""Verify that the observed GCP project has sufficient quota.
Args:
title: [string] What the quota is for, for logging purposes only.
gcp_agent: [GcpAgent] Observation agent on the desired project.
project_quota: [dict] Minimum desired values keyed by quota metric for
the observed project.
regions: [array of (name, dict) tuple]: A list of regions and their
individual quotas to check.
Returns:
json_contract.ContractVerifyResult against the quota check.
"""
execution_context = ExecutionContext()
contract = make_quota_contract(gcp_agent, project_quota, regions)
verify_results = None
context_relation = 'ERROR'
try:
JournalLogger.begin_context(title)
verify_results = contract.verify(execution_context)
context_relation = 'VALID' if verify_results else 'INVALID'
finally:
if verify_results is not None:
journal = get_global_journal()
if journal is not None:
journal.store(verify_results)
JournalLogger.end_context(relation=context_relation)
return verify_results
def verify_quota(title, gcp_agent, project_quota, regions):
"""Verify that the observed GCP project has sufficient quota.
Args:
title: [string] What the quota is for, for logging purposes only.
gcp_agent: [GcpAgent] Observation agent on the desired project.
project_quota: [dict] Minimum desired values keyed by quota metric for
the observed project.
regions: [array of (name, dict) tuple]: A list of regions and their
individual quotas to check.
Returns:
json_contract.ContractVerifyResult against the quota check.
"""
execution_context = ExecutionContext()
contract = make_quota_contract(gcp_agent, project_quota, regions)
verify_results = None
context_relation = 'ERROR'
try:
JournalLogger.begin_context(title)
verify_results = contract.verify(execution_context)
context_relation = 'VALID' if verify_results else 'INVALID'
finally:
if verify_results is not None:
journal = get_global_journal()
if journal is not None:
journal.store(verify_results)
JournalLogger.end_context(relation=context_relation)
return verify_results
def wait(self, poll_every_secs=None, max_secs=None):
"""Wait until the status reaches a final state.
Args:
poll_every_secs: [float] Interval to refresh() from the proxy.
This could also be a function taking an attempt number and
returning number of seconds for that attempt.
The default is default_wait_time_func.
max_secs: [float] Most seconds to wait before giving up.
0 is a poll, None is unbounded. Otherwise, number of seconds.
"""
if self.finished:
return
if max_secs is None:
max_secs = self.operation.max_wait_secs
if max_secs is not None and max_secs < 0:
raise ValueError()
message = 'Wait on id={0}, max_secs={1}'.format(self.id, max_secs)
JournalLogger.begin_context(message)
context_relation = 'ERROR'
try:
self.refresh()
self.__wait_helper(poll_every_secs, max_secs)
context_relation = 'VALID' if self.finished_ok else 'INVALID'
finally:
JournalLogger.end_context(relation=context_relation)
def new_gce_instance_from_bindings(cls, name, status_factory, bindings,
port):
"""Create a new Spinnaker HttpAgent talking to the specified server port.
Args:
name: [string] The name of agent we are creating for reporting only.
status_factory: [SpinnakerStatus (SpinnakerAgent, HttpResponseType)]
Factory method for creating specialized SpinnakerStatus instances.
bindings: [dict] List of bindings to configure the endpoint
GCE_PROJECT: The GCE project ID that the endpoint is in.
GCE_ZONE: The GCE zone that the endpoint is in.
GCE_INSTANCE: The GCE instance that the endpoint is in.
GCE_SSH_PASSPHRASE_FILE: If not empty, the SSH passphrase key
for tunneling if needed to connect through a GCE firewall.
GCE_SERVICE_ACCOUNT: If not empty, the GCE service account to use
when interacting with the GCE instance.
port: [int] The port of the endpoint we want to connect to.
Returns:
A SpinnakerAgent connected to the specified instance port.
"""
project = bindings['GCE_PROJECT']
zone = bindings['GCE_ZONE']
instance = bindings['GCE_INSTANCE']
ssh_passphrase_file = bindings.get('GCE_SSH_PASSPHRASE_FILE', None)
service_account = bindings.get('GCE_SERVICE_ACCOUNT', None)
logger = logging.getLogger(__name__)
JournalLogger.begin_context('Locating {0}...'.format(name))
context_relation = 'ERROR'
try:
gcloud = gcp.GCloudAgent(project=project,
zone=zone,
service_account=service_account,
ssh_passphrase_file=ssh_passphrase_file)
netloc = gce_util.establish_network_connectivity(gcloud=gcloud,
instance=instance,
target_port=port)
if not netloc:
error = 'Could not locate {0}.'.format(name)
logger.error(error)
context_relation = 'INVALID'
raise RuntimeError(error)
protocol = bindings['NETWORK_PROTOCOL']
base_url = '{protocol}://{netloc}'.format(protocol=protocol,
netloc=netloc)
logger.info('%s is available at %s. Using %s', name, netloc,
base_url)
deployed_config = scrape_spring_config(
os.path.join(base_url, 'resolvedEnv'))
spinnaker_agent = cls(base_url, status_factory)
spinnaker_agent.__deployed_config = deployed_config
context_relation = 'VALID'
except:
logger.exception('Failed to create spinnaker agent.')
raise
finally:
JournalLogger.end_context(relation=context_relation)
return spinnaker_agent
def wait(self,
poll_every_secs=1,
max_secs=None,
trace_every=False,
trace_first=True):
"""Wait until the status reaches a final state.
Args:
poll_every_secs: [float] Interval to refresh() from the proxy.
max_secs: [float] Most seconds to wait before giving up.
0 is a poll, None is unbounded. Otherwise, number of seconds.
trace_every: [bool] Whether or not to log every poll request.
trace_first: [bool] Whether to log the first poll request.
"""
if self.finished:
return
if max_secs is None:
max_secs = self.operation.max_wait_secs
if max_secs < 0 and max_secs is not None:
raise ValueError()
message = 'Wait on id={0}, max_secs={1}'.format(self.id, max_secs)
JournalLogger.begin_context(message)
context_relation = 'ERROR'
try:
self.refresh(trace=trace_first)
self.__wait_helper(poll_every_secs, max_secs, trace_every)
context_relation = 'VALID' if self.finished_ok else 'INVALID'
finally:
JournalLogger.end_context(relation=context_relation)
def verify(self, context):
"""Attempt to make an observation and verify it.
This call will repeatedly attempt to observe new data and verify it
until either the verification passes, or it times out base on
the retryable_for_secs specified in the constructor.
Args:
context: Runtime citest execution context may contain operation status
and other testing parameters used by downstream verifiers.
Returns:
ContractClauseVerifyResult with details.
"""
JournalLogger.begin_context(
'Verifying ContractClause: {0}'.format(self.__title))
context_relation = 'ERROR'
try:
JournalLogger.delegate("store", self, _title='Clause Specification')
result = self.__do_verify(context)
context_relation = 'VALID' if result else 'INVALID'
finally:
JournalLogger.end_context(relation=context_relation)
return result
def new_gce_instance_from_bindings(
cls, name, status_factory, bindings, port):
"""Create a new Spinnaker HttpAgent talking to the specified server port.
Args:
name: [string] The name of agent we are creating for reporting only.
status_factory: [SpinnakerStatus (SpinnakerAgent, HttpResponseType)]
Factory method for creating specialized SpinnakerStatus instances.
bindings: [dict] List of bindings to configure the endpoint
GCE_PROJECT: The GCE project ID that the endpoint is in.
GCE_ZONE: The GCE zone that the endpoint is in.
GCE_INSTANCE: The GCE instance that the endpoint is in.
GCE_SSH_PASSPHRASE_FILE: If not empty, the SSH passphrase key
for tunneling if needed to connect through a GCE firewall.
GCE_SERVICE_ACCOUNT: If not empty, the GCE service account to use
when interacting with the GCE instance.
port: [int] The port of the endpoint we want to connect to.
Returns:
A SpinnakerAgent connected to the specified instance port.
"""
project = bindings['GCE_PROJECT']
zone = bindings['GCE_ZONE']
instance = bindings['GCE_INSTANCE']
ssh_passphrase_file = bindings.get('GCE_SSH_PASSPHRASE_FILE', None)
service_account = bindings.get('GCE_SERVICE_ACCOUNT', None)
logger = logging.getLogger(__name__)
JournalLogger.begin_context('Locating {0}...'.format(name))
context_relation = 'ERROR'
try:
gcloud = gcp.GCloudAgent(
project=project, zone=zone, service_account=service_account,
ssh_passphrase_file=ssh_passphrase_file)
netloc = gce_util.establish_network_connectivity(
gcloud=gcloud, instance=instance, target_port=port)
if not netloc:
error = 'Could not locate {0}.'.format(name)
logger.error(error)
context_relation = 'INVALID'
raise RuntimeError(error)
protocol = bindings['NETWORK_PROTOCOL']
base_url = '{protocol}://{netloc}'.format(protocol=protocol,
netloc=netloc)
logger.info('%s is available at %s. Using %s', name, netloc, base_url)
deployed_config = scrape_spring_config(
os.path.join(base_url, 'resolvedEnv'))
JournalLogger.journal_or_log_detail(
'{0} configuration'.format(name), deployed_config)
spinnaker_agent = cls(base_url, status_factory)
spinnaker_agent.__deployed_config = deployed_config
context_relation = 'VALID'
except:
logger.exception('Failed to create spinnaker agent.')
raise
finally:
JournalLogger.end_context(relation=context_relation)
return spinnaker_agent
def run_test_case(self, test_case, context=None, **kwargs):
"""Run the specified test operation from start to finish.
Args:
test_case: [OperationContract] To test.
context: [ExecutionContext] The citest execution context to run in.
timeout_ok: [bool] Whether an AgentOperationStatus timeout implies
a test failure. If it is ok to timeout, then we'll still verify the
contracts, but skip the final status check if there is no final
status yet.
max_retries: [int] Number of independent retries permitted on
individual operations if the operation status fails. A value of 0
indicates that a test should only be given a single attempt.
retry_interval_secs: [int] The number of seconds to wait between retries.
max_wait_secs: [int] How long to wait for status completion.
Default=Determined by operation in the test case.
"""
if context is None:
context = ExecutionContext()
# This is complicated because of all the individual parts
# that we want to ensure excute, so we'll break it up into
# helper functions based on scope and use the context to
# pass back some shared state variables since they make sense
# to communicate in the context anyway.
#
# This particular method is responsible for the logging context
# and the post-execution cleanup, if any.
#
# It will delegate to a helper function for the execution and
# pre/post hooks
#
# To get the context relation, we'll peek inside the execution
# context to see how the status and validation turned out.
JournalLogger.begin_context('Test "{0}"'.format(test_case.title))
try:
self._do_run_test_case_with_hooks(test_case, context, **kwargs)
finally:
try:
if test_case.cleanup:
attempt_info = context.get(self.CONTEXT_KEY_ATTEMPT_INFO, None)
if attempt_info is None or attempt_info.status is None:
self.logger.info('Skipping operation cleanup because'
' operation could not be performed at all.')
else:
self.logger.info('Invoking injected operation cleanup.')
test_case.cleanup(context)
finally:
verify_results = context.get(
self.CONTEXT_KEY_CONTRACT_VERIFY_RESULTS, None)
if verify_results is None:
context_relation = 'ERROR'
else:
final_status_ok = context.get(self.CONTEXT_KEY_FINAL_STATUS_OK, False)
context_relation = ('VALID' if (final_status_ok and verify_results)
else 'INVALID')
JournalLogger.end_context(relation=context_relation)
def list_available_images(self):
"""Creates a test that confirms expected available images.
Returns:
st.OperationContract
"""
logger = logging.getLogger(__name__)
# Get the list of images available (to the service account we are using).
context = citest.base.ExecutionContext()
gcp_agent = self.gcp_observer
JournalLogger.begin_context("Collecting expected available images")
relation_context = "ERROR"
try:
logger.debug("Looking up available images.")
json_doc = gcp_agent.list_resource(context, "images")
for project in GCP_STANDARD_IMAGES.keys():
logger.info("Looking for images from project=%s", project)
found = gcp_agent.list_resource(context,
"images",
project=project)
for image in found:
if not image.get("deprecated", None):
json_doc.append(image)
# Produce the list of images that we expect to receive from spinnaker
# (visible to the primary service account).
spinnaker_account = self.bindings["SPINNAKER_GOOGLE_ACCOUNT"]
logger.debug('Configured with Spinnaker account "%s"',
spinnaker_account)
expect_images = [{
"account": spinnaker_account,
"imageName": image["name"]
} for image in json_doc]
expect_images = sorted(expect_images, key=lambda k: k["imageName"])
relation_context = "VALID"
finally:
JournalLogger.end_context(relation=relation_context)
# pylint: disable=bad-continuation
builder = HttpContractBuilder(self.agent)
(builder.new_clause_builder("Has Expected Images").get_url_path(
"/gce/images/find").EXPECT(
ov_factory.value_list_matches(
[
jp.DICT_SUBSET(image_entry)
for image_entry in expect_images
],
strict=True,
unique=True,
)))
return st.OperationContract(NoOpOperation("List Available Images"),
contract=builder.build())
def list_available_images(self):
"""Creates a test that confirms expected available images.
Returns:
st.OperationContract
"""
logger = logging.getLogger(__name__)
# Get the list of images available (to the service account we are using).
context = citest.base.ExecutionContext()
gcp_agent = self.gcp_observer
JournalLogger.begin_context('Collecting expected available images')
relation_context = 'ERROR'
try:
logger.debug('Looking up available images.')
json_doc = gcp_agent.list_resource(context, 'images')
for project in GCP_STANDARD_IMAGES.keys():
logger.info('Looking for images from project=%s', project)
found = gcp_agent.list_resource(context,
'images',
project=project)
for image in found:
if not image.get('deprecated', None):
json_doc.append(image)
# Produce the list of images that we expect to receive from spinnaker
# (visible to the primary service account).
spinnaker_account = self.agent.deployed_config.get(
'providers.google.primaryCredentials.name')
logger.debug('Configured with Spinnaker account "%s"',
spinnaker_account)
expect_images = [{
'account': spinnaker_account,
'imageName': image['name']
} for image in json_doc]
expect_images = sorted(expect_images, key=lambda k: k['imageName'])
relation_context = 'VALID'
finally:
JournalLogger.end_context(relation=relation_context)
# pylint: disable=bad-continuation
builder = HttpContractBuilder(self.agent)
(builder.new_clause_builder('Has Expected Images').get_url_path(
'/gce/images/find').contains_match(
[jp.DICT_SUBSET(image_entry) for image_entry in expect_images],
match_kwargs={
'strict': True,
'unique': True
}))
return st.OperationContract(NoOpOperation('List Available Images'),
contract=builder.build())
def new_gce_instance_from_bindings(cls, name, status_factory, bindings, port):
"""Create a new Spinnaker HttpAgent talking to the specified server port.
Args:
name: [string] The name of agent we are creating for reporting only.
status_factory: [SpinnakerStatus (SpinnakerAgent, HttpResponseType)]
Factory method for creating specialized SpinnakerStatus instances.
bindings: [dict] List of bindings to configure the endpoint
GCE_PROJECT: The GCE project ID that the endpoint is in.
GCE_ZONE: The GCE zone that the endpoint is in.
GCE_INSTANCE: The GCE instance that the endpoint is in.
GCE_SSH_PASSPHRASE_FILE: If not empty, the SSH passphrase key
for tunneling if needed to connect through a GCE firewall.
GCE_SERVICE_ACCOUNT: If not empty, the GCE service account to use
when interacting with the GCE instance.
port: [int] The port of the endpoint we want to connect to.
Returns:
A SpinnakerAgent connected to the specified instance port.
"""
project = bindings["GCE_PROJECT"]
zone = bindings["GCE_ZONE"]
instance = bindings["GCE_INSTANCE"]
ssh_passphrase_file = bindings.get("GCE_SSH_PASSPHRASE_FILE", None)
service_account = bindings.get("GCE_SERVICE_ACCOUNT", None)
logger = logging.getLogger(__name__)
JournalLogger.begin_context("Locating {0}...".format(name))
context_relation = "ERROR"
try:
gcloud = gcp.GCloudAgent(
project=project, zone=zone, service_account=service_account, ssh_passphrase_file=ssh_passphrase_file
)
netloc = gce_util.establish_network_connectivity(gcloud=gcloud, instance=instance, target_port=port)
if not netloc:
error = "Could not locate {0}.".format(name)
logger.error(error)
context_relation = "INVALID"
raise RuntimeError(error)
approx_config = cls.__get_deployed_local_yaml_bindings(gcloud, instance)
protocol = approx_config.get("services.default.protocol", "http")
base_url = "{protocol}://{netloc}".format(protocol=protocol, netloc=netloc)
logger.info("%s is available at %s", name, base_url)
deployed_config = scrape_spring_config(os.path.join(base_url, "env"))
spinnaker_agent = cls(base_url, status_factory)
spinnaker_agent.__deployed_config = deployed_config
context_relation = "VALID"
except:
logger.exception("Failed to create spinnaker agent.")
raise
finally:
JournalLogger.end_context(relation=context_relation)
return spinnaker_agent
def post_run_hook(self, info, test_case, context):
if not info:
return
scanner, before_usage = info
analyzer = self.__google_resource_analyzer
JournalLogger.begin_context('Capturing final quota usage')
try:
after_usage = analyzer.collect_resource_usage(self.gcp_observer, scanner)
finally:
JournalLogger.end_context()
analyzer.log_delta_resource_usage(
test_case, scanner, before_usage, after_usage)
def post_run_hook(self, info, test_case, context):
if not info:
return
scanner, before_usage = info
analyzer = self.__google_resource_analyzer
JournalLogger.begin_context('Capturing final quota usage')
try:
after_usage = analyzer.collect_resource_usage(self.gcp_observer, scanner)
finally:
JournalLogger.end_context()
analyzer.log_delta_resource_usage(
test_case, scanner, before_usage, after_usage)
def list_available_images(self):
"""Creates a test that confirms expected available images.
Returns:
st.OperationContract
"""
logger = logging.getLogger(__name__)
# Get the list of images available (to the service account we are using).
context = citest.base.ExecutionContext()
gcp_agent = self.gcp_observer
JournalLogger.begin_context('Collecting expected available images')
relation_context = 'ERROR'
try:
logger.debug('Looking up available images.')
json_doc = gcp_agent.list_resource(context, 'images')
for project in GCP_STANDARD_IMAGES.keys():
logger.info('Looking for images from project=%s', project)
found = gcp_agent.list_resource(context, 'images', project=project)
for image in found:
if not image.get('deprecated', None):
json_doc.append(image)
# Produce the list of images that we expect to receive from spinnaker
# (visible to the primary service account).
spinnaker_account = self.bindings['SPINNAKER_GOOGLE_ACCOUNT']
logger.debug('Configured with Spinnaker account "%s"', spinnaker_account)
expect_images = [{'account': spinnaker_account, 'imageName': image['name']}
for image in json_doc]
expect_images = sorted(expect_images, key=lambda k: k['imageName'])
relation_context = 'VALID'
finally:
JournalLogger.end_context(relation=relation_context)
# pylint: disable=bad-continuation
builder = HttpContractBuilder(self.agent)
(builder.new_clause_builder('Has Expected Images')
.get_url_path('/gce/images/find')
.EXPECT(
ov_factory.value_list_matches(
[jp.DICT_SUBSET(image_entry) for image_entry in expect_images],
strict=True,
unique=True)))
return st.OperationContract(
NoOpOperation('List Available Images'),
contract=builder.build())
def list_available_images(self):
"""Creates a test that confirms expected available images.
Returns:
st.OperationContract
"""
logger = logging.getLogger(__name__)
# Get the list of images available (to the service account we are using).
context = citest.base.ExecutionContext()
gcp_agent = self.gcp_observer
JournalLogger.begin_context('Collecting expected available images')
relation_context = 'ERROR'
try:
logger.debug('Looking up available images.')
json_doc = gcp_agent.list_resource(context, 'images')
for project in GCP_STANDARD_IMAGES.keys():
logger.info('Looking for images from project=%s', project)
found = gcp_agent.list_resource(context, 'images', project=project)
for image in found:
if not image.get('deprecated', None):
json_doc.append(image)
# Produce the list of images that we expect to receive from spinnaker
# (visible to the primary service account).
spinnaker_account = self.agent.deployed_config.get(
'providers.google.primaryCredentials.name')
logger.debug('Configured with Spinnaker account "%s"', spinnaker_account)
expect_images = [{'account': spinnaker_account, 'imageName': image['name']}
for image in json_doc]
expect_images = sorted(expect_images, key=lambda k: k['imageName'])
relation_context = 'VALID'
finally:
JournalLogger.end_context(relation=relation_context)
# pylint: disable=bad-continuation
builder = HttpContractBuilder(self.agent)
(builder.new_clause_builder('Has Expected Images')
.get_url_path('/gce/images/find')
.add_constraint(jp.PathPredicate(jp.DONT_ENUMERATE_TERMINAL,
jp.EQUIVALENT(expect_images))))
return st.OperationContract(
NoOpOperation('List Available Images'),
contract=builder.build())
def __init__(self, bindings, agent=None):
"""Constructor
Args:
bindings: [dict] The parameter bindings for overriding the test
scenario configuration.
agent: [SpinnakerAgent] The Spinnaker agent to bind to the scenario.
"""
super(SpinnakerTestScenario, self).__init__(bindings, agent)
self.__google_resource_analyzer = None
agent = self.agent
bindings = self.bindings
# For read-only tests that don't make mutating calls to Spinnaker,
# there is nothing to update in the bindings, e.g. GCP quota test.
if agent is not None:
for key, value in agent.runtime_config.items():
try:
if bindings[key]:
continue # keep existing value already set within citest
except KeyError:
pass
bindings[key] = value # use value from agent's configuration
JournalLogger.begin_context("Configure Scenario Bindings")
self.__platform_support = {}
for klas in PLATFORM_SUPPORT_CLASSES:
try:
support = klas(self)
self.__platform_support[support.platform_name] = support
except:
logger = logging.getLogger(__name__)
logger.exception(
"Failed to initialize support class %s:\n%s",
str(klas),
traceback.format_exc(),
)
try:
self._do_init_bindings()
except:
logger = logging.getLogger(__name__)
logger.exception("Failed to initialize spinnaker agent.")
raise
finally:
JournalLogger.end_context()
def __init__(self, bindings, agent=None):
"""Constructor
Args:
bindings: [dict] The parameter bindings for overriding the test
scenario configuration.
agent: [SpinnakerAgent] The Spinnaker agent to bind to the scenario.
"""
super(SpinnakerTestScenario, self).__init__(bindings, agent)
agent = self.agent
self.__update_bindings_with_subsystem_configuration(agent)
JournalLogger.begin_context('Configure Cloud Bindings')
try:
self.__init_google_bindings()
self.__init_aws_bindings()
self.__init_kubernetes_bindings()
finally:
JournalLogger.end_context()
def __init__(self, bindings, agent=None):
"""Constructor
Args:
bindings: [dict] The parameter bindings for overriding the test
scenario configuration.
agent: [SpinnakerAgent] The Spinnaker agent to bind to the scenario.
"""
super(SpinnakerTestScenario, self).__init__(bindings, agent)
agent = self.agent
self.__update_bindings_with_subsystem_configuration(agent)
JournalLogger.begin_context('Configure Cloud Bindings')
try:
self.__init_google_bindings()
self.__init_aws_bindings()
self.__init_kubernetes_bindings()
finally:
JournalLogger.end_context()
def __init__(self, bindings, agent=None):
"""Constructor
Args:
bindings: [dict] The parameter bindings for overriding the test
scenario configuration.
agent: [SpinnakerAgent] The Spinnaker agent to bind to the scenario.
"""
super(SpinnakerTestScenario, self).__init__(bindings, agent)
self.__google_resource_analyzer = None
agent = self.agent
bindings = self.bindings
# For read-only tests that don't make mutating calls to Spinnaker,
# there is nothing to update in the bindings, e.g. GCP quota test.
if agent is not None:
for key, value in agent.runtime_config.items():
try:
if bindings[key]:
continue # keep existing value already set within citest
except KeyError:
pass
bindings[key] = value # use value from agent's configuration
JournalLogger.begin_context('Configure Scenario Bindings')
self.__platform_support = {}
for klas in PLATFORM_SUPPORT_CLASSES:
try:
support = klas(self)
self.__platform_support[support.platform_name] = support
except:
logger = logging.getLogger(__name__)
logger.exception('Failed to initialize support class %s:\n%s',
str(klas), traceback.format_exc())
try:
self._do_init_bindings()
except:
logger = logging.getLogger(__name__)
logger.exception('Failed to initialize spinnaker agent.')
raise
finally:
JournalLogger.end_context()
def pre_run_hook(self, test_case, context):
if not self.bindings.get('RECORD_GCP_RESOURCE_USAGE'):
return None
if self.__google_resource_analyzer is None:
from google_scenario_support import GcpResourceUsageAnalyzer
self.__google_resource_analyzer = GcpResourceUsageAnalyzer(self)
analyzer = self.__google_resource_analyzer
scanner = analyzer.make_gcp_api_scanner(
self.bindings.get('GOOGLE_ACCOUNT_PROJECT'),
self.bindings.get('GOOGLE_CREDENTIALS_PATH'),
include_apis=['compute'],
exclude_apis=['compute.*Operations'])
JournalLogger.begin_context('Capturing initial quota usage')
try:
usage = analyzer.collect_resource_usage(self.gcp_observer, scanner)
finally:
JournalLogger.end_context()
return (scanner, usage)
def pre_run_hook(self, test_case, context):
if not self.bindings.get('RECORD_GCP_RESOURCE_USAGE'):
return None
if self.__google_resource_analyzer is None:
from google_scenario_support import GcpResourceUsageAnalyzer
self.__google_resource_analyzer = GcpResourceUsageAnalyzer(self)
analyzer = self.__google_resource_analyzer
scanner = analyzer.make_gcp_api_scanner(
self.bindings.get('GOOGLE_ACCOUNT_PROJECT'),
self.bindings.get('GOOGLE_CREDENTIALS_PATH'),
include_apis=['compute'],
exclude_apis=['compute.*Operations'])
JournalLogger.begin_context('Capturing initial quota usage')
try:
usage = analyzer.collect_resource_usage(self.gcp_observer, scanner)
finally:
JournalLogger.end_context()
return (scanner, usage)
def run_test_case(self, test_case, context=None, **kwargs):
"""Run the specified test operation from start to finish.
Args:
test_case: [OperationContract] To test.
context: [ExecutionContext] The citest execution context to run in.
timeout_ok: [bool] Whether an AgentOperationStatus timeout implies
a test failure. If it is ok to timeout, then we'll still verify the
contracts, but skip the final status check if there is no final
status yet.
max_retries: [int] Number of independent retries permitted on
individual operations if the operation status fails. A value of 0
indicates that a test should only be given a single attempt.
retry_interval_secs: [int] The number of seconds to wait between retries.
full_trace: [bool] If true, then apply as much tracing as possible, else
use the default tracing. The intent here is to be able to crank up
the tracing when needed but not be overwhelmed by data when the
default tracing is typically sufficient.
poll_every_secs: [int] Number of seconds between wait polls. Default=1.
"""
if context is None:
context = ExecutionContext()
timeout_ok = kwargs.pop('timeout_ok', False)
max_retries = kwargs.pop('max_retries', 0)
retry_interval_secs = kwargs.pop('retry_interval_secs', 5)
poll_every_secs = kwargs.pop('poll_every_secs', 1)
full_trace = kwargs.pop('full_trace', False)
if kwargs:
raise TypeError('Unrecognized arguments {0}'.format(kwargs.keys()))
self.log_start_test(test_case.title)
if max_retries < 0:
raise ValueError(
'max_retries={max} cannot be negative'.format(max=max_retries))
if retry_interval_secs < 0:
raise ValueError(
'retry_interval_secs={secs} cannot be negative'.format(
secs=retry_interval_secs))
execution_trace = OperationContractExecutionTrace(test_case)
verify_results = None
final_status_ok = None
context_relation = None
attempt_info = None
status = None
try:
JournalLogger.begin_context('Test "{0}"'.format(test_case.title))
JournalLogger.delegate(
"store",
test_case.operation,
_title='Operation "{0}" Specification'.format(
test_case.operation.title))
max_tries = 1 + max_retries
# We attempt the operation on the agent multiple times until the agent
# thinks that it succeeded. But we will only verify once the agent thinks
# it succeeded. We do not give multiple chances to satisfy the
# verification.
for i in range(max_tries):
context.clear_key('OperationStatus')
context.clear_key('AttemptInfo')
attempt_info = execution_trace.new_attempt()
status = None
status = test_case.operation.execute(agent=self.testing_agent)
status.wait(poll_every_secs=poll_every_secs,
trace_every=full_trace)
summary = status.error or ('Operation status OK'
if status.finished_ok else
'Operation status Unknown')
# Write the status (and attempt_info) into the execution_context
# to make it available to contract verifiers. For example, to
# make specific details in the status (e.g. new resource names)
# available to downstream validators for their consideration.
context.set_internal('AttemptInfo', attempt_info)
context.set_internal('OperationStatus', status)
attempt_info.set_status(status, summary)
if test_case.status_extractor:
test_case.status_extractor(status, context)
if not status.exception_details:
execution_trace.set_operation_summary('Completed test.')
break
if max_tries - i > 1:
self.logger.warning(
'Got an exception: %s.\nTrying again in %r secs...',
status.exception_details, retry_interval_secs)
time.sleep(retry_interval_secs)
elif max_tries > 1:
execution_trace.set_operation_summary(
'Gave up retrying operation.')
self.logger.error('Giving up retrying test.')
# We're always going to verify the contract, even if the request itself
# failed. We set the verification on the attempt here, but do not assert
# anything. We'll assert below outside this try/catch handler.
verify_results = test_case.contract.verify(context)
execution_trace.set_verify_results(verify_results)
final_status_ok = self.verify_final_status_ok(
status,
timeout_ok=timeout_ok,
final_attempt=attempt_info,
execution_trace=execution_trace)
context_relation = ('VALID' if (final_status_ok and verify_results)
else 'INVALID')
except BaseException as ex:
context_relation = 'ERROR'
execution_trace.set_exception(ex)
if attempt_info is None:
execution_trace.set_exception(ex,
traceback_module.format_exc())
elif not attempt_info.completed:
# Exception happened during the attempt as opposed to during our
# verification afterwards.
attempt_info.set_exception(ex, traceback_module.format_exc())
try:
self.logger.error('Test failed with exception: %s', ex)
self.logger.error('Last status was:\n%s', str(status))
self.logger.debug('Exception was at:\n%s',
traceback_module.format_exc())
except BaseException as unexpected:
self.logger.error(
'Unexpected error %s\nHandling original exception %s',
unexpected, ex)
self.logger.debug('Unexpected exception was at:\n%s',
traceback_module.format_exc())
raise
finally:
try:
context.set_internal('ContractVerifyResults', verify_results)
self.log_end_test(test_case.title)
self.report(execution_trace)
if test_case.cleanup:
if status is None:
self.logger.info(
'Skipping operation cleanup because'
' operation could not be performed at all.')
else:
self.logger.info(
'Invoking injected operation cleanup.')
test_case.cleanup(context)
finally:
JournalLogger.end_context(relation=context_relation)
if not final_status_ok:
self.raise_final_status_not_ok(status, attempt_info)
if verify_results is not None:
self.assertVerifyResults(verify_results)
def new_gce_instance_from_bindings(cls, name, status_factory, bindings, port):
"""Create a new Spinnaker HttpAgent talking to the specified server port.
Args:
name: [string] The name of agent we are creating for reporting only.
status_factory: [SpinnakerStatus (SpinnakerAgent, HttpResponseType)]
Factory method for creating specialized SpinnakerStatus instances.
bindings: [dict] List of bindings to configure the endpoint
GCE_PROJECT: The GCE project ID that the endpoint is in.
GCE_ZONE: The GCE zone that the endpoint is in.
GCE_INSTANCE: The GCE instance that the endpoint is in.
GCE_SSH_PASSPHRASE_FILE: If not empty, the SSH passphrase key
for tunneling if needed to connect through a GCE firewall.
GCE_SERVICE_ACCOUNT: If not empty, the GCE service account to use
when interacting with the GCE instance.
IGNORE_SSL_CERT_VERIFICATION: If True, ignores SSL certificate
verification when scraping spring config.
port: [int] The port of the endpoint we want to connect to.
Returns:
A SpinnakerAgent connected to the specified instance port.
"""
project = bindings["GCE_PROJECT"]
zone = bindings["GCE_ZONE"]
instance = bindings["GCE_INSTANCE"]
ssh_passphrase_file = bindings.get("GCE_SSH_PASSPHRASE_FILE", None)
service_account = bindings.get("GCE_SERVICE_ACCOUNT", None)
ignore_ssl_cert_verification = bindings["IGNORE_SSL_CERT_VERIFICATION"]
logger = logging.getLogger(__name__)
JournalLogger.begin_context("Locating {0}...".format(name))
context_relation = "ERROR"
try:
gcloud = gcp.GCloudAgent(
project=project,
zone=zone,
service_account=service_account,
ssh_passphrase_file=ssh_passphrase_file,
)
netloc = gce_util.establish_network_connectivity(
gcloud=gcloud, instance=instance, target_port=port
)
if not netloc:
error = "Could not locate {0}.".format(name)
logger.error(error)
context_relation = "INVALID"
raise RuntimeError(error)
protocol = bindings["NETWORK_PROTOCOL"]
base_url = "{protocol}://{netloc}".format(protocol=protocol, netloc=netloc)
logger.info("%s is available at %s. Using %s", name, netloc, base_url)
deployed_config = scrape_spring_config(
os.path.join(base_url, "resolvedEnv"),
ignore_ssl_cert_verification=ignore_ssl_cert_verification,
)
JournalLogger.journal_or_log_detail(
"{0} configuration".format(name), deployed_config
)
spinnaker_agent = cls(base_url, status_factory)
spinnaker_agent.__deployed_config = deployed_config
context_relation = "VALID"
except:
logger.exception("Failed to create spinnaker agent.")
raise
finally:
JournalLogger.end_context(relation=context_relation)
return spinnaker_agent
def run_test_case(self, test_case, context=None, **kwargs):
"""Run the specified test operation from start to finish.
Args:
test_case: [OperationContract] To test.
context: [ExecutionContext] The citest execution context to run in.
timeout_ok: [bool] Whether an AgentOperationStatus timeout implies
a test failure. If it is ok to timeout, then we'll still verify the
contracts, but skip the final status check if there is no final
status yet.
max_retries: [int] Number of independent retries permitted on
individual operations if the operation status fails. A value of 0
indicates that a test should only be given a single attempt.
retry_interval_secs: [int] The number of seconds to wait between retries.
max_wait_secs: [int] How long to wait for status completion.
Default=Determined by operation in the test case.
"""
if context is None:
context = ExecutionContext()
# This is complicated because of all the individual parts
# that we want to ensure excute, so we'll break it up into
# helper functions based on scope and use the context to
# pass back some shared state variables since they make sense
# to communicate in the context anyway.
#
# This particular method is responsible for the logging context
# and the post-execution cleanup, if any.
#
# It will delegate to a helper function for the execution and
# pre/post hooks
#
# To get the context relation, we'll peek inside the execution
# context to see how the status and validation turned out.
JournalLogger.begin_context('Test "{0}"'.format(test_case.title))
try:
self._do_run_test_case_with_hooks(test_case, context, **kwargs)
finally:
try:
if test_case.cleanup:
attempt_info = context.get(self.CONTEXT_KEY_ATTEMPT_INFO,
None)
if attempt_info is None or attempt_info.status is None:
self.logger.info(
'Skipping operation cleanup because'
' operation could not be performed at all.')
else:
self.logger.info(
'Invoking injected operation cleanup.')
test_case.cleanup(context)
finally:
verify_results = context.get(
self.CONTEXT_KEY_CONTRACT_VERIFY_RESULTS, None)
if verify_results is None:
context_relation = 'ERROR'
else:
final_status_ok = context.get(
self.CONTEXT_KEY_FINAL_STATUS_OK, False)
context_relation = ('VALID' if
(final_status_ok
and verify_results) else 'INVALID')
JournalLogger.end_context(relation=context_relation)
def run_test_case(self, test_case, context=None, **kwargs):
"""Run the specified test operation from start to finish.
Args:
test_case: [OperationContract] To test.
context: [ExecutionContext] The citest execution context to run in.
timeout_ok: [bool] Whether an AgentOperationStatus timeout implies
a test failure. If it is ok to timeout, then we'll still verify the
contracts, but skip the final status check if there is no final
status yet.
max_retries: [int] Number of independent retries permitted on
individual operations if the operation status fails. A value of 0
indicates that a test should only be given a single attempt.
retry_interval_secs: [int] The number of seconds to wait between retries.
poll_every_secs: [int] Number of seconds between wait polls. Default=1.
"""
if context is None:
context = ExecutionContext()
timeout_ok = kwargs.pop('timeout_ok', False)
max_retries = kwargs.pop('max_retries', 0)
retry_interval_secs = kwargs.pop('retry_interval_secs', 5)
poll_every_secs = kwargs.pop('poll_every_secs', 1)
full_trace = kwargs.pop('full_trace', False) # Deprecated
if kwargs:
raise TypeError('Unrecognized arguments {0}'.format(kwargs.keys()))
self.log_start_test(test_case.title)
if max_retries < 0:
raise ValueError(
'max_retries={max} cannot be negative'.format(max=max_retries))
if retry_interval_secs < 0:
raise ValueError(
'retry_interval_secs={secs} cannot be negative'.format(
secs=retry_interval_secs))
execution_trace = OperationContractExecutionTrace(test_case)
verify_results = None
final_status_ok = None
context_relation = None
attempt_info = None
status = None
try:
JournalLogger.begin_context('Test "{0}"'.format(test_case.title))
JournalLogger.delegate(
"store", test_case.operation,
_title='Operation "{0}" Specification'.format(
test_case.operation.title))
max_tries = 1 + max_retries
# We attempt the operation on the agent multiple times until the agent
# thinks that it succeeded. But we will only verify once the agent thinks
# it succeeded. We do not give multiple chances to satisfy the
# verification.
for i in range(max_tries):
context.clear_key('OperationStatus')
context.clear_key('AttemptInfo')
attempt_info = execution_trace.new_attempt()
status = None
status = test_case.operation.execute(agent=self.testing_agent)
status.wait(poll_every_secs=poll_every_secs)
summary = status.error or ('Operation status OK' if status.finished_ok
else 'Operation status Unknown')
# Write the status (and attempt_info) into the execution_context
# to make it available to contract verifiers. For example, to
# make specific details in the status (e.g. new resource names)
# available to downstream validators for their consideration.
context.set_internal('AttemptInfo', attempt_info)
context.set_internal('OperationStatus', status)
attempt_info.set_status(status, summary)
if test_case.status_extractor:
test_case.status_extractor(status, context)
if not status.exception_details:
execution_trace.set_operation_summary('Completed test.')
break
if max_tries - i > 1:
self.logger.warning(
'Got an exception: %s.\nTrying again in %r secs...',
status.exception_details, retry_interval_secs)
time.sleep(retry_interval_secs)
elif max_tries > 1:
execution_trace.set_operation_summary('Gave up retrying operation.')
self.logger.error('Giving up retrying test.')
# We're always going to verify the contract, even if the request itself
# failed. We set the verification on the attempt here, but do not assert
# anything. We'll assert below outside this try/catch handler.
verify_results = test_case.contract.verify(context)
execution_trace.set_verify_results(verify_results)
final_status_ok = self.verify_final_status_ok(
status, timeout_ok=timeout_ok,
final_attempt=attempt_info,
execution_trace=execution_trace)
context_relation = ('VALID' if (final_status_ok and verify_results)
else 'INVALID')
except BaseException as ex:
context_relation = 'ERROR'
execution_trace.set_exception(ex)
if attempt_info is None:
execution_trace.set_exception(ex, traceback_module.format_exc())
elif not attempt_info.completed:
# Exception happened during the attempt as opposed to during our
# verification afterwards.
attempt_info.set_exception(ex, traceback_module.format_exc())
try:
self.logger.error('Test failed with exception: %s', ex)
self.logger.error('Last status was:\n%s', str(status))
self.logger.debug('Exception was at:\n%s',
traceback_module.format_exc())
except BaseException as unexpected:
self.logger.error(
'Unexpected error %s\nHandling original exception %s',
unexpected, ex)
self.logger.debug('Unexpected exception was at:\n%s',
traceback_module.format_exc())
raise
finally:
try:
context.set_internal('ContractVerifyResults', verify_results)
self.log_end_test(test_case.title)
self.report(execution_trace)
if test_case.cleanup:
if status is None:
self.logger.info('Skipping operation cleanup because'
' operation could not be performed at all.')
else:
self.logger.info('Invoking injected operation cleanup.')
test_case.cleanup(context)
finally:
JournalLogger.end_context(relation=context_relation)
if not final_status_ok:
self.raise_final_status_not_ok(status, attempt_info)
if verify_results is not None:
self.assertVerifyResults(verify_results)
