def _emit(self, level, msg, *args):
s = msg if len(args) == 0 else msg.format(*args)
dt = date_time_provider().now()
s = LOG_FORMAT.format(dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second,
str(dt.microsecond)[0:3], level, s)
print(s)
def base_test(self,
test_method,
instance_id,
expect_under_utilized,
expect_over_utilized,
cpu_high=90,
cpu_low=10,
interval=None):
parameters = {
lt.PARAM_CPU_PERC_LOW: cpu_low,
lt.PARAM_CPU_PERC_HIGH: cpu_high,
lt.PARAM_CPU_LOW_TAGS: testing.tags.common_placeholder_tags() + ",UNDER-UTILIZED=TRUE",
lt.PARAM_CPU_HIGH_TAGS: testing.tags.common_placeholder_tags() + ",OVER-UTILIZED=TRUE",
lt.ACTION_PARAM_INTERVAL: interval
}
launch_time = self.ec2.get_instance(instance_id)["LaunchTime"].replace(tzinfo=pytz.utc)
start = date_time_provider().utcnow().replace(tzinfo=pytz.utc, second=0, microsecond=0) - timedelta(
minutes=INTERVAL_MINUTES + 5)
if launch_time > start:
wait_for = launch_time - start + timedelta(minutes=1)
self.logger.test("Waiting {} for sufficient metrics data to become available for instance ", wait_for, instance_id)
time.sleep(wait_for.total_seconds())
self.logger.test("Running task")
self.task_runner.run(parameters, task_name=test_method)
self.assertTrue(self.task_runner.success(expected_executed_tasks=1), "Task executed successfully")
self.logger.test("[X] Task completed")
check_result = getattr(self.task_runner.results[0], "ActionResult", {})
self.assertEqual(check_result["instances-checked"], 1, "Instances checked must be 1")
self.logger.test("[X] Expected number of instances checked")
overutilized = check_result.get("overutilized-instances", [])
self.assertEqual(1 if expect_over_utilized else 0, len(overutilized), "overutilized instance")
self.logger.test("[X] Instance is {}overutilized", "" if expect_over_utilized else "not ")
underutilized = check_result.get("underutilized-instances", [])
self.assertEqual(1 if expect_under_utilized else 0, len(underutilized), "underutilized instance")
self.logger.test("[X] Instance is {}underutilized", "" if expect_under_utilized else "not ")
tags = self.ec2.get_instance_tags(instance_id)
self.assertTrue(testing.tags.verify_placeholder_tags(tags), "Expected tags")
self.assertEqual(tags.get("UNDER-UTILIZED", "FALSE"), "TRUE" if expect_under_utilized else "FALSE", "underutilized tag")
self.assertEqual(tags.get("OVER-UTILIZED", "FALSE"), "TRUE" if expect_over_utilized else "FALSE", "overutilized tag")
self.logger.test("[X] Tags created for instance")
def __init__(self, arguments, action_parameters):
self._assumed_role_ = None
self._context_ = None
self._debug_ = None
self._dryrun_ = None
self._event_ = None
self._events_ = None
self._logger_ = None
self._resources_ = None
self._session_ = None
self._stack_ = None
self._stack_id_ = None
self._stack_resources_ = None
self._start_result_ = None
self._started_at_ = None
self._tagfilter_ = None
self._task_ = None
self._task_id_ = None
self._task_timezone_ = None
self._timeout_ = None
self._timeout_event_ = None
self._datetime_ = date_time_provider()
for a in arguments:
setattr(self, "_{}_".format(a), arguments[a])
if not services.get_service_class(
self._event_.get(actions.ACTION_SERVICE)).is_regional():
self._region_ = self._session_.region_name
else:
action_properties = actions.get_action_properties(
self._event_[actions.ACTION])
aggregation_level = action_properties.get(
actions.ACTION_AGGREGATION,
actions.ACTION_AGGREGATION_RESOURCE)
if aggregation_level is not None and isinstance(
aggregation_level, types.FunctionType):
aggregation_level = aggregation_level(action_parameters)
if aggregation_level in [
actions.ACTION_AGGREGATION_REGION,
actions.ACTION_AGGREGATION_RESOURCE
]:
if isinstance(self._resources_, list):
if len(self._resources_) > 0:
self._region_ = self._resources_[0]["Region"]
else:
if self._resources_ is not None:
self._region_ = self._resources_["Region"]
if self._region_ is None:
self._region_ = self._session_.region_name
else:
self._region_ = self._session_.region_name
self._account_ = self.get_account_for_task()
if self._debug_ is None:
self._debug_ = False
if self._dryrun_ is None:
self._dryrun_ = False
for ap in action_parameters:
setattr(self, "_{}_".format(pascal_to_snake_case(ap)),
action_parameters[ap])
def build_tags_from_template(tags_str,
task, task_id,
timezone="UTC",
account=None,
region=None,
tag_variables=None,
restricted_value_set=False,
include_deleted_tags=True):
tag_vars = {} if tag_variables is None else copy.copy(tag_variables)
tz = timezone if timezone not in ["", None] else "UTC"
dt = date_time_provider().now(tz=pytz.timezone(tz))
dt = dt.replace(microsecond=0)
# variables used in tag names/values
tag_vars.update({
TAG_VAL_ACCOUNT: account if account is not None else "",
TAG_VAL_AUTOMATOR_STACK: os.getenv(handlers.ENV_STACK_NAME, ""),
TAG_VAL_DATE: "{:0>4d}{:0>2d}{:0>2d}".format(dt.year, dt.month, dt.day),
TAG_VAL_DATE_TIME: "{:0>4d}{:0>2d}{:0>2d}{:0>2d}{:0>2d}{:0>2d}".format(dt.year, dt.month, dt.day, dt.hour, dt.minute,
dt.second),
TAG_VAL_DAY: "{:0>2d}".format(dt.day),
TAG_VAL_HOUR: "{:0>2d}".format(dt.hour),
TAG_VAL_ISO_DATE: dt.date().isoformat(),
TAG_VAL_ISO_DATETIME: dt.isoformat(),
TAG_VAL_ISO_TIME: dt.time().isoformat(),
TAG_VAL_ISO_WEEKDAY: dt.isoweekday(),
TAG_VAL_MINUTE: "{:0>2d}".format(dt.minute),
TAG_VAL_MONTH: "{:0>2d}".format(dt.month),
TAG_VAL_MONTH_NAME: dt.strftime("%b"),
TAG_VAL_MONTH_NAME_LONG: dt.strftime("%B"),
TAG_VAL_REGION: region if region is not None else "",
TAG_VAL_SECOND: "{:0>2d}".format(dt.second),
TAG_VAL_TASK_TAG: os.getenv(handlers.ENV_AUTOMATOR_TAG_NAME),
TAG_VAL_TASK: task,
TAG_VAL_TASK_ID: task_id,
TAG_VAL_TIME: "{:0>2d}{:0>2d}{:0>2d}".format(dt.hour, dt.minute, dt.second),
TAG_VAL_TIMEZONE: dt.tzname(),
TAG_VAL_WEEKDAY: dt.strftime("%a"),
TAG_VAL_WEEKDAY_LONG: dt.strftime("%A"),
TAG_VAL_YEAR: "{:0>4d}".format(dt.year)
})
# get ssm parameter values and add to variables
names = re.findall("{ssm:(.+?)\}", tags_str)
if len(names) > 0:
resp = boto3.client("ssm").get_parameters(Names=list(set(names)))
for p in resp.get("Parameters", []):
tag_vars["ssm:{}".format(p["Name"])] = p["Value"].split(",") if p["Type"] == "StringList" else p["Value"]
# variables as strings
for v in list(tag_vars.keys()):
if tag_vars[v] is None:
value = ""
elif isinstance(tag_vars[v], list):
value = ",".join(tag_vars[v])
elif isinstance(tag_vars[v], dict):
value = safe_json(tag_vars[v])
else:
value = str(tag_vars[v])
tag_vars[v] = value
# build tag names with unprocessed values
lastkey = None
tags = {}
for t in tags_str.split(","):
t = t.strip()
if "=" in t:
t = t.partition("=")
key = t[0].strip()
for v in tag_vars:
key = key.replace(TAG_VAL_STR.format(v), tag_vars[v])
tags[key] = t[2].strip()
lastkey = key
elif lastkey is not None:
tags[lastkey] = ",".join([tags[lastkey], t])
# process values
for t in tags:
if tags[t] not in ["", None]:
for v in tag_vars:
tags[t] = tags[t].replace(TAG_VAL_STR.format(v), tag_vars[v])
else:
if tags[t] is None:
del tags[t]
if restricted_value_set:
clean_tag_set(tags)
if not include_deleted_tags:
for t in list(tags.keys()):
if tags[t] == TAG_DELETE:
del tags[t]
return tags
def run(self,
parameters,
complete_check_polling_interval=60,
task_timeout=None,
task_name=None,
datetime_delta=None,
events=None,
tag_filter=None,
run_in_regions=None,
action_select_params=None,
debug=False,
run_after_select=None):
self.results = []
self.executed_tasks = []
self.parameters = parameters
self.action_select_parameters = action_select_params if action_select_params is not None else {}
self.task_name = task_name if task_name is not None else "{}-test".format(
self.action_name).lower()
self._ensure_action_stack()
self.run_after_select = run_after_select
self.context = Context()
self._events = events if events is not None else {}
self._tag_filter = tag_filter
save_debug = self.debug
self.debug = debug
self.logger._debug = self.debug
self.interval = parameters.get(actions.ACTION_PARAM_INTERVAL, None)
self.run_in_regions = run_in_regions if run_in_regions is not None else [
self.test_region
]
try:
if datetime_delta is not None:
set_datetime_delta(datetime_delta)
actions.set_date_time_provider(DatetimeProvider)
self.logger.test(
"Setting simulated test execution date and time to {}",
actions.date_time_provider().now())
for executed_task in self._get_tasks_to_execute():
try:
self.executed_tasks.append(executed_task)
self.logger.test(
"Start execution of action {} using assumed role {} in region {}",
self.action_name, self._assumed_role,
self.tested_region)
start_result = executed_task.execute()
if not executed_task.get(
actions.ACTION_PARAM_HAS_COMPLETION, False):
self.logger.test("Action completed with result {}",
safe_json(start_result, indent=3))
setattr(executed_task, handlers.TASK_TR_RESULT,
start_result)
setattr(executed_task, handlers.TASK_TR_STATUS,
handlers.STATUS_COMPLETED)
else:
self.logger.test("Waiting for task to complete")
setattr(executed_task, handlers.TASK_TR_START_RESULT,
start_result)
# noinspection PyProtectedMember
timeout = executed_task._timeout_
if timeout is None:
timeout = task_timeout * 60 if task_timeout is not None else 60
timeout *= 60
with Timer(timeout_seconds=timeout) as timer:
while True:
is_completed = getattr(
executed_task, handlers.COMPLETION_METHOD,
None)
if is_completed is None:
raise Exception(
"Tested action needs completion but does not implement the required {} method",
handlers.COMPLETION_METHOD)
complete_result = executed_task.is_completed(
start_result)
if complete_result is not None:
self.logger.test(
"Action completed with result {}",
safe_json(complete_result, indent=3))
setattr(executed_task,
handlers.TASK_TR_STATUS,
handlers.STATUS_COMPLETED)
setattr(executed_task,
handlers.TASK_TR_RESULT,
complete_result)
break
if timer.timeout:
self.logger.test("Action timed out")
setattr(executed_task,
handlers.TASK_TR_STATUS,
handlers.STATUS_TIMED_OUT)
setattr(executed_task,
handlers.TASK_TR_ERROR, "Timeout")
break
self.logger.test(
"Action not completed yet, waiting {} seconds",
complete_check_polling_interval)
time.sleep(complete_check_polling_interval)
self.results.append(executed_task)
except Exception as ex:
self.logger.test("Action failed {}", str(ex))
setattr(executed_task, handlers.TASK_TR_STATUS,
handlers.STATUS_FAILED)
setattr(executed_task, handlers.TASK_TR_ERROR, str(ex))
self.results.append(executed_task)
finally:
if datetime_delta is not None:
actions.reset_date_provider()
self.debug = save_debug
self.logger._debug = self.debug
return self.results