def create_instance(self, test_id: str, test_duration: int, # pylint: disable=too-many-arguments
instance_type: str = "", root_disk_size_gb: int = 0,
restore_monitor: bool = False, restored_test_id: str = "") -> Any:
LOGGER.info("Creating SCT Runner instance...")
image = self.image
if not image:
LOGGER.error("SCT Runner image was not found in %s! "
"Use `hydra create-runner-image --cloud-provider %s --region %s'",
self.region_name, self.CLOUD_PROVIDER, self.region_name)
return None
tags = {
"TestId": test_id,
"NodeType": self.NODE_TYPE,
"RunByUser": get_username(),
"keep": str(ceil(test_duration / 60) + 6), # keep SCT Runner for 6h more than test_duration
"keep_action": "terminate",
}
if restore_monitor and restored_test_id:
tags.update({"RestoredTestId": restored_test_id})
return self._create_instance(
instance_type=instance_type or self.instance_type(test_duration=test_duration),
base_image=self._get_base_image(self.image),
root_disk_size_gb=root_disk_size_gb,
tags=tags,
instance_name=f"{self.image_name}-{'restored-monitor-' if restore_monitor else ''}instance-{test_id[:8]}",
region_az=self.region_az(
region_name=self.region_name,
availability_zone=self.availability_zone,
),
test_duration=test_duration,
)
def create_instance(self,
test_id: str,
test_duration: int,
instance_type: str = "") -> Any:
LOGGER.info("Creating SCT Runner instance...")
image = self.image
if not image:
LOGGER.error(
"SCT Runner image was not found in %s! "
"Use `hydra create-runner-image --cloud-provider %s --region %s'",
self.region_name, self.CLOUD_PROVIDER, self.region_name)
return None
return self._create_instance(
instance_type=instance_type
or self.instance_type(test_duration=test_duration),
base_image=self._get_base_image(self.image),
tags={
"TestId": test_id,
"NodeType": self.NODE_TYPE,
"RunByUser": get_username(),
"keep":
str(ceil(test_duration / 60) +
6), # keep SCT Runner for 6h more than test_duration
"keep_action": "terminate",
},
instance_name=f"{self.image_name}-instance-{test_id[:8]}",
region_az=self.region_az(
region_name=self.region_name,
availability_zone=self.availability_zone,
),
test_duration=test_duration,
)
def create_instance(self, test_id: str, test_duration: int, region_az: str):
"""
:param test_duration: used to set keep-alive flags, measured in MINUTES
"""
LOGGER.info("Creating SCT Runner instance...")
image = self.image
if not image:
LOGGER.error("SCT Runner image was not found in %s! "
"Use hydra create-runner-image --cloud-privider %s --region %s",
self.region_name, self.cloud_provider, self.region_name)
sys.exit(1)
lt_datetime = datetime.datetime.now(tz=pytz.utc)
return self._create_instance(
instance_type=self.instance_type(test_duration=test_duration),
base_image=self._get_base_image(self.image),
tags_list=[
{"Key": "Name", "Value": self.RUNNER_NAME},
{"Key": "TestId", "Value": test_id},
{"Key": "NodeType", "Value": self.NODE_TYPE},
{"Key": "RunByUser", "Value": get_username()},
{"Key": "keep", "Value": str(ceil(test_duration / 60) + 6)},
{"Key": "keep_action", "Value": "terminate"},
{"Key": "launch_time", "Value": lt_datetime.strftime("%B %d, %Y, %H:%M:%S")},
],
instance_name=f"{self.image_name}-instance-{test_id[:8]}",
region_az=region_az,
test_duration=test_duration,
)
def __init__(self, test_id: str, regions: list[str]):
self._containers = {}
self.tags = {
"TestId": test_id,
"NodeType": AWS_MOCK_NODE_TYPE,
"RunByUser": get_username(),
}
self.regions = regions
self.test_id = test_id
def common_tags(cls) -> Dict[str, str]:
job_name = os.environ.get('JOB_NAME')
tags = dict(RunByUser=get_username(),
TestName=str(cls.test_name()),
TestId=str(cls.test_id()),
version=job_name.split('/', 1)[0] if job_name else "unknown")
build_tag = os.environ.get('BUILD_TAG')
if build_tag:
tags["JenkinsJobTag"] = build_tag
return tags
def from_sct_config(cls, test_id: UUID, test_module_path: str,
sct_config: SCTConfiguration) -> TestRunWithHeartbeat:
# pylint: disable=too-many-locals
if cls.TESTRUN_INSTANCE:
raise ArgusTestRunError("Instance already initialized")
release_name = os.getenv("GIT_BRANCH", get_git_current_branch()).split("/")[-1]
if release_name not in cls.AVAILABLE_RELEASES:
raise ArgusTestRunError("Refusing to track a non-whitelisted branch", release_name, cls.AVAILABLE_RELEASES)
LOGGER.info("Preparing Test Details...")
test_group, *_ = test_module_path.split(".")
config_files = sct_config.get("config_files")
job_name = get_job_name()
job_url = get_job_url()
started_by = get_username()
details = TestDetails(name=get_test_name(), scm_revision_id=get_git_commit_id(), started_by=started_by,
build_job_name=job_name, build_job_url=job_url,
yaml_test_duration=sct_config.get("test_duration"), start_time=int(time.time()),
config_files=config_files, packages=[])
LOGGER.info("Preparing Resource Setup...")
backend = sct_config.get("cluster_backend")
raw_regions = sct_config.get("region_name") or sct_config.get("gce_datacenter") or "undefined_region"
regions = raw_regions.split()
primary_region = regions[0]
sct_runner_info = CloudInstanceDetails(public_ip=get_sct_runner_ip(), provider=backend,
region=primary_region, private_ip=get_my_ip())
cloud_setup = cls.BACKEND_MAP.get(backend, _prepare_unknown_resource_setup)(sct_config)
setup_details = TestResourcesSetup(sct_runner_host=sct_runner_info, region_name=regions,
cloud_setup=cloud_setup)
logs = TestLogs()
resources = TestResources()
results = TestResults(status=TestStatus.CREATED)
run_info = TestRunInfo(details=details, setup=setup_details, resources=resources, logs=logs, results=results)
LOGGER.info("Initializing TestRun...")
cls.TESTRUN_INSTANCE = TestRunWithHeartbeat(test_id=test_id, group=test_group, release_name=release_name,
assignee="",
run_info=run_info,
config=cls.config())
return cls.TESTRUN_INSTANCE
def create_instance(self, test_id: str, test_duration: int,
region_az: str):
"""
:param test_duration: used to set keep-alive flags, measured in MINUTES
"""
LOGGER.info("Creating SCT Runner instance...")
image = self.image
if not image:
LOGGER.error(
f"SCT Runner image was not found in {self.region_name}! "
f"Use hydra create-runner-image --cloud-privider aws --region {self.region_name}"
)
sys.exit(1)
return self._create_instance(
instance_type=self.instance_type(test_duration=test_duration),
image_id=self.image.image_id,
tags_list=[
{
"Key": "Name",
"Value": self.RUNNER_NAME
},
{
"Key": "TestId",
"Value": test_id
},
{
"Key": "NodeType",
"Value": self.NODE_TYPE
},
{
"Key": "RunByUser",
"Value": get_username()
},
{
"Key": "keep",
"Value": str(ceil(test_duration / 60) + 6)
},
{
"Key": "keep_action",
"Value": "terminate"
},
],
region_az=region_az,
)
def tags(self):
return {
"RunByUser": get_username(),
"TestId": self.test_id,
"keep_action": "terminate",
}
def send_email(test_id=None, test_status=None, start_time=None, started_by=None, email_recipients=None, logdir=None):
if started_by is None:
started_by = get_username()
add_file_logger()
from sdcm.send_email import get_running_instances_for_email_report, read_email_data_from_file, build_reporter
if not email_recipients:
LOGGER.warning("No email recipients. Email will not be sent")
return
LOGGER.info('Email will be sent to next recipients: %s', email_recipients)
if not logdir:
logdir = os.path.expanduser('~/sct-results')
test_results = None
if start_time is None:
start_time = format_timestamp(time.time())
else:
start_time = format_timestamp(int(start_time))
testrun_dir = get_testrun_dir(test_id=test_id, base_dir=logdir)
if testrun_dir:
with open(os.path.join(testrun_dir, 'test_id'), 'r') as file:
test_id = file.read().strip()
email_results_file = os.path.join(testrun_dir, "email_data.json")
test_results = read_email_data_from_file(email_results_file)
else:
LOGGER.warning("Failed to find test directory for %s", test_id)
if test_results:
reporter = test_results.get("reporter", "")
test_results['nodes'] = get_running_instances_for_email_report(test_id)
else:
LOGGER.warning("Failed to read test results for %s", test_id)
reporter = "TestAborted"
if not test_status:
test_status = 'ABORTED'
test_results = {
"build_url": os.environ.get("BUILD_URL"),
"subject": f"{test_status}: {os.environ.get('JOB_NAME')}: {start_time}",
"start_time": start_time,
"end_time": format_timestamp(time.time()),
"grafana_screenshots": "",
"grafana_snapshots": "",
"nodes": "",
"test_id": "",
"username": ""
}
if started_by:
test_results["username"] = started_by
if test_id:
test_results.update({
"test_id": test_id,
"nodes": get_running_instances_for_email_report(test_id)
})
test_results['logs_links'] = list_logs_by_test_id(test_results.get('test_id', test_id))
email_recipients = email_recipients.split(',')
reporter = build_reporter(reporter, email_recipients, testrun_dir)
if not reporter:
LOGGER.warning("No reporter found")
sys.exit(1)
try:
reporter.send_report(test_results)
except Exception: # pylint: disable=broad-except
LOGGER.error("Failed to create email due to the following error:\n%s", traceback.format_exc())
build_reporter("TestAborted", email_recipients, testrun_dir).send_report({
"build_url": os.environ.get("BUILD_URL"),
"subject": f"FAILED: {os.environ.get('JOB_NAME')}: {start_time}",
})
def from_sct_config(cls, test_id: UUID,
sct_config: SCTConfiguration) -> TestRunWithHeartbeat:
# pylint: disable=too-many-locals
if cls.TESTRUN_INSTANCE:
raise ArgusTestRunError("Instance already initialized")
LOGGER.info("Preparing Test Details...")
job_name = get_job_name()
job_url = get_job_url()
if job_name == "local_run":
raise ArgusTestRunError("Will not track a locally run job")
config_files = sct_config.get("config_files")
started_by = get_username()
details = TestDetails(
scm_revision_id=get_git_commit_id(),
started_by=started_by,
build_job_url=job_url,
yaml_test_duration=sct_config.get("test_duration"),
start_time=datetime.utcnow().replace(microsecond=0),
config_files=config_files,
packages=[])
LOGGER.info("Preparing Resource Setup...")
backend = sct_config.get("cluster_backend")
region_key = cls.REGION_PROPERTY_MAP.get(
backend, cls.REGION_PROPERTY_MAP["default"])
raw_regions = sct_config.get(region_key) or "undefined_region"
regions = raw_regions.split() if isinstance(raw_regions,
str) else raw_regions
primary_region = regions[0]
sct_runner_info = CloudInstanceDetails(public_ip=get_sct_runner_ip(),
provider=backend,
region=primary_region,
private_ip=get_my_ip())
cloud_setup = cls.BACKEND_MAP.get(
backend, _prepare_unknown_resource_setup)(sct_config)
setup_details = TestResourcesSetup(sct_runner_host=sct_runner_info,
region_name=regions,
cloud_setup=cloud_setup)
logs = TestLogs()
resources = TestResources()
results = TestResults(status=TestStatus.CREATED)
run_info = TestRunInfo(details=details,
setup=setup_details,
resources=resources,
logs=logs,
results=results)
LOGGER.info("Initializing TestRun...")
cls.TESTRUN_INSTANCE = TestRunWithHeartbeat(test_id=test_id,
build_id=job_name,
assignee=None,
run_info=run_info,
config=cls.config())
return cls.TESTRUN_INSTANCE