def create_instance(self, test_id: str, test_duration: int, # pylint: disable=too-many-arguments instance_type: str = "", root_disk_size_gb: int = 0, restore_monitor: bool = False, restored_test_id: str = "") -> Any: LOGGER.info("Creating SCT Runner instance...") image = self.image if not image: LOGGER.error("SCT Runner image was not found in %s! " "Use `hydra create-runner-image --cloud-provider %s --region %s'", self.region_name, self.CLOUD_PROVIDER, self.region_name) return None tags = { "TestId": test_id, "NodeType": self.NODE_TYPE, "RunByUser": get_username(), "keep": str(ceil(test_duration / 60) + 6), # keep SCT Runner for 6h more than test_duration "keep_action": "terminate", } if restore_monitor and restored_test_id: tags.update({"RestoredTestId": restored_test_id}) return self._create_instance( instance_type=instance_type or self.instance_type(test_duration=test_duration), base_image=self._get_base_image(self.image), root_disk_size_gb=root_disk_size_gb, tags=tags, instance_name=f"{self.image_name}-{'restored-monitor-' if restore_monitor else ''}instance-{test_id[:8]}", region_az=self.region_az( region_name=self.region_name, availability_zone=self.availability_zone, ), test_duration=test_duration, )
def create_instance(self, test_id: str, test_duration: int, instance_type: str = "") -> Any: LOGGER.info("Creating SCT Runner instance...") image = self.image if not image: LOGGER.error( "SCT Runner image was not found in %s! " "Use `hydra create-runner-image --cloud-provider %s --region %s'", self.region_name, self.CLOUD_PROVIDER, self.region_name) return None return self._create_instance( instance_type=instance_type or self.instance_type(test_duration=test_duration), base_image=self._get_base_image(self.image), tags={ "TestId": test_id, "NodeType": self.NODE_TYPE, "RunByUser": get_username(), "keep": str(ceil(test_duration / 60) + 6), # keep SCT Runner for 6h more than test_duration "keep_action": "terminate", }, instance_name=f"{self.image_name}-instance-{test_id[:8]}", region_az=self.region_az( region_name=self.region_name, availability_zone=self.availability_zone, ), test_duration=test_duration, )
def create_instance(self, test_id: str, test_duration: int, region_az: str): """ :param test_duration: used to set keep-alive flags, measured in MINUTES """ LOGGER.info("Creating SCT Runner instance...") image = self.image if not image: LOGGER.error("SCT Runner image was not found in %s! " "Use hydra create-runner-image --cloud-privider %s --region %s", self.region_name, self.cloud_provider, self.region_name) sys.exit(1) lt_datetime = datetime.datetime.now(tz=pytz.utc) return self._create_instance( instance_type=self.instance_type(test_duration=test_duration), base_image=self._get_base_image(self.image), tags_list=[ {"Key": "Name", "Value": self.RUNNER_NAME}, {"Key": "TestId", "Value": test_id}, {"Key": "NodeType", "Value": self.NODE_TYPE}, {"Key": "RunByUser", "Value": get_username()}, {"Key": "keep", "Value": str(ceil(test_duration / 60) + 6)}, {"Key": "keep_action", "Value": "terminate"}, {"Key": "launch_time", "Value": lt_datetime.strftime("%B %d, %Y, %H:%M:%S")}, ], instance_name=f"{self.image_name}-instance-{test_id[:8]}", region_az=region_az, test_duration=test_duration, )
def __init__(self, test_id: str, regions: list[str]): self._containers = {} self.tags = { "TestId": test_id, "NodeType": AWS_MOCK_NODE_TYPE, "RunByUser": get_username(), } self.regions = regions self.test_id = test_id
def common_tags(cls) -> Dict[str, str]: job_name = os.environ.get('JOB_NAME') tags = dict(RunByUser=get_username(), TestName=str(cls.test_name()), TestId=str(cls.test_id()), version=job_name.split('/', 1)[0] if job_name else "unknown") build_tag = os.environ.get('BUILD_TAG') if build_tag: tags["JenkinsJobTag"] = build_tag return tags
def from_sct_config(cls, test_id: UUID, test_module_path: str, sct_config: SCTConfiguration) -> TestRunWithHeartbeat: # pylint: disable=too-many-locals if cls.TESTRUN_INSTANCE: raise ArgusTestRunError("Instance already initialized") release_name = os.getenv("GIT_BRANCH", get_git_current_branch()).split("/")[-1] if release_name not in cls.AVAILABLE_RELEASES: raise ArgusTestRunError("Refusing to track a non-whitelisted branch", release_name, cls.AVAILABLE_RELEASES) LOGGER.info("Preparing Test Details...") test_group, *_ = test_module_path.split(".") config_files = sct_config.get("config_files") job_name = get_job_name() job_url = get_job_url() started_by = get_username() details = TestDetails(name=get_test_name(), scm_revision_id=get_git_commit_id(), started_by=started_by, build_job_name=job_name, build_job_url=job_url, yaml_test_duration=sct_config.get("test_duration"), start_time=int(time.time()), config_files=config_files, packages=[]) LOGGER.info("Preparing Resource Setup...") backend = sct_config.get("cluster_backend") raw_regions = sct_config.get("region_name") or sct_config.get("gce_datacenter") or "undefined_region" regions = raw_regions.split() primary_region = regions[0] sct_runner_info = CloudInstanceDetails(public_ip=get_sct_runner_ip(), provider=backend, region=primary_region, private_ip=get_my_ip()) cloud_setup = cls.BACKEND_MAP.get(backend, _prepare_unknown_resource_setup)(sct_config) setup_details = TestResourcesSetup(sct_runner_host=sct_runner_info, region_name=regions, cloud_setup=cloud_setup) logs = TestLogs() resources = TestResources() results = TestResults(status=TestStatus.CREATED) run_info = TestRunInfo(details=details, setup=setup_details, resources=resources, logs=logs, results=results) LOGGER.info("Initializing TestRun...") cls.TESTRUN_INSTANCE = TestRunWithHeartbeat(test_id=test_id, group=test_group, release_name=release_name, assignee="", run_info=run_info, config=cls.config()) return cls.TESTRUN_INSTANCE
def create_instance(self, test_id: str, test_duration: int, region_az: str): """ :param test_duration: used to set keep-alive flags, measured in MINUTES """ LOGGER.info("Creating SCT Runner instance...") image = self.image if not image: LOGGER.error( f"SCT Runner image was not found in {self.region_name}! " f"Use hydra create-runner-image --cloud-privider aws --region {self.region_name}" ) sys.exit(1) return self._create_instance( instance_type=self.instance_type(test_duration=test_duration), image_id=self.image.image_id, tags_list=[ { "Key": "Name", "Value": self.RUNNER_NAME }, { "Key": "TestId", "Value": test_id }, { "Key": "NodeType", "Value": self.NODE_TYPE }, { "Key": "RunByUser", "Value": get_username() }, { "Key": "keep", "Value": str(ceil(test_duration / 60) + 6) }, { "Key": "keep_action", "Value": "terminate" }, ], region_az=region_az, )
def tags(self): return { "RunByUser": get_username(), "TestId": self.test_id, "keep_action": "terminate", }
def send_email(test_id=None, test_status=None, start_time=None, started_by=None, email_recipients=None, logdir=None): if started_by is None: started_by = get_username() add_file_logger() from sdcm.send_email import get_running_instances_for_email_report, read_email_data_from_file, build_reporter if not email_recipients: LOGGER.warning("No email recipients. Email will not be sent") return LOGGER.info('Email will be sent to next recipients: %s', email_recipients) if not logdir: logdir = os.path.expanduser('~/sct-results') test_results = None if start_time is None: start_time = format_timestamp(time.time()) else: start_time = format_timestamp(int(start_time)) testrun_dir = get_testrun_dir(test_id=test_id, base_dir=logdir) if testrun_dir: with open(os.path.join(testrun_dir, 'test_id'), 'r') as file: test_id = file.read().strip() email_results_file = os.path.join(testrun_dir, "email_data.json") test_results = read_email_data_from_file(email_results_file) else: LOGGER.warning("Failed to find test directory for %s", test_id) if test_results: reporter = test_results.get("reporter", "") test_results['nodes'] = get_running_instances_for_email_report(test_id) else: LOGGER.warning("Failed to read test results for %s", test_id) reporter = "TestAborted" if not test_status: test_status = 'ABORTED' test_results = { "build_url": os.environ.get("BUILD_URL"), "subject": f"{test_status}: {os.environ.get('JOB_NAME')}: {start_time}", "start_time": start_time, "end_time": format_timestamp(time.time()), "grafana_screenshots": "", "grafana_snapshots": "", "nodes": "", "test_id": "", "username": "" } if started_by: test_results["username"] = started_by if test_id: test_results.update({ "test_id": test_id, "nodes": get_running_instances_for_email_report(test_id) }) test_results['logs_links'] = list_logs_by_test_id(test_results.get('test_id', test_id)) email_recipients = email_recipients.split(',') reporter = build_reporter(reporter, email_recipients, testrun_dir) if not reporter: LOGGER.warning("No reporter found") sys.exit(1) try: reporter.send_report(test_results) except Exception: # pylint: disable=broad-except LOGGER.error("Failed to create email due to the following error:\n%s", traceback.format_exc()) build_reporter("TestAborted", email_recipients, testrun_dir).send_report({ "build_url": os.environ.get("BUILD_URL"), "subject": f"FAILED: {os.environ.get('JOB_NAME')}: {start_time}", })
def from_sct_config(cls, test_id: UUID, sct_config: SCTConfiguration) -> TestRunWithHeartbeat: # pylint: disable=too-many-locals if cls.TESTRUN_INSTANCE: raise ArgusTestRunError("Instance already initialized") LOGGER.info("Preparing Test Details...") job_name = get_job_name() job_url = get_job_url() if job_name == "local_run": raise ArgusTestRunError("Will not track a locally run job") config_files = sct_config.get("config_files") started_by = get_username() details = TestDetails( scm_revision_id=get_git_commit_id(), started_by=started_by, build_job_url=job_url, yaml_test_duration=sct_config.get("test_duration"), start_time=datetime.utcnow().replace(microsecond=0), config_files=config_files, packages=[]) LOGGER.info("Preparing Resource Setup...") backend = sct_config.get("cluster_backend") region_key = cls.REGION_PROPERTY_MAP.get( backend, cls.REGION_PROPERTY_MAP["default"]) raw_regions = sct_config.get(region_key) or "undefined_region" regions = raw_regions.split() if isinstance(raw_regions, str) else raw_regions primary_region = regions[0] sct_runner_info = CloudInstanceDetails(public_ip=get_sct_runner_ip(), provider=backend, region=primary_region, private_ip=get_my_ip()) cloud_setup = cls.BACKEND_MAP.get( backend, _prepare_unknown_resource_setup)(sct_config) setup_details = TestResourcesSetup(sct_runner_host=sct_runner_info, region_name=regions, cloud_setup=cloud_setup) logs = TestLogs() resources = TestResources() results = TestResults(status=TestStatus.CREATED) run_info = TestRunInfo(details=details, setup=setup_details, resources=resources, logs=logs, results=results) LOGGER.info("Initializing TestRun...") cls.TESTRUN_INSTANCE = TestRunWithHeartbeat(test_id=test_id, build_id=job_name, assignee=None, run_info=run_info, config=cls.config()) return cls.TESTRUN_INSTANCE