def create_es_connection(self): ks = KeyStore() es_conf = ks.get_elasticsearch_credentials() self.es = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=False, http_auth=(es_conf["es_user"], es_conf["es_password"]))
def upload_sstables(node, test_data: TestDataInventory, keyspace_name: str = 'keyspace1'): key_store = KeyStore() creds = key_store.get_scylladb_upload_credentials() # Download the sstable files from S3 remote_get_file(node.remoter, test_data.sstable_url, test_data.sstable_file, hash_expected=test_data.sstable_md5, retries=2, user_agent=creds['user_agent']) result = node.remoter.sudo( f"ls -t /var/lib/scylla/data/{keyspace_name}/") upload_dir = result.stdout.split()[0] if node.is_docker(): node.remoter.run( f'tar xvfz {test_data.sstable_file} -C /' f'var/lib/scylla/data/{keyspace_name}/{upload_dir}/upload/') else: node.remoter.sudo( f'tar xvfz {test_data.sstable_file} -C /var/lib/scylla/data/{keyspace_name}/{upload_dir}/upload/', user='******') # Scylla Enterprise 2019.1 doesn't support to load schema.cql and manifest.json, let's remove them node.remoter.sudo( f'rm -f /var/lib/scylla/data/{keyspace_name}/{upload_dir}/upload/schema.cql' ) node.remoter.sudo( f'rm -f /var/lib/scylla/data/{keyspace_name}/{upload_dir}/upload/manifest.json' )
def fix_es_mapping(index_name): ks = KeyStore() es_conf = ks.get_elasticsearch_credentials() mapping_url = "{es_url}/{index_name}/_mapping".format( index_name=index_name, **es_conf) res = requests.get(mapping_url, auth=(es_conf["es_user"], es_conf["es_password"])) output = res.json()[index_name] output['mappings']['test_stats']['dynamic'] = False output['mappings']['test_stats']['properties']['coredumps'] = dict( type='object') output['mappings']['test_stats']['properties']['setup_details'][ 'properties']['db_cluster_details'] = dict(type='object') output['mappings']['test_stats']['properties']['system_details'] = { "dynamic": False, "properties": {} } res = requests.put(mapping_url + "/test_stats", json=output['mappings'], auth=(es_conf["es_user"], es_conf["es_password"])) print(res.text) res.raise_for_status() click.secho("fixed {index_name}".format(index_name=index_name), fg='green')
def create_key_pair(self): LOGGER.info("Creating SCT Key Pair...") if self.sct_keypair: LOGGER.warning(f"SCT Key Pair already exists in {self.region_name}!") else: ks = KeyStore() sct_key_pair = ks.get_ec2_ssh_key_pair() self.resource.import_key_pair(KeyName=self.KEY_PAIR_NAME, # pylint: disable=no-member PublicKeyMaterial=sct_key_pair.public_key) LOGGER.info("SCT Key Pair created.")
def __init__(self, instance_name: str, zone: str): credentials = KeyStore().get_gcp_credentials() self.credentials = service_account.Credentials.from_service_account_info( credentials) self.project_id = credentials['project_id'] self.instance_name = instance_name self.zone = zone
def _get_gcloud_container(self) -> Container: """Create Google Cloud SDK container. Cloud SDK requires to enable some authorization method first. Because of that we start a container which runs forever using `cat' command (like Jenkins do), put a service account credentials and activate them. All consequent gcloud commands run using container.exec_run() method. """ container = ContainerManager.run_container(self, "gcloud") credentials = KeyStore().get_gcp_credentials() credentials["client_email"] = f"{credentials['client_email']}" shell_command = f"umask 077 && echo '{json.dumps(credentials)}' > /tmp/gcloud_svc_account.json" shell_command += " && echo 'kubeletConfig:\n cpuManagerPolicy: static' > /tmp/system_config.yaml" # NOTE: use 'bash' in case of non-alpine sdk image and 'sh' when it is 'alpine' one. res = container.exec_run(["bash", "-c", shell_command]) if res.exit_code: raise DockerException(f"{container}: {res.output.decode('utf-8')}") res = container.exec_run([ "gcloud", "auth", "activate-service-account", credentials["client_email"], "--key-file", "/tmp/gcloud_svc_account.json", "--project", credentials["project_id"] ]) if res.exit_code: raise DockerException( f"{container}[]: {res.output.decode('utf-8')}") return container
def _get_gcloud_container(self) -> Container: """Create Google Cloud SDK container. Cloud SDK requires to enable some authorization method first. Because of that we start a container which runs forever using `cat' command (like Jenkins do), put a service account credentials and activate them. All consequent gcloud commands run using container.exec_run() method. """ container = ContainerManager.run_container(self, "gcloud") credentials = KeyStore().get_gcp_credentials() credentials[ "client_email"] = f"{credentials['project_id']}@appspot.gserviceaccount.com" shell_command = f"umask 077 && echo '{json.dumps(credentials)}' > /tmp/gcloud_svc_account.json" res = container.exec_run(["sh", "-c", shell_command]) if res.exit_code: raise DockerException(f"{container}: {res.output.decode('utf-8')}") res = container.exec_run([ "gcloud", "auth", "activate-service-account", credentials["client_email"], "--key-file", "/tmp/gcloud_svc_account.json", "--project", credentials["project_id"] ]) if res.exit_code: raise DockerException( f"{container}[]: {res.output.decode('utf-8')}") return container
def definition(image_id, image_type): return InstanceDefinition(name="test-vm-1", image_id=image_id, type=image_type, user_name="tester", ssh_key=KeyStore().get_ec2_ssh_key_pair(), tags={'test-tag': 'test_value'}, user_data=[PrintingTestUserDataObject()])
def __init__(self, cloud_instances: CloudInstances, user=None): super().__init__(cloud_instances, static_ips=None, html_template="per_qa_user.html") self.user = user self.report = { 7: defaultdict(list), 5: defaultdict(list), 3: defaultdict(list) } self.qa_users = KeyStore().get_qa_users()
def __init__(self, cloud_instances): super(PerUserSummaryReport, self).__init__(cloud_instances, html_template="per_user_summary.html") self.report = { "results": { "qa": {}, "others": {} }, "cloud_providers": tuple() } self.qa_users = KeyStore().get_qa_users()
def __init__(self, cloud_instances: CloudInstances, static_ips: StaticIPs): super().__init__(cloud_instances, static_ips, html_template="per_user_summary.html") self.report = { "results": { "qa": {}, "others": {} }, "cloud_providers": CLOUD_PROVIDERS } self.qa_users = KeyStore().get_qa_users()
def __init__(self, region_name): self.region_name = region_name info = KeyStore().get_gcp_credentials() self.project = info['project_id'] credentials = service_account.Credentials.from_service_account_info(info) self.iam = build('iam', 'v1', credentials=credentials) self.network_client = compute_v1.NetworksClient(credentials=credentials) self.firewall_client = compute_v1.FirewallsClient(credentials=credentials) self.storage_client = storage.Client(credentials=credentials)
def test_can_create_basic_scylla_instance_definition_from_sct_config(): """Test for azure_region_definition_builder""" EnvConfig = namedtuple('EnvConfig', [ "SCT_CLUSTER_BACKEND", "SCT_TEST_ID", "SCT_CONFIG_FILES", "SCT_AZURE_REGION_NAME", "SCT_N_DB_NODES", "SCT_USER_PREFIX", "SCT_AZURE_IMAGE_DB", "SCT_N_LOADERS", "SCT_N_MONITORS_NODES" ]) env_config = EnvConfig( SCT_CLUSTER_BACKEND="azure", SCT_TEST_ID="example_test_id", SCT_CONFIG_FILES= f'["{Path(__file__).parent.absolute()}/azure_default_config.yaml"]', SCT_AZURE_REGION_NAME="['eastus', 'easteu']", SCT_N_DB_NODES="3 1", SCT_USER_PREFIX="unit-test", SCT_AZURE_IMAGE_DB="some_image_id", SCT_N_LOADERS="2 0", SCT_N_MONITORS_NODES="1") os.environ.update(env_config._asdict()) config = SCTConfiguration() tags = TestConfig.common_tags() ssh_key = KeyStore().get_gce_ssh_key_pair() prefix = config.get('user_prefix') test_config = TestConfig() builder = region_definition_builder.get_builder(params=config, test_config=test_config) region_definitions = builder.build_all_region_definitions() instance_definition = InstanceDefinition( name=f"{prefix}-db-node-eastus-1", image_id=env_config.SCT_AZURE_IMAGE_DB, type="Standard_L8s_v2", user_name="scyllaadm", root_disk_size=30, tags=tags | { "NodeType": "scylla-db", "keep_action": "", 'NodeIndex': '1' }, ssh_key=ssh_key) assert len(region_definitions) == 2 actual_region_definition = region_definitions[0] assert actual_region_definition.test_id == env_config.SCT_TEST_ID assert actual_region_definition.backend == "azure" assert actual_region_definition.region == "eastus" # ignoring user_data in this validation actual_region_definition.definitions[ 0].user_data = instance_definition.user_data # ssh_key is not shown, if actual looks the same as expected possibly ssh_key differ assert instance_definition == actual_region_definition.definitions[0]
def get_all_gce_regions(): from sdcm.keystore import KeyStore gcp_credentials = KeyStore().get_gcp_credentials() gce_driver = get_driver(Provider.GCE) compute_engine = gce_driver(gcp_credentials["project_id"] + "@appspot.gserviceaccount.com", gcp_credentials["private_key"], project=gcp_credentials["project_id"]) all_gce_regions = [ region_obj.name for region_obj in compute_engine.region_list ] return all_gce_regions
def list_instances_gce(tags_dict, running=False): """ list all instances with specific tags GCE :param tags_dict: a dict of the tag to select the instances, e.x. {"TestId": "9bc6879f-b1ef-47e1-99ab-020810aedbcc"} :return: None """ # avoid cyclic dependency issues, since too many things import utils.py from sdcm.keystore import KeyStore gcp_credentials = KeyStore().get_gcp_credentials() gce_driver = get_driver(Provider.GCE) compute_engine = gce_driver(gcp_credentials["project_id"] + "@appspot.gserviceaccount.com", gcp_credentials["private_key"], project=gcp_credentials["project_id"]) logger.info("Going to get all instances from GCE") all_gce_instances = compute_engine.list_nodes() # filter instances by tags since libcloud list_nodes() doesn't offer any filtering if tags_dict: instances = filter_gce_by_tags(tags_dict=tags_dict, instances=all_gce_instances) else: instances = all_gce_instances if running: # https://libcloud.readthedocs.io/en/latest/compute/api.html#libcloud.compute.types.NodeState instances = [i for i in instances if i.state == 'running'] else: instances = [i for i in instances if not i.state == 'terminated'] logger.info("Done. Found total of %s instances.", len(instances)) return instances
def key_pair(): ks = KeyStore() return ks.get_ec2_ssh_key_pair()
def get_docker_hub_credentials() -> dict: LOGGER.info("Get Docker Hub credentials") return KeyStore().get_docker_hub_credentials()
def get_gce_services(regions: list) -> dict: credentials = KeyStore().get_gcp_credentials() return {region_az: _get_gce_service(credentials, region_az) for region_az in map(append_zone, regions)}
def from_keystore_creds(cls) -> "HousekeepingDB": return cls.from_params(KeyStore().get_housekeeping_db_credentials())
def azure_credentials(self) -> dict[str, str]: # pylint: disable=no-self-use; pylint doesn't now about cached_property return KeyStore().get_azure_credentials()
class ES(elasticsearch.Elasticsearch): """ Provides interface for Elasticsearch DB """ def __init__(self): self._conf = self.get_conf() super().__init__(hosts=[self._conf["es_url"]], verify_certs=False, http_auth=(self._conf["es_user"], self._conf["es_password"])) def get_conf(self): self.key_store = KeyStore() return self.key_store.get_elasticsearch_credentials() def _create_index(self, index): self.indices.create(index=index, ignore=400) # pylint: disable=unexpected-keyword-arg def create_doc(self, index, doc_type, doc_id, body): """ Add document in json format """ LOGGER.info('Create doc') LOGGER.info('INDEX: %s', index) LOGGER.info('DOC_TYPE: %s', doc_type) LOGGER.info('DOC_ID: %s', doc_id) LOGGER.info('BODY: %s', body) self._create_index(index) if self.exists(index=index, doc_type=doc_type, id=doc_id): self.update(index=index, doc_type=doc_type, id=doc_id, body={'doc': body}) else: self.create(index=index, doc_type=doc_type, id=doc_id, body=body) def update_doc(self, index, doc_type, doc_id, body): """ Update document with partial data """ LOGGER.info('Update doc %s with info %s', doc_id, body) self.update(index=index, doc_type=doc_type, id=doc_id, body=dict(doc=body)) def get_all(self, index, limit=1000): """ Search for documents for the certain index """ return self.search(index=index, size=limit) # pylint: disable=unexpected-keyword-arg def get_doc(self, index, doc_id, doc_type='_all'): """ Get document by id """ doc = self.get( index=index, doc_type=doc_type, id=doc_id, ignore=[ # pylint: disable=unexpected-keyword-arg 400, 404 ]) if not doc['found']: LOGGER.warning('Document not found: %s %s', doc_id, doc_type) return None return doc def delete_doc(self, index, doc_type, doc_id): """ Delete document """ if self.get_doc(index, doc_id, doc_type): self.delete(index=index, doc_type=doc_type, id=doc_id)
def _retrieve_credentials(self): keystore = KeyStore() creds = keystore.get_email_credentials() self._user = creds["user"] self._password = creds["password"]
def key_pair(self) -> SSHKey: return KeyStore().get_ec2_ssh_key_pair()
def key_pair(self) -> SSHKey: return KeyStore().get_gce_ssh_key_pair() # scylla-test
def _get_ssh_key() -> SSHKey: return KeyStore().get_gce_ssh_key_pair()
def config(cls): if cls._config is None: cls._config = ArgusConfig(**KeyStore().get_argusdb_credentials(), keyspace_name="argus") return cls._config
def get_conf(self): self.key_store = KeyStore() return self.key_store.get_elasticsearch_credentials()
def set_backup_azure_blob_credentials(cls) -> None: cls.backup_azure_blob_credentials = KeyStore().get_backup_azure_blob_credentials()
def migrate(old_index_name, dry_run, new_index, days): # pylint: disable=too-many-locals logging.basicConfig(level=logging.DEBUG) logging.config.dictConfig({ 'version': 1, 'disable_existing_loggers': True, 'formatters': { 'standard': { 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' }, }, 'handlers': { 'default': { 'level': 'INFO', 'formatter': 'standard', 'class': 'logging.StreamHandler', 'stream': 'ext://sys.stdout', # Default is stderr }, }, 'loggers': { '': { # root logger 'handlers': ['default'], 'level': 'WARNING', 'propagate': False }, '__main__': { # if __name__ == '__main__' 'handlers': ['default'], 'level': 'DEBUG', 'propagate': False }, } }) ks = KeyStore() es_conf = ks.get_elasticsearch_credentials() elastic_search = Elasticsearch(hosts=[es_conf["es_url"]], verify_certs=True, http_auth=(es_conf["es_user"], es_conf["es_password"])) if not elastic_search.indices.exists(index=new_index): elastic_search.indices.create(index=new_index) def post_to_new(doc): if dry_run: return elastic_search.index(index=new_index, doc_type='nemesis', body=doc) res = scan(elastic_search, index=old_index_name, query={ "query": { "range": { "test_details.start_time": { "gte": (datetime.datetime.utcnow() - datetime.timedelta(days=days)).timestamp(), "lte": datetime.datetime.utcnow().timestamp(), "boost": 2.0 } } } }, size=300, scroll='3h') for num, hit in enumerate(res): nemesis_list = hit["_source"]["nemesis"] test_data = hit["_source"] LOGGER.info("%s: %s", num, test_data['test_details']['test_id']) if 'scylla-server' not in test_data['versions']: LOGGER.debug("%s: No version for %s - %s", num, test_data['test_details']['test_id'], test_data['test_details']['job_name']) if not test_data['test_details']['job_name']: LOGGER.debug(test_data) continue for nemesis_class, data in nemesis_list.items(): for failure in data['failures']: new_nemesis_data = dict( test_id=test_data['test_details']['test_id'], job_name=test_data['test_details']['job_name'], test_name=test_data['test_details']['test_name'], scylla_version=test_data['versions']['scylla-server'] ['version'], scylla_git_sha=test_data['versions']['scylla-server'] ['commit_id'], ) new_nemesis_data.update( dict(nemesis_name=nemesis_class, nemesis_duration=failure['duration'], start_time=datetime.datetime.utcfromtimestamp( failure['start']), end_time=datetime.datetime.utcfromtimestamp( failure['end']), target_node=failure['node'], outcome="failure", failure_message=failure['error'])) post_to_new(new_nemesis_data) for run in data['runs']: new_nemesis_data = dict( test_id=test_data['test_details']['test_id'], job_name=test_data['test_details']['job_name'], test_name=test_data['test_details']['test_name'], scylla_version=test_data['versions']['scylla-server'] ['version'], scylla_git_sha=test_data['versions']['scylla-server'] ['commit_id'], ) new_nemesis_data.update( dict(nemesis_name=nemesis_class, nemesis_duration=run['duration'], start_time=datetime.datetime.utcfromtimestamp( run['start']), end_time=datetime.datetime.utcfromtimestamp( run['end']), target_node=run['node'], outcome="passed")) if run.get('type', '') == 'skipped': new_nemesis_data['outcome'] = 'skipped' new_nemesis_data['skip_reason'] = run['skip_reason'] post_to_new(new_nemesis_data)
def get_gce_service(region: str) -> GceDriver: credentials = KeyStore().get_gcp_credentials() return _get_gce_service(credentials, append_zone(region))