def apply_oc_resource(
template_name,
cluster_path,
_templating,
template_data=None,
template_dir="ocs-deployment",
):
"""
Apply an oc resource after rendering the specified template with
the rook data from cluster_conf.
Args:
template_name (str): Name of the ocs-deployment config template
cluster_path (str): Path to cluster directory, where files will be
written
_templating (Templating): Object of Templating class used for
templating
template_data (dict): Data for render template (default: {})
template_dir (str): Directory under templates dir where template
exists (default: ocs-deployment)
"""
if template_data is None:
template_data = {}
template_path = os.path.join(template_dir, template_name)
template = _templating.render_template(template_path, template_data)
cfg_file = os.path.join(cluster_path, template_name)
with open(cfg_file, "w") as f:
f.write(template)
log.info(f"Applying rook resource from {template_name}")
occli = OCP()
occli.apply(cfg_file)
class ElasticSearch(object):
"""
ElasticSearch Environment
"""
def __init__(self):
"""
Initializer function
"""
log.info("Initializing the Elastic-Search environment object")
self.namespace = "elastic-system"
self.eck_file = "ocs_ci/templates/app-pods/eck.1.3.1-all-in-one.yaml"
self.dumper_file = "ocs_ci/templates/app-pods/esclient.yaml"
self.pvc = "ocs_ci/templates/app-pods/es-pvc.yaml"
self.crd = "ocs_ci/templates/app-pods/esq.yaml"
# Creating some different types of OCP objects
self.ocp = OCP(kind="pod",
resource_name="elastic-operator-0",
namespace=self.namespace)
self.ns_obj = OCP(kind="namespace", namespace=self.namespace)
self.es = OCP(resource_name="quickstart-es-http",
namespace=self.namespace)
self.elasticsearch = OCP(namespace=self.namespace,
kind="elasticsearch")
self.password = OCP(
kind="secret",
resource_name="quickstart-es-elastic-user",
namespace=self.namespace,
)
# Deploy the ECK all-in-one.yaml file
self._deploy_eck()
# Deploy the Elastic-Search server
self._deploy_es()
# Verify that ES is Up & Running
timeout = 600
while timeout > 0:
if self.get_health():
log.info("The ElasticSearch server is ready !")
break
else:
log.warning("The ElasticSearch server is not ready yet")
log.info("going to sleep for 30 sec. before next check")
time.sleep(30)
timeout -= 30
self._deploy_data_dumper_client()
# Connect to the server
self.con = self._es_connect()
def _deploy_eck(self):
"""
Deploying the ECK environment for the Elasticsearch, and make sure it
is in Running mode
"""
log.info("Deploying the ECK environment for the ES cluster")
self.ocp.apply(self.eck_file)
for es_pod in TimeoutSampler(300, 10, get_pod_name_by_pattern,
"elastic-operator", self.namespace):
try:
if es_pod[0] is not None:
self.eckpod = es_pod[0]
log.info(f"The ECK pod {self.eckpod} is ready !")
break
except IndexError:
log.info("ECK operator pod not ready yet")
def _deploy_data_dumper_client(self):
"""
Deploying elastic search client pod with utility which dump all the data
from the server to .tgz file
"""
log.info("Deploying the es client for dumping all data")
self.ocp.apply(self.dumper_file)
for dmp_pod in TimeoutSampler(300, 10, get_pod_name_by_pattern,
"es-dumper", self.namespace):
try:
if dmp_pod[0] is not None:
self.dump_pod = dmp_pod[0]
log.info(
f"The dumper client pod {self.dump_pod} is ready !")
break
except IndexError:
log.info("Dumper pod not ready yet")
def get_ip(self):
"""
This function return the IP address of the Elasticsearch cluster.
this IP is to use inside the OCP cluster
Return
str : String that represent the Ip Address.
"""
return self.es.get()["spec"]["clusterIP"]
def get_port(self):
"""
This function return the port of the Elasticsearch cluster.
Return
str : String that represent the port.
"""
return self.es.get()["spec"]["ports"][0]["port"]
def _deploy_es(self):
log.info("Deploy the PVC for the ElasticSearch cluster")
self.ocp.apply(self.pvc)
log.info("Deploy the ElasticSearch cluster")
self.ocp.apply(self.crd)
for es_pod in TimeoutSampler(300, 20, get_pod_name_by_pattern,
"quickstart-es-default", self.namespace):
try:
if es_pod[0] is not None:
self.espod = es_pod[0]
log.info(f"The ElasticSearch pod {self.espod} Started")
break
except IndexError:
log.info("elasticsearch pod not ready yet")
es_pod = OCP(kind="pod", namespace=self.namespace)
log.info("Waiting for ElasticSearch to Run")
assert es_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=self.espod,
sleep=30,
timeout=600,
)
log.info("Elastic Search is ready !!!")
def get_health(self):
"""
This method return the health status of the Elasticsearch.
Returns:
bool : True if the status is green (OK) otherwise - False
"""
return self.elasticsearch.get(
)["items"][0]["status"]["health"] == "green"
def get_password(self):
"""
This method return the password used to connect the Elasticsearch.
Returns:
str : The password as text
"""
return base64.b64decode(
self.password.get()["data"]["elastic"]).decode("utf-8")
def cleanup(self):
"""
Cleanup the environment from all Elasticsearch components, and from the
port forwarding process.
"""
log.info("Teardown the Elasticsearch environment")
log.info("Deleting all resources")
log.info("Deleting the dumper client pod")
self.ocp.delete(yaml_file=self.dumper_file)
log.info("Deleting the es resource")
self.ocp.delete(yaml_file=self.crd)
log.info("Deleting the es project")
self.ns_obj.delete_project(project_name=self.namespace)
self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=180)
def _es_connect(self):
"""
Create a connection to the local ES
Returns:
Elasticsearch: elasticsearch connection object
Raise:
ConnectionError: if can not connect to the server
"""
try:
es = Elasticsearch([{
"host": self.get_ip(),
"port": self.get_port()
}])
except esexp.ConnectionError:
log.error("Can not connect to ES server in the LocalServer")
raise
return es
def get_indices(self):
"""
Getting list of all indices in the ES server - all created by the test,
the installation of the ES was without any indexes pre-installed.
Returns:
list : list of all indices defined in the ES server
"""
results = []
log.info("Getting all indices")
for ind in self.con.indices.get_alias("*"):
results.append(ind)
return results
def _copy(self, es):
"""
Copy All data from the internal ES server to the main ES.
**This is deprecated function** , use the dump function, and load
the data from the files for the main ES server
Args:
es (obj): elasticsearch object which connected to the main ES
"""
query = {"size": 1000, "query": {"match_all": {}}}
for ind in self.get_indices():
log.info(f"Reading {ind} from internal ES server")
try:
result = self.con.search(index=ind, body=query)
except esexp.NotFoundError:
log.warning(f"{ind} Not found in the Internal ES.")
continue
log.debug(f"The results from internal ES for {ind} are :{result}")
log.info(f"Writing {ind} into main ES server")
for doc in result["hits"]["hits"]:
log.debug(f"Going to write : {doc}")
es.index(index=ind, doc_type="_doc", body=doc["_source"])
def dumping_all_data(self, target_path):
"""
Dump All data from the internal ES server to .tgz file.
Args:
target_path (str): the path where the results file will be copy into
Return:
bool: True if the dump operation succeed and return the results data to the host
otherwise False
"""
log.info("dumping data from ES server to .tgz file")
rsh_cmd = f"rsh {self.dump_pod} /elasticsearch-dump/esdumper.py --ip {self.get_ip()} --port {self.get_port()}"
result = self.ocp.exec_oc_cmd(rsh_cmd,
out_yaml_format=False,
timeout=1200)
if "ES dump is done." not in result:
log.error("There is no data in the Elasticsearch server")
return False
else:
src_file = result.split()[-1]
log.info(f"Copy {src_file} from the client pod")
cp_command = f"cp {self.dump_pod}:{src_file} {target_path}/FullResults.tgz"
result = self.ocp.exec_oc_cmd(cp_command, timeout=120)
log.info(f"The output from the POD is {result}")
log.info("Extracting the FullResults.tgz file")
kwargs = {"cwd": target_path}
results = run_command(f"tar zxvf {target_path}/FullResults.tgz",
**kwargs)
log.debug(f"The untar results is {results}")
if "Error in command" in results:
log.warning("Can not untar the dumped file")
return False
return True
class ElasticSearch(object):
"""
ElasticSearch Environment
"""
def __init__(self):
"""
Initializer function
"""
log.info('Initializing the Elastic-Search environment object')
self.namespace = "elastic-system"
self.eck_path = "https://download.elastic.co/downloads/eck/1.1.2"
self.eck_file = "all-in-one.yaml"
self.pvc = "ocs_ci/templates/app-pods/es-pvc.yaml"
self.crd = "ocs_ci/templates/app-pods/esq.yaml"
self.lspid = None
# Creating some different types of OCP objects
self.ocp = OCP(
kind="pod",
resource_name="elastic-operator-0",
namespace=self.namespace
)
self.ns_obj = OCP(kind='namespace', namespace=self.namespace)
self.es = OCP(
resource_name="quickstart-es-http", namespace=self.namespace
)
self.elasticsearch = OCP(namespace=self.namespace, kind='elasticsearch')
self.password = OCP(
kind='secret',
resource_name='quickstart-es-elastic-user',
namespace=self.namespace
)
# Fetch the all-in-one.yaml from the official repository
self._get_eck_file()
# Deploy the ECK all-in-one.yaml file
self._deploy_eck()
# Deploy the Elastic-Search server
self._deploy_es()
# Verify that ES is Up & Running
timeout = 600
while timeout > 0:
if self.get_health():
log.info('The ElasticSearch server is ready !')
break
else:
log.warning('The ElasticSearch server is not ready yet')
log.info('going to sleep gor 30 sec. before next check')
time.sleep(30)
timeout -= 30
# Starting LocalServer process - port forwarding
self.local_server()
def _get_eck_file(self):
"""
Getting the ECK file from the official Elasticsearch web site and store
it as a temporary file.
Current version is 1.1.2, this need to be update with new versions,
after testing it, and also it may need to update the CRD file (esq.yaml)
with the new version as well.
"""
self.dir = tempfile.mkdtemp(prefix='elastic-system_')
src_file = f'{self.eck_path}/{self.eck_file}'
trg_file = f'{self.dir}/{self.eck_file}'
log.info(f'Retrieving the ECK CR file from {src_file} into {trg_file}')
try:
urllib.request.urlretrieve(src_file, trg_file)
except urllib.error.HTTPError as e:
log.error(f'Can not connect to {src_file} : {e}')
raise e
def _deploy_eck(self):
"""
Deploying the ECK environment for the Elasticsearch, and make sure it
is in Running mode
"""
log.info('Deploying the ECK environment for the ES cluster')
self.ocp.apply(f'{self.dir}/{self.eck_file}')
for es_pod in TimeoutSampler(
300, 10, get_pod_name_by_pattern, 'elastic-operator', self.namespace
):
try:
if es_pod[0] is not None:
self.eckpod = es_pod[0]
log.info(f'The ECK pod {self.eckpod} is ready !')
break
except IndexError:
log.info('ECK operator pod not ready yet')
def get_ip(self):
"""
This function return the IP address of the Elasticsearch cluster.
this IP is to use inside the OCP cluster
Return
str : String that represent the Ip Address.
"""
return self.es.get()["spec"]["clusterIP"]
def get_port(self):
"""
This function return the port of the Elasticsearch cluster.
Return
str : String that represent the port.
"""
return self.es.get()["spec"]["ports"][0]["port"]
def _deploy_es(self):
log.info('Deploy the PVC for the ElasticSearch cluster')
self.ocp.apply(self.pvc)
log.info('Deploy the ElasticSearch cluster')
self.ocp.apply(self.crd)
for es_pod in TimeoutSampler(
300, 20, get_pod_name_by_pattern, 'quickstart-es-default', self.namespace
):
try:
if es_pod[0] is not None:
self.espod = es_pod[0]
log.info(f'The ElasticSearch pod {self.espod} Started')
break
except IndexError:
log.info('elasticsearch pod not ready yet')
es_pod = OCP(kind='pod', namespace=self.namespace)
log.info('Waiting for ElasticSearch to Run')
assert es_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=self.espod,
sleep=30,
timeout=600
)
log.info('Elastic Search is ready !!!')
def get_health(self):
"""
This method return the health status of the Elasticsearch.
Returns:
bool : True if the status is green (OK) otherwise - False
"""
return self.elasticsearch.get()['items'][0]['status']['health'] == 'green'
def get_password(self):
"""
This method return the password used to connect the Elasticsearch.
Returns:
str : The password as text
"""
return base64.b64decode(self.password.get()['data']['elastic']).decode('utf-8')
def cleanup(self):
"""
Cleanup the environment from all Elasticsearch components, and from the
port forwarding process.
"""
log.info('Teardown the Elasticsearch environment')
log.info(f'Killing the local server process ({self.lspid})')
os.kill(self.lspid, signal.SIGKILL)
log.info('Deleting all resources')
subprocess.run(f'oc delete -f {self.crd}', shell=True)
subprocess.run(f'oc delete -f {self.eck_file}', shell=True, cwd=self.dir)
self.ns_obj.wait_for_delete(resource_name=self.namespace)
def local_server(self):
"""
Starting sub-process that will do port-forwarding, to allow access from
outside the open-shift cluster into the Elasticsearch server.
"""
cmd = f'oc -n {self.namespace } '
cmd += f'port-forward service/quickstart-es-http {self.get_port()}'
log.info(f'Going to run : {cmd}')
proc = subprocess.Popen(cmd, shell=True)
log.info(f'Starting LocalServer with PID of {proc.pid}')
self.lspid = proc.pid
class OCS(object):
"""
Base OCSClass
"""
def __init__(self, **kwargs):
"""
Initializer function
Args:
kwargs (dict):
1) For existing resource, use OCP.reload() to get the
resource's dictionary and use it to pass as **kwargs
2) For new resource, use yaml files templates under
/templates/CSI like:
obj_dict = load_yaml(
os.path.join(
TEMPLATE_DIR, "some_resource.yaml"
)
)
"""
self.data = kwargs
self._api_version = self.data.get('api_version')
self._kind = self.data.get('kind')
self._namespace = None
if 'metadata' in self.data:
self._namespace = self.data.get('metadata').get('namespace')
self._name = self.data.get('metadata').get('name')
self.ocp = OCP(api_version=self._api_version,
kind=self.kind,
namespace=self._namespace)
self.temp_yaml = tempfile.NamedTemporaryFile(mode='w+',
prefix=self._kind,
delete=False)
# This _is_delete flag is set to True if the delete method was called
# on object of this class and was successfull.
self._is_deleted = False
@property
def api_version(self):
return self._api_version
@property
def kind(self):
return self._kind
@property
def namespace(self):
return self._namespace
@property
def name(self):
return self._name
@property
def is_deleted(self):
return self._is_deleted
def reload(self):
"""
Reloading the OCS instance with the new information from its actual
data.
After creating a resource from a yaml file, the actual yaml file is
being changed and more information about the resource is added.
"""
self.data = self.get()
self.__init__(**self.data)
def get(self, out_yaml_format=True):
return self.ocp.get(resource_name=self.name,
out_yaml_format=out_yaml_format)
def describe(self):
return self.ocp.describe(resource_name=self.name)
def create(self, do_reload=True):
log.info(f"Adding {self.kind} with name {self.name}")
templating.dump_data_to_temp_yaml(self.data, self.temp_yaml.name)
status = self.ocp.create(yaml_file=self.temp_yaml.name)
if do_reload:
self.reload()
return status
def delete(self, wait=True, force=False):
"""
Delete the OCS object if its not already deleted
(using the internal is_deleted flag)
Args:
wait (bool): Wait for object to be deleted
force (bool): Force delete object
Returns:
bool: True if deleted, False otherwise
"""
# Avoid accidental delete of default storageclass and secret
if (self.name == constants.DEFAULT_STORAGECLASS_CEPHFS
or self.name == constants.DEFAULT_STORAGECLASS_RBD):
log.info(f"Attempt to delete default Secret or StorageClass")
return
if self._is_deleted:
log.info(f"Attempt to remove resource: {self.name} which is"
f"already deleted! Skipping delete of this resource!")
result = True
else:
result = self.ocp.delete(resource_name=self.name,
wait=wait,
force=force)
self._is_deleted = True
return result
def apply(self, **data):
with open(self.temp_yaml.name, 'w') as yaml_file:
yaml.dump(data, yaml_file)
assert self.ocp.apply(
yaml_file=self.temp_yaml.name), (f"Failed to apply changes {data}")
self.reload()
def add_label(self, label):
"""
Addss a new label
Args:
label (str): New label to be assigned for this pod
E.g: "label=app='rook-ceph-mds'"
"""
status = self.ocp.add_label(resource_name=self.name, label=label)
self.reload()
return status
def delete_temp_yaml_file(self):
utils.delete_file(self.temp_yaml.name)
def test_upgrade():
ceph_cluster = CephCluster()
with CephHealthMonitor(ceph_cluster):
namespace = config.ENV_DATA['cluster_namespace']
version_before_upgrade = config.ENV_DATA.get("ocs_version")
upgrade_version = config.UPGRADE.get("upgrade_ocs_version",
version_before_upgrade)
ocs_registry_image = config.UPGRADE.get('upgrade_ocs_registry_image')
if ocs_registry_image:
upgrade_version = get_ocs_version_from_image(ocs_registry_image)
parsed_version_before_upgrade = parse_version(version_before_upgrade)
parsed_upgrade_version = parse_version(upgrade_version)
assert parsed_upgrade_version >= parsed_version_before_upgrade, (
f"Version you would like to upgrade to: {upgrade_version} "
f"is not higher or equal to the version you currently running: "
f"{version_before_upgrade}")
operator_selector = get_selector_for_ocs_operator()
package_manifest = PackageManifest(
resource_name=OCS_OPERATOR_NAME,
selector=operator_selector,
)
channel = config.DEPLOYMENT.get('ocs_csv_channel')
csv_name_pre_upgrade = package_manifest.get_current_csv(channel)
log.info(f"CSV name before upgrade is: {csv_name_pre_upgrade}")
csv_pre_upgrade = CSV(resource_name=csv_name_pre_upgrade,
namespace=namespace)
pre_upgrade_images = get_images(csv_pre_upgrade.get())
version_change = parsed_upgrade_version > parsed_version_before_upgrade
if version_change:
version_config_file = os.path.join(constants.CONF_DIR,
'ocs_version',
f'ocs-{upgrade_version}.yaml')
load_config_file(version_config_file)
ocs_catalog = CatalogSource(
resource_name=constants.OPERATOR_CATALOG_SOURCE_NAME,
namespace=constants.MARKETPLACE_NAMESPACE,
)
if not ocs_catalog.is_exist():
log.info("OCS catalog source doesn't exist. Creating new one.")
create_catalog_source(ocs_registry_image, ignore_upgrade=True)
image_url = ocs_catalog.get_image_url()
image_tag = ocs_catalog.get_image_name()
log.info(f"Current image is: {image_url}, tag: {image_tag}")
if ocs_registry_image:
image_url, new_image_tag = ocs_registry_image.split(':')
elif config.UPGRADE.get('upgrade_to_latest', True) or version_change:
new_image_tag = get_latest_ds_olm_tag()
else:
new_image_tag = get_next_version_available_for_upgrade(image_tag)
cs_data = deepcopy(ocs_catalog.data)
image_for_upgrade = ':'.join([image_url, new_image_tag])
log.info(f"Image: {image_for_upgrade} will be used for upgrade.")
cs_data['spec']['image'] = image_for_upgrade
with NamedTemporaryFile() as cs_yaml:
dump_data_to_temp_yaml(cs_data, cs_yaml.name)
ocs_catalog.apply(cs_yaml.name)
# Wait for the new package manifest for upgrade.
operator_selector = get_selector_for_ocs_operator()
package_manifest = PackageManifest(
resource_name=OCS_OPERATOR_NAME,
selector=operator_selector,
)
package_manifest.wait_for_resource()
channel = config.DEPLOYMENT.get('ocs_csv_channel')
if not channel:
channel = package_manifest.get_default_channel()
# update subscription
subscription = OCP(
resource_name=constants.OCS_SUBSCRIPTION,
kind='subscription',
namespace=config.ENV_DATA['cluster_namespace'],
)
subscription_data = deepcopy(subscription.data)
subscription_data['spec']['channel'] = channel
# TODO: once we GA 4.3 we will make possibility to upgrade from live
# 4.2 to live 4.3.
subscription_data['spec'][
'source'] = constants.OPERATOR_CATALOG_SOURCE_NAME
with NamedTemporaryFile() as subscription_yaml:
dump_data_to_temp_yaml(subscription_data, subscription_yaml.name)
subscription.apply(subscription_yaml.name)
subscription_plan_approval = config.DEPLOYMENT.get(
'subscription_plan_approval')
if subscription_plan_approval == 'Manual':
wait_for_install_plan_and_approve(namespace)
attempts = 145
for attempt in range(1, attempts):
if attempts == attempt:
raise TimeoutException("No new CSV found after upgrade!")
log.info(f"Attempt {attempt}/{attempts} to check CSV upgraded.")
csv_name_post_upgrade = package_manifest.get_current_csv(channel)
if csv_name_post_upgrade == csv_name_pre_upgrade:
log.info(f"CSV is still: {csv_name_post_upgrade}")
sleep(5)
else:
log.info(f"CSV now upgraded to: {csv_name_post_upgrade}")
break
csv_post_upgrade = CSV(resource_name=csv_name_post_upgrade,
namespace=namespace)
log.info(
f"Waiting for CSV {csv_name_post_upgrade} to be in succeeded state"
)
if version_before_upgrade == '4.2' and upgrade_version == '4.3':
log.info("Force creating Ceph toolbox after upgrade 4.2 -> 4.3")
setup_ceph_toolbox(force_setup=True)
csv_post_upgrade.wait_for_phase("Succeeded", timeout=600)
post_upgrade_images = get_images(csv_post_upgrade.get())
old_images, _, _ = get_upgrade_image_info(pre_upgrade_images,
post_upgrade_images)
verify_image_versions(old_images, parsed_upgrade_version)
ocs_install_verification(
timeout=600,
skip_osd_distribution_check=True,
ocs_registry_image=ocs_registry_image,
)
class ElasticSearch(object):
"""
ElasticSearch Environment
"""
def __init__(self):
"""
Initializer function
"""
log.info("Initializing the Elastic-Search environment object")
self.namespace = "elastic-system"
self.eck_path = "https://download.elastic.co/downloads/eck/1.1.2"
self.eck_file = "all-in-one.yaml"
self.pvc = "ocs_ci/templates/app-pods/es-pvc.yaml"
self.crd = "ocs_ci/templates/app-pods/esq.yaml"
self.lspid = None
# Creating some different types of OCP objects
self.ocp = OCP(kind="pod",
resource_name="elastic-operator-0",
namespace=self.namespace)
self.ns_obj = OCP(kind="namespace", namespace=self.namespace)
self.es = OCP(resource_name="quickstart-es-http",
namespace=self.namespace)
self.elasticsearch = OCP(namespace=self.namespace,
kind="elasticsearch")
self.password = OCP(
kind="secret",
resource_name="quickstart-es-elastic-user",
namespace=self.namespace,
)
# Fetch the all-in-one.yaml from the official repository
self._get_eck_file()
# Deploy the ECK all-in-one.yaml file
self._deploy_eck()
# Deploy the Elastic-Search server
self._deploy_es()
# Verify that ES is Up & Running
timeout = 600
while timeout > 0:
if self.get_health():
log.info("The ElasticSearch server is ready !")
break
else:
log.warning("The ElasticSearch server is not ready yet")
log.info("going to sleep for 30 sec. before next check")
time.sleep(30)
timeout -= 30
# Starting LocalServer process - port forwarding
self.local_server()
# Connect to the server
self.con = self._es_connect()
def _get_eck_file(self):
"""
Getting the ECK file from the official Elasticsearch web site and store
it as a temporary file.
Current version is 1.1.2, this need to be update with new versions,
after testing it, and also it may need to update the CRD file (esq.yaml)
with the new version as well.
"""
self.dir = tempfile.mkdtemp(prefix="elastic-system_")
src_file = f"{self.eck_path}/{self.eck_file}"
trg_file = f"{self.dir}/{self.eck_file}"
log.info(f"Retrieving the ECK CR file from {src_file} into {trg_file}")
try:
urllib.request.urlretrieve(src_file, trg_file)
except urllib.error.HTTPError as e:
log.error(f"Can not connect to {src_file} : {e}")
raise e
def _deploy_eck(self):
"""
Deploying the ECK environment for the Elasticsearch, and make sure it
is in Running mode
"""
log.info("Deploying the ECK environment for the ES cluster")
self.ocp.apply(f"{self.dir}/{self.eck_file}")
for es_pod in TimeoutSampler(300, 10, get_pod_name_by_pattern,
"elastic-operator", self.namespace):
try:
if es_pod[0] is not None:
self.eckpod = es_pod[0]
log.info(f"The ECK pod {self.eckpod} is ready !")
break
except IndexError:
log.info("ECK operator pod not ready yet")
def get_ip(self):
"""
This function return the IP address of the Elasticsearch cluster.
this IP is to use inside the OCP cluster
Return
str : String that represent the Ip Address.
"""
return self.es.get()["spec"]["clusterIP"]
def get_port(self):
"""
This function return the port of the Elasticsearch cluster.
Return
str : String that represent the port.
"""
return self.es.get()["spec"]["ports"][0]["port"]
def _deploy_es(self):
log.info("Deploy the PVC for the ElasticSearch cluster")
self.ocp.apply(self.pvc)
log.info("Deploy the ElasticSearch cluster")
self.ocp.apply(self.crd)
for es_pod in TimeoutSampler(300, 20, get_pod_name_by_pattern,
"quickstart-es-default", self.namespace):
try:
if es_pod[0] is not None:
self.espod = es_pod[0]
log.info(f"The ElasticSearch pod {self.espod} Started")
break
except IndexError:
log.info("elasticsearch pod not ready yet")
es_pod = OCP(kind="pod", namespace=self.namespace)
log.info("Waiting for ElasticSearch to Run")
assert es_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=self.espod,
sleep=30,
timeout=600,
)
log.info("Elastic Search is ready !!!")
def get_health(self):
"""
This method return the health status of the Elasticsearch.
Returns:
bool : True if the status is green (OK) otherwise - False
"""
return self.elasticsearch.get(
)["items"][0]["status"]["health"] == "green"
def get_password(self):
"""
This method return the password used to connect the Elasticsearch.
Returns:
str : The password as text
"""
return base64.b64decode(
self.password.get()["data"]["elastic"]).decode("utf-8")
def cleanup(self):
"""
Cleanup the environment from all Elasticsearch components, and from the
port forwarding process.
"""
log.info("Teardown the Elasticsearch environment")
log.info(f"Killing the local server process ({self.lspid})")
os.kill(self.lspid, signal.SIGKILL)
log.info("Deleting all resources")
subprocess.run(f"oc delete -f {self.crd}", shell=True)
subprocess.run(f"oc delete -f {self.eck_file}",
shell=True,
cwd=self.dir)
self.ns_obj.wait_for_delete(resource_name=self.namespace)
def local_server(self):
"""
Starting sub-process that will do port-forwarding, to allow access from
outside the open-shift cluster into the Elasticsearch server.
"""
cmd = f"oc -n {self.namespace } "
cmd += f"port-forward service/quickstart-es-http {self.get_port()}"
log.info(f"Going to run : {cmd}")
proc = subprocess.Popen(cmd, shell=True)
log.info(f"Starting LocalServer with PID of {proc.pid}")
self.lspid = proc.pid
def _es_connect(self):
"""
Create a connection to the ES via the localhost port-fwd
Returns:
Elasticsearch: elasticsearch connection object
Raise:
ConnectionError: if can not connect to the server
"""
try:
es = Elasticsearch([{
"host": "localhost",
"port": self.get_port()
}])
except esexp.ConnectionError:
log.error("Can not connect to ES server in the LocalServer")
raise
return es
def get_indices(self):
"""
Getting list of all indices in the ES server - all created by the test,
the installation of the ES was without any indexes pre-installed.
Returns:
list : list of all indices defined in the ES server
"""
results = []
log.info("Getting all indices")
for ind in self.con.indices.get_alias("*"):
results.append(ind)
return results
def _copy(self, es):
"""
Copy All data from the internal ES server to the main ES
Args:
es (obj): elasticsearch object which connected to the main ES
"""
query = {"size": 1000, "query": {"match_all": {}}}
for ind in self.get_indices():
log.info(f"Reading {ind} from internal ES server")
try:
result = self.con.search(index=ind, body=query)
except esexp.NotFoundError:
log.warning(f"{ind} Not found in the Internal ES.")
continue
log.debug(f"The results from internal ES for {ind} are :{result}")
log.info(f"Writing {ind} into main ES server")
for doc in result["hits"]["hits"]:
log.debug(f"Going to write : {doc}")
es.index(index=ind, doc_type="_doc", body=doc["_source"])
class ElasticSearch(object):
"""
ElasticSearch Environment
"""
def __init__(self, **kwargs):
"""
Initializer function
"""
log.info("Initializing the Elastic-Search environment object")
self.args = kwargs
self.namespace = "elastic-system"
self.repo = self.args.get("repo", constants.OCS_WORKLOADS)
self.branch = self.args.get("branch", "master")
self.dir = tempfile.mkdtemp(prefix="eck_")
# Clone the ECK repo locally
self._clone()
self.eck_path = os.path.join(self.dir, "ocs-workloads/eck")
self.eck_file = os.path.join(self.eck_path, "crds.yaml")
self.dumper_file = os.path.join(constants.TEMPLATE_APP_POD_DIR,
"esclient.yaml")
self.crd = os.path.join(constants.TEMPLATE_APP_POD_DIR, "esq.yaml")
# Creating some different types of OCP objects
self.ocp = OCP(kind="pod",
resource_name="elastic-operator-0",
namespace=self.namespace)
self.ns_obj = OCP(kind="namespace", namespace=self.namespace)
self.es = OCP(resource_name="quickstart-es-http",
namespace=self.namespace)
self.elasticsearch = OCP(namespace=self.namespace,
kind="elasticsearch")
self.password = OCP(
kind="secret",
resource_name="quickstart-es-elastic-user",
namespace=self.namespace,
)
# Deploy the ECK all-in-one.yaml file
self._deploy_eck()
# Deploy the Elastic-Search server
self._deploy_es()
# Verify that ES is Up & Running
sample = TimeoutSampler(timeout=180, sleep=10, func=self.get_health)
if not sample.wait_for_func_status(True):
raise Exception("Elasticsearch deployment Failed")
# Deploy the elasticsearch dumper pod
self._deploy_data_dumper_client()
# Connect to the server
self.con = self._es_connect()
def _clone(self):
"""
clone the ECK repo into temp directory
"""
try:
log.info(f"Cloning ECK in {self.dir}")
git_clone_cmd = f"git clone -b {self.branch} {self.repo} --depth 1"
run(git_clone_cmd, shell=True, cwd=self.dir, check=True)
except (CommandFailed, CalledProcessError) as cf:
log.error("Error during cloning of ECK repository")
raise cf
def _pod_is_found(self, pattern):
"""
Boolean function which check if pod (by pattern) is exist.
Args:
pattern (str): the pattern of the pod to look for
Returns:
bool : True if pod found, otherwise False
"""
return len(get_pod_name_by_pattern(pattern, self.namespace)) > 0
def _deploy_eck(self):
"""
Deploying the ECK environment for the Elasticsearch, and make sure it
is in Running mode
"""
log.info("Deploying the ECK environment for the ES cluster")
log.info("Deploy the ECK CRD's")
self.ocp.apply(self.eck_file)
log.info("deploy the ECK operator")
self.ocp.apply(f"{self.eck_path}/operator.yaml")
sample = TimeoutSampler(timeout=300,
sleep=10,
func=self._pod_is_found,
pattern="elastic-operator")
if not sample.wait_for_func_status(True):
err_msg = "ECK deployment Failed"
log.error(err_msg)
self.cleanup()
raise Exception(err_msg)
log.info("The ECK pod is ready !")
def _deploy_data_dumper_client(self):
"""
Deploying elastic search client pod with utility which dump all the data
from the server to .tgz file
"""
log.info("Deploying the es client for dumping all data")
self.ocp.apply(self.dumper_file)
sample = TimeoutSampler(timeout=300,
sleep=10,
func=self._pod_is_found,
pattern="es-dumper")
if not sample.wait_for_func_status(True):
self.cleanup()
raise Exception("Dumper pod deployment Failed")
self.dump_pod = get_pod_name_by_pattern("es-dumper", self.namespace)[0]
log.info(f"The dumper client pod {self.dump_pod} is ready !")
def get_ip(self):
"""
This function return the IP address of the Elasticsearch cluster.
this IP is to use inside the OCP cluster
Return
str : String that represent the Ip Address.
"""
return self.es.get()["spec"]["clusterIP"]
def get_port(self):
"""
This function return the port of the Elasticsearch cluster.
Return
str : String that represent the port.
"""
return self.es.get()["spec"]["ports"][0]["port"]
def _deploy_es(self):
"""
Deploying the Elasticsearch server
"""
# Creating PVC for the elasticsearch server and wait until it bound
log.info("Creating 10 GiB PVC for the ElasticSearch cluster on")
self.pvc_obj = create_pvc(
sc_name=constants.CEPHBLOCKPOOL_SC,
namespace=self.namespace,
pvc_name="elasticsearch-data-quickstart-es-default-0",
access_mode=constants.ACCESS_MODE_RWO,
size="10Gi",
)
wait_for_resource_state(self.pvc_obj, constants.STATUS_BOUND)
self.pvc_obj.reload()
log.info("Deploy the ElasticSearch cluster")
self.ocp.apply(self.crd)
sample = TimeoutSampler(
timeout=300,
sleep=10,
func=self._pod_is_found,
pattern="quickstart-es-default",
)
if not sample.wait_for_func_status(True):
self.cleanup()
raise Exception("The ElasticSearch pod deployment Failed")
self.espod = get_pod_name_by_pattern("quickstart-es-default",
self.namespace)[0]
log.info(f"The ElasticSearch pod {self.espod} Started")
es_pod = OCP(kind="pod", namespace=self.namespace)
log.info("Waiting for ElasticSearch to Run")
assert es_pod.wait_for_resource(
condition=constants.STATUS_RUNNING,
resource_name=self.espod,
sleep=30,
timeout=600,
)
log.info("Elastic Search is ready !!!")
def get_health(self):
"""
This method return the health status of the Elasticsearch.
Returns:
bool : True if the status is green (OK) otherwise - False
"""
return self.elasticsearch.get(
)["items"][0]["status"]["health"] == "green"
def get_password(self):
"""
This method return the password used to connect the Elasticsearch.
Returns:
str : The password as text
"""
return base64.b64decode(
self.password.get()["data"]["elastic"]).decode("utf-8")
def cleanup(self):
"""
Cleanup the environment from all Elasticsearch components, and from the
port forwarding process.
"""
log.info("Teardown the Elasticsearch environment")
log.info("Deleting all resources")
log.info("Deleting the dumper client pod")
self.ocp.delete(yaml_file=self.dumper_file)
log.info("Deleting the es resource")
self.ocp.delete(yaml_file=self.crd)
log.info("Deleting the es project")
# self.ns_obj.delete_project(project_name=self.namespace)
self.ocp.delete(f"{self.eck_path}/operator.yaml")
self.ocp.delete(yaml_file=self.eck_file)
self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=180)
def _es_connect(self):
"""
Create a connection to the local ES
Returns:
Elasticsearch: elasticsearch connection object, None if Cannot connect to ES
"""
try:
es = Elasticsearch([{
"host": self.get_ip(),
"port": self.get_port()
}])
except esexp.ConnectionError:
log.warning("Cannot connect to ES server in the LocalServer")
es = None
return es
def get_indices(self):
"""
Getting list of all indices in the ES server - all created by the test,
the installation of the ES was without any indexes pre-installed.
Returns:
list : list of all indices defined in the ES server
"""
results = []
log.info("Getting all indices")
for ind in self.con.indices.get_alias("*"):
results.append(ind)
return results
def dumping_all_data(self, target_path):
"""
Dump All data from the internal ES server to .tgz file.
Args:
target_path (str): the path where the results file will be copy into
Return:
bool: True if the dump operation succeed and return the results data to the host
otherwise False
"""
log.info("dumping data from ES server to .tgz file")
rsh_cmd = f"rsh {self.dump_pod} /elasticsearch-dump/esdumper.py --ip {self.get_ip()} --port {self.get_port()}"
result = self.ocp.exec_oc_cmd(rsh_cmd,
out_yaml_format=False,
timeout=1200)
if "ES dump is done." not in result:
log.error("There is no data in the Elasticsearch server")
return False
else:
src_file = result.split()[-1]
log.info(f"Copy {src_file} from the client pod")
cp_command = f"cp {self.dump_pod}:{src_file} {target_path}/FullResults.tgz"
result = self.ocp.exec_oc_cmd(cp_command, timeout=120)
log.info(f"The output from the POD is {result}")
log.info("Extracting the FullResults.tgz file")
kwargs = {"cwd": target_path}
results = run_command(f"tar zxvf {target_path}/FullResults.tgz",
**kwargs)
log.debug(f"The untar results is {results}")
if "Error in command" in results:
log.warning("Cannot untar the dumped file")
return False
return True
class OCS(object):
"""
Base OCSClass
"""
def __init__(self, **kwargs):
"""
Initializer function
Args:
kwargs (dict):
1) For existing resource, use OCP.reload() to get the
resource's dictionary and use it to pass as **kwargs
2) For new resource, use yaml files templates under
/templates/CSI like:
obj_dict = load_yaml_to_dict(
os.path.join(
TEMPLATE_DIR, "some_resource.yaml"
)
)
"""
self.data = kwargs
self._api_version = self.data.get('api_version')
self._kind = self.data.get('kind')
self._namespace = None
if 'metadata' in self.data:
self._namespace = self.data.get('metadata').get('namespace')
self._name = self.data.get('metadata').get('name')
self.ocp = OCP(api_version=self._api_version,
kind=self.kind,
namespace=self._namespace)
self.temp_yaml = tempfile.NamedTemporaryFile(mode='w+',
prefix=self._kind,
delete=False)
@property
def api_version(self):
return self._api_version
@property
def kind(self):
return self._kind
@property
def namespace(self):
return self._namespace
@property
def name(self):
return self._name
def reload(self):
"""
Reloading the OCS instance with the new information from its actual
data.
After creating a resource from a yaml file, the actual yaml file is
being changed and more information about the resource is added.
"""
self.data = self.get()
self.__init__(**self.data)
def get(self, out_yaml_format=True):
return self.ocp.get(resource_name=self.name,
out_yaml_format=out_yaml_format)
def create(self):
log.info(f"Adding {self.kind} with name {self.name}")
templating.dump_dict_to_temp_yaml(self.data, self.temp_yaml.name)
status = self.ocp.create(yaml_file=self.temp_yaml.name)
self.reload()
return status
def delete(self, wait=True):
return self.ocp.delete(resource_name=self.name, wait=wait)
def apply(self, **data):
with open(self.temp_yaml.name, 'w') as yaml_file:
yaml.dump(data, yaml_file)
assert self.ocp.apply(
yaml_file=self.temp_yaml.name), (f"Failed to apply changes {data}")
self.reload()
def add_label(self, label):
"""
Addss a new label
Args:
label (str): New label to be assigned for this pod
E.g: "label=app='rook-ceph-mds'"
"""
status = self.ocp.add_label(resource_name=self.name, label=label)
self.reload()
return status
def delete_temp_yaml_file(self):
utils.delete_file(self.temp_yaml.name)
