def health_check(cluster_id, report_uri, username):
with util.lock('cluster_health_check') as lock:
if not lock:
raise Exception('failed to acquire lock to check health')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user
)
deploy_manager = DeployManager(adapter_info, cluster_info, None)
try:
deploy_manager.check_cluster_health(report_uri)
except Exception as exc:
logging.error("health_check exception: ============= %s" % exc)
data = {'state': 'error', 'error_message': str(exc), 'report': {}}
reports = health_check_db.list_health_reports(
cluster_id, user=user)
if not reports:
# Exception before executing command remotely for health check.
# No reports names sending back yet. Create a report
name = 'pre_remote_health_check'
health_check_db.add_report_record(
cluster_id, name, user=user, **data
)
health_check_db.update_multi_reports(cluster_id, user=user, **data)
def cluster_installed(cluster_id, clusterhosts_ready, username=None):
"""Callback when cluster is installed.
:param cluster_id: cluster id
:param clusterhosts_ready: clusterhosts that should trigger ready.
.. note::
The function should be called out of database session.
"""
with util.lock("serialized_action") as lock:
if not lock:
raise Exception("failed to acquire lock to " "do the post action after cluster installation")
logging.info("package installed on cluster %s with clusterhosts ready %s", cluster_id, clusterhosts_ready)
if username:
user = user_db.get_user_object(username)
else:
user = None
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(cluster_id, clusterhosts_ready.keys(), user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
deploy_manager.cluster_installed()
util.ActionHelper.cluster_ready(cluster_id, True, user)
for host_id, clusterhost_ready in clusterhosts_ready.items():
if clusterhost_ready:
util.ActionHelper.cluster_host_ready(cluster_id, host_id, False, user)
def health_check(cluster_id, report_uri, username):
with util.lock('cluster_health_check') as lock:
if not lock:
raise Exception('failed to acquire lock to check health')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user
)
deploy_manager = DeployManager(adapter_info, cluster_info, None)
try:
deploy_manager.check_cluster_health(report_uri)
except Exception as exc:
logging.error("health_check exception: ============= %s" % exc)
data = {'state': 'error', 'error_message': str(exc), 'report': {}}
reports = health_check_db.list_health_reports(
cluster_id, user=user)
if not reports:
# Exception before executing command remotely for health check.
# No reports names sending back yet. Create a report
name = 'pre_remote_health_check'
health_check_db.add_report_record(
cluster_id, name, user=user, **data
)
health_check_db.update_multi_reports(cluster_id, user=user, **data)
def deploy(cluster_id, hosts_id_list, username=None):
"""Deploy clusters.
:param cluster_hosts: clusters and hosts in each cluster to deploy.
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=1000) as lock:
if not lock:
raise Exception('failed to acquire lock to deploy')
user = user_db.get_user_object(username)
cluster_info = ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = ActionHelper.get_adapter_info(adapter_id, cluster_id,
user)
hosts_info = ActionHelper.get_hosts_info(cluster_id, hosts_id_list,
user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
#deploy_manager.prepare_for_deploy()
logging.debug('Created deploy manager with %s %s %s'
% (adapter_info, cluster_info, hosts_info))
deployed_config = deploy_manager.deploy()
ActionHelper.save_deployed_config(deployed_config, user)
ActionHelper.update_state(cluster_id, hosts_id_list, user)
def redeploy(cluster_id, hosts_id_list, username=None):
"""Deploy clusters.
:param cluster_hosts: clusters and hosts in each cluster to deploy.
:type cluster_hosts: dict of int or str to list of int or str
"""
with util.lock('serialized_action') as lock:
if not lock:
raise Exception('failed to acquire lock to deploy')
user = user_db.get_user_object(username)
cluster_info = ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = ActionHelper.get_adapter_info(adapter_id,
cluster_id,
user)
hosts_info = ActionHelper.get_hosts_info(cluster_id,
hosts_id_list,
user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
# deploy_manager.prepare_for_deploy()
deploy_manager.redeploy()
ActionHelper.update_state(cluster_id, hosts_id_list, user)
def delete_cluster_host(
cluster_id, host_id,
username=None, delete_underlying_host=False
):
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete clusterhost')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
logging.debug('Created deploy manager with %s %s %s'
% (adapter_info, cluster_info, hosts_info))
deploy_manager.remove_hosts(
package_only=not delete_underlying_host,
delete_cluster=False
)
util.ActionHelper.delete_cluster_host(
cluster_id, host_id, user,
delete_underlying_host
)
def delete_host(
host_id, cluster_id_list, username=None
):
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete host')
user = user_db.get_user_object(username)
for cluster_id in cluster_id_list:
cluster_info = util.ActionHelper.get_cluster_info(
cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(
adapter_info, cluster_info, hosts_info)
deploy_manager.remove_hosts(
package_only=True,
delete_cluster=False
)
util.ActionHelper.delete_host(
host_id, user
)
def delete_host(host_id, cluster_id_list, username=None):
"""Delete host and all clusterhosts on it.
:param host_id: id of the host.
:type host_id: int
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete host')
user = user_db.get_user_object(username)
for cluster_id in cluster_id_list:
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(adapter_info, cluster_info,
hosts_info)
deploy_manager.remove_hosts(package_only=True,
delete_cluster=False)
util.ActionHelper.delete_host(host_id, user)
def os_installed(
host_id, clusterhosts_ready, clusters_os_ready,
username=None
):
"""Callback when os is installed.
:param host_id: host that os is installed.
:type host_id: integer
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action') as lock:
if not lock:
raise Exception(
'failed to acquire lock to '
'do the post action after os installation'
)
logging.info(
'os installed on host %s '
'with cluster host ready %s cluster os ready %s',
host_id, clusterhosts_ready, clusters_os_ready
)
if username:
user = user_db.get_user_object(username)
else:
user = None
os_installed_triggered = False
for cluster_id, clusterhost_ready in clusterhosts_ready.items():
if not clusterhost_ready and os_installed_triggered:
continue
cluster_info = util.ActionHelper.get_cluster_info(
cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(
adapter_info, cluster_info, hosts_info)
if not os_installed_triggered:
deploy_manager.os_installed()
util.ActionHelper.host_ready(host_id, True, user)
os_installed_triggered = True
if clusterhost_ready:
#deploy_manager.cluster_os_installed()
util.ActionHelper.cluster_host_ready(
cluster_id, host_id, False, user
)
if util.ActionHelper.is_cluster_os_ready(cluster_id, user):
logging.info("deploy_manager begin cluster_os_installed")
deploy_manager.cluster_os_installed()
def delete_cluster(
cluster_id, host_id_list,
username=None, delete_underlying_host=False
):
"""Delete cluster.
:param cluster_id: id of the cluster.
:type cluster_id: int
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete cluster')
user = user_db.get_user_object(username)
for host_id in host_id_list:
cluster_api.update_cluster_host_state(
user, cluster_id, host_id, state='ERROR'
)
cluster_api.update_cluster_state(
user, cluster_id, state='ERROR'
)
cluster_api.update_cluster(
user, cluster_id, reinstall_distributed_system=True
)
for host_id in host_id_list:
cluster_api.update_cluster_host(
user, cluster_id, host_id, reinstall_os=True
)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, host_id_list, user)
logging.debug('adapter info: %s', adapter_info)
logging.debug('cluster info: %s', cluster_info)
logging.debug('hosts info: %s', hosts_info)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
deploy_manager.remove_hosts(
package_only=not delete_underlying_host,
delete_cluster=True
)
util.ActionHelper.delete_cluster(
cluster_id, host_id_list, user,
delete_underlying_host
)
def package_installed(
cluster_id, host_id, cluster_ready,
host_ready, username=None
):
"""Callback when package is installed.
:param cluster_id: cluster id.
:param host_id: host id.
:param cluster_ready: if the cluster should trigger ready.
:param host_ready: if the host should trigger ready.
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action') as lock:
if not lock:
raise Exception(
'failed to acquire lock to '
'do the post action after package installation'
)
logging.info(
'package installed on cluster %s host %s '
'with cluster ready %s host ready %s',
cluster_id, host_id, cluster_ready, host_ready
)
if username:
user = user_db.get_user_object(username)
else:
user = None
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
deploy_manager.package_installed()
util.ActionHelper.cluster_host_ready(cluster_id, host_id, True, user)
if cluster_ready:
util.ActionHelper.cluster_ready(cluster_id, False, user)
if host_ready:
util.ActionHelper.host_ready(host_id, False, user)
def deploy(cluster_id, hosts_id_list, username=None):
"""Deploy clusters.
:param cluster_hosts: clusters and hosts in each cluster to deploy.
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=1000) as lock:
if not lock:
raise Exception('failed to acquire lock to deploy')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, hosts_id_list, user)
deploy_successful = True
try:
deploy_manager = DeployManager(
adapter_info, cluster_info, hosts_info)
# deploy_manager.prepare_for_deploy()
logging.debug('Created deploy manager with %s %s %s'
% (adapter_info, cluster_info, hosts_info))
deployed_config = deploy_manager.deploy()
except Exception as error:
logging.exception(error)
deploy_successful = False
if deploy_successful:
util.ActionHelper.save_deployed_config(deployed_config, user)
util.ActionHelper.update_state(
cluster_id, hosts_id_list, user, state='INSTALLING'
)
else:
util.ActionHelper.update_state(
cluster_id, hosts_id_list, user, state='ERROR',
message='failed to start deployment', severity='ERROR'
)
def deploy(cluster_id, hosts_id_list, username=None):
"""Deploy clusters.
:param cluster_hosts: clusters and hosts in each cluster to deploy.
:type cluster_hosts: dict of int or str to list of int or str
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=1000) as lock:
if not lock:
raise Exception('failed to acquire lock to deploy')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, hosts_id_list, user)
deploy_successful = True
try:
deploy_manager = DeployManager(
adapter_info, cluster_info, hosts_info)
# deploy_manager.prepare_for_deploy()
logging.debug('Created deploy manager with %s %s %s'
% (adapter_info, cluster_info, hosts_info))
deployed_config = deploy_manager.deploy()
except Exception as error:
logging.exception(error)
deploy_successful = False
if deploy_successful:
util.ActionHelper.save_deployed_config(deployed_config, user)
util.ActionHelper.update_state(
cluster_id, hosts_id_list, user, state='INSTALLING'
)
else:
util.ActionHelper.update_state(
cluster_id, hosts_id_list, user, state='ERROR',
message='failed to start deployment', severity='ERROR'
)
def test_init_DeployManager(self):
adapter_info = deepcopy(config_data.adapter_test_config)
cluster_info = deepcopy(config_data.cluster_test_config)
hosts_info = deepcopy(config_data.hosts_test_config)
DeployManager._get_installer = Mock()
DeployManager._get_installer.return_value = "mock_installer"
test_manager = DeployManager(adapter_info, cluster_info, hosts_info)
self.assertIsNotNone(test_manager)
def redeploy(cluster_id, username=None):
"""Deploy clusters.
:param cluster_hosts: clusters and hosts in each cluster to deploy.
:type cluster_hosts: dict of int or str to list of int or str
"""
with util.lock('serialized_action') as lock:
if not lock:
raise Exception('failed to acquire lock to deploy')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user)
hosts_id_list = [host['id'] for host in cluster_hosts]
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, hosts_id_list, user)
deploy_successful = True
try:
deploy_manager = DeployManager(
adapter_info, cluster_info, hosts_info)
# deploy_manager.prepare_for_deploy()
deploy_manager.redeploy()
except Exception as error:
logging.exception(error)
deploy_successful = False
if deploy_successful:
util.ActionHelper.update_state(
cluster_id, hosts_id_list, user, state='INSTALLING',
)
else:
util.ActionHelper.update_state(
cluster_id, hosts_id_list, user, state='ERROR',
message='failed to start redeployment', severity='ERROR'
)
def delete_cluster(
cluster_id, host_id_list,
username=None, delete_underlying_host=False
):
"""Delete cluster and all clusterhosts on it.
:param cluster_id: id of the cluster.
:type cluster_id: int
:param host_id_list: list of host id.
:type host_id_list: list of int.
If delete_underlying_host is set, all underlying hosts will
be deleted.
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete cluster')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, host_id_list, user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
deploy_manager.remove_hosts(
package_only=not delete_underlying_host,
delete_cluster=True
)
util.ActionHelper.delete_cluster(
cluster_id, host_id_list, user,
delete_underlying_host
)
def delete_cluster(cluster_id,
host_id_list,
username=None,
delete_underlying_host=False):
"""Delete cluster and all clusterhosts on it.
:param cluster_id: id of the cluster.
:type cluster_id: int
:param host_id_list: list of host id.
:type host_id_list: list of int.
If delete_underlying_host is set, all underlying hosts will
be deleted.
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete cluster')
user = user_db.get_user_object(username)
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(cluster_id, host_id_list,
user)
deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info)
deploy_manager.remove_hosts(package_only=not delete_underlying_host,
delete_cluster=True)
util.ActionHelper.delete_cluster(cluster_id, host_id_list, user,
delete_underlying_host)
def delete_host(
host_id, cluster_id_list, username=None
):
"""Delete host and all clusterhosts on it.
:param host_id: id of the host.
:type host_id: int
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action', timeout=100) as lock:
if not lock:
raise Exception('failed to acquire lock to delete host')
user = user_db.get_user_object(username)
for cluster_id in cluster_id_list:
cluster_info = util.ActionHelper.get_cluster_info(
cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(
adapter_info, cluster_info, hosts_info)
deploy_manager.remove_hosts(
package_only=True,
delete_cluster=False
)
util.ActionHelper.delete_host(
host_id, user
)
def os_installed(host_id,
clusterhosts_ready,
clusters_os_ready,
username=None):
"""Callback when os is installed.
:param host_id: host that os is installed.
:type host_id: integer
:param clusterhosts_ready: the clusterhosts that should trigger ready.
:param clusters_os_ready: the cluster that should trigger os ready.
.. note::
The function should be called out of database session.
"""
with util.lock('serialized_action') as lock:
if not lock:
raise Exception('failed to acquire lock to '
'do the post action after os installation')
logging.info(
'os installed on host %s '
'with cluster host ready %s cluster os ready %s', host_id,
clusterhosts_ready, clusters_os_ready)
if username:
user = user_db.get_user_object(username)
else:
user = None
os_installed_triggered = False
for cluster_id, clusterhost_ready in clusterhosts_ready.items():
if not clusterhost_ready and os_installed_triggered:
continue
cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user)
adapter_id = cluster_info[const.ADAPTER_ID]
adapter_info = util.ActionHelper.get_adapter_info(
adapter_id, cluster_id, user)
hosts_info = util.ActionHelper.get_hosts_info(
cluster_id, [host_id], user)
deploy_manager = DeployManager(adapter_info, cluster_info,
hosts_info)
if not os_installed_triggered:
deploy_manager.os_installed()
util.ActionHelper.host_ready(host_id, True, user)
os_installed_triggered = True
if clusterhost_ready:
# deploy_manager.cluster_os_installed()
util.ActionHelper.cluster_host_ready(cluster_id, host_id,
False, user)
if util.ActionHelper.is_cluster_os_ready(cluster_id, user):
logging.info("deploy_manager begin cluster_os_installed")
deploy_manager.cluster_os_installed()
