def health_check(cluster_id, report_uri, username): with util.lock('cluster_health_check') as lock: if not lock: raise Exception('failed to acquire lock to check health') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user ) deploy_manager = DeployManager(adapter_info, cluster_info, None) try: deploy_manager.check_cluster_health(report_uri) except Exception as exc: logging.error("health_check exception: ============= %s" % exc) data = {'state': 'error', 'error_message': str(exc), 'report': {}} reports = health_check_db.list_health_reports( cluster_id, user=user) if not reports: # Exception before executing command remotely for health check. # No reports names sending back yet. Create a report name = 'pre_remote_health_check' health_check_db.add_report_record( cluster_id, name, user=user, **data ) health_check_db.update_multi_reports(cluster_id, user=user, **data)
def cluster_installed(cluster_id, clusterhosts_ready, username=None): """Callback when cluster is installed. :param cluster_id: cluster id :param clusterhosts_ready: clusterhosts that should trigger ready. .. note:: The function should be called out of database session. """ with util.lock("serialized_action") as lock: if not lock: raise Exception("failed to acquire lock to " "do the post action after cluster installation") logging.info("package installed on cluster %s with clusterhosts ready %s", cluster_id, clusterhosts_ready) if username: user = user_db.get_user_object(username) else: user = None cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info(adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info(cluster_id, clusterhosts_ready.keys(), user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) deploy_manager.cluster_installed() util.ActionHelper.cluster_ready(cluster_id, True, user) for host_id, clusterhost_ready in clusterhosts_ready.items(): if clusterhost_ready: util.ActionHelper.cluster_host_ready(cluster_id, host_id, False, user)
def health_check(cluster_id, report_uri, username): with util.lock('cluster_health_check') as lock: if not lock: raise Exception('failed to acquire lock to check health') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user ) deploy_manager = DeployManager(adapter_info, cluster_info, None) try: deploy_manager.check_cluster_health(report_uri) except Exception as exc: logging.error("health_check exception: ============= %s" % exc) data = {'state': 'error', 'error_message': str(exc), 'report': {}} reports = health_check_db.list_health_reports( cluster_id, user=user) if not reports: # Exception before executing command remotely for health check. # No reports names sending back yet. Create a report name = 'pre_remote_health_check' health_check_db.add_report_record( cluster_id, name, user=user, **data ) health_check_db.update_multi_reports(cluster_id, user=user, **data)
def deploy(cluster_id, hosts_id_list, username=None): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=1000) as lock: if not lock: raise Exception('failed to acquire lock to deploy') user = user_db.get_user_object(username) cluster_info = ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = ActionHelper.get_adapter_info(adapter_id, cluster_id, user) hosts_info = ActionHelper.get_hosts_info(cluster_id, hosts_id_list, user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) #deploy_manager.prepare_for_deploy() logging.debug('Created deploy manager with %s %s %s' % (adapter_info, cluster_info, hosts_info)) deployed_config = deploy_manager.deploy() ActionHelper.save_deployed_config(deployed_config, user) ActionHelper.update_state(cluster_id, hosts_id_list, user)
def redeploy(cluster_id, hosts_id_list, username=None): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. :type cluster_hosts: dict of int or str to list of int or str """ with util.lock('serialized_action') as lock: if not lock: raise Exception('failed to acquire lock to deploy') user = user_db.get_user_object(username) cluster_info = ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = ActionHelper.get_adapter_info(adapter_id, cluster_id, user) hosts_info = ActionHelper.get_hosts_info(cluster_id, hosts_id_list, user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) # deploy_manager.prepare_for_deploy() deploy_manager.redeploy() ActionHelper.update_state(cluster_id, hosts_id_list, user)
def delete_cluster_host( cluster_id, host_id, username=None, delete_underlying_host=False ): with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete clusterhost') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) logging.debug('Created deploy manager with %s %s %s' % (adapter_info, cluster_info, hosts_info)) deploy_manager.remove_hosts( package_only=not delete_underlying_host, delete_cluster=False ) util.ActionHelper.delete_cluster_host( cluster_id, host_id, user, delete_underlying_host )
def delete_host( host_id, cluster_id_list, username=None ): with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete host') user = user_db.get_user_object(username) for cluster_id in cluster_id_list: cluster_info = util.ActionHelper.get_cluster_info( cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager( adapter_info, cluster_info, hosts_info) deploy_manager.remove_hosts( package_only=True, delete_cluster=False ) util.ActionHelper.delete_host( host_id, user )
def delete_host(host_id, cluster_id_list, username=None): """Delete host and all clusterhosts on it. :param host_id: id of the host. :type host_id: int .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete host') user = user_db.get_user_object(username) for cluster_id in cluster_id_list: cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) deploy_manager.remove_hosts(package_only=True, delete_cluster=False) util.ActionHelper.delete_host(host_id, user)
def os_installed( host_id, clusterhosts_ready, clusters_os_ready, username=None ): """Callback when os is installed. :param host_id: host that os is installed. :type host_id: integer .. note:: The function should be called out of database session. """ with util.lock('serialized_action') as lock: if not lock: raise Exception( 'failed to acquire lock to ' 'do the post action after os installation' ) logging.info( 'os installed on host %s ' 'with cluster host ready %s cluster os ready %s', host_id, clusterhosts_ready, clusters_os_ready ) if username: user = user_db.get_user_object(username) else: user = None os_installed_triggered = False for cluster_id, clusterhost_ready in clusterhosts_ready.items(): if not clusterhost_ready and os_installed_triggered: continue cluster_info = util.ActionHelper.get_cluster_info( cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager( adapter_info, cluster_info, hosts_info) if not os_installed_triggered: deploy_manager.os_installed() util.ActionHelper.host_ready(host_id, True, user) os_installed_triggered = True if clusterhost_ready: #deploy_manager.cluster_os_installed() util.ActionHelper.cluster_host_ready( cluster_id, host_id, False, user ) if util.ActionHelper.is_cluster_os_ready(cluster_id, user): logging.info("deploy_manager begin cluster_os_installed") deploy_manager.cluster_os_installed()
def delete_cluster( cluster_id, host_id_list, username=None, delete_underlying_host=False ): """Delete cluster. :param cluster_id: id of the cluster. :type cluster_id: int .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete cluster') user = user_db.get_user_object(username) for host_id in host_id_list: cluster_api.update_cluster_host_state( user, cluster_id, host_id, state='ERROR' ) cluster_api.update_cluster_state( user, cluster_id, state='ERROR' ) cluster_api.update_cluster( user, cluster_id, reinstall_distributed_system=True ) for host_id in host_id_list: cluster_api.update_cluster_host( user, cluster_id, host_id, reinstall_os=True ) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, host_id_list, user) logging.debug('adapter info: %s', adapter_info) logging.debug('cluster info: %s', cluster_info) logging.debug('hosts info: %s', hosts_info) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) deploy_manager.remove_hosts( package_only=not delete_underlying_host, delete_cluster=True ) util.ActionHelper.delete_cluster( cluster_id, host_id_list, user, delete_underlying_host )
def package_installed( cluster_id, host_id, cluster_ready, host_ready, username=None ): """Callback when package is installed. :param cluster_id: cluster id. :param host_id: host id. :param cluster_ready: if the cluster should trigger ready. :param host_ready: if the host should trigger ready. .. note:: The function should be called out of database session. """ with util.lock('serialized_action') as lock: if not lock: raise Exception( 'failed to acquire lock to ' 'do the post action after package installation' ) logging.info( 'package installed on cluster %s host %s ' 'with cluster ready %s host ready %s', cluster_id, host_id, cluster_ready, host_ready ) if username: user = user_db.get_user_object(username) else: user = None cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) deploy_manager.package_installed() util.ActionHelper.cluster_host_ready(cluster_id, host_id, True, user) if cluster_ready: util.ActionHelper.cluster_ready(cluster_id, False, user) if host_ready: util.ActionHelper.host_ready(host_id, False, user)
def deploy(cluster_id, hosts_id_list, username=None): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=1000) as lock: if not lock: raise Exception('failed to acquire lock to deploy') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, hosts_id_list, user) deploy_successful = True try: deploy_manager = DeployManager( adapter_info, cluster_info, hosts_info) # deploy_manager.prepare_for_deploy() logging.debug('Created deploy manager with %s %s %s' % (adapter_info, cluster_info, hosts_info)) deployed_config = deploy_manager.deploy() except Exception as error: logging.exception(error) deploy_successful = False if deploy_successful: util.ActionHelper.save_deployed_config(deployed_config, user) util.ActionHelper.update_state( cluster_id, hosts_id_list, user, state='INSTALLING' ) else: util.ActionHelper.update_state( cluster_id, hosts_id_list, user, state='ERROR', message='failed to start deployment', severity='ERROR' )
def deploy(cluster_id, hosts_id_list, username=None): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. :type cluster_hosts: dict of int or str to list of int or str .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=1000) as lock: if not lock: raise Exception('failed to acquire lock to deploy') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, hosts_id_list, user) deploy_successful = True try: deploy_manager = DeployManager( adapter_info, cluster_info, hosts_info) # deploy_manager.prepare_for_deploy() logging.debug('Created deploy manager with %s %s %s' % (adapter_info, cluster_info, hosts_info)) deployed_config = deploy_manager.deploy() except Exception as error: logging.exception(error) deploy_successful = False if deploy_successful: util.ActionHelper.save_deployed_config(deployed_config, user) util.ActionHelper.update_state( cluster_id, hosts_id_list, user, state='INSTALLING' ) else: util.ActionHelper.update_state( cluster_id, hosts_id_list, user, state='ERROR', message='failed to start deployment', severity='ERROR' )
def test_init_DeployManager(self): adapter_info = deepcopy(config_data.adapter_test_config) cluster_info = deepcopy(config_data.cluster_test_config) hosts_info = deepcopy(config_data.hosts_test_config) DeployManager._get_installer = Mock() DeployManager._get_installer.return_value = "mock_installer" test_manager = DeployManager(adapter_info, cluster_info, hosts_info) self.assertIsNotNone(test_manager)
def redeploy(cluster_id, username=None): """Deploy clusters. :param cluster_hosts: clusters and hosts in each cluster to deploy. :type cluster_hosts: dict of int or str to list of int or str """ with util.lock('serialized_action') as lock: if not lock: raise Exception('failed to acquire lock to deploy') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) cluster_hosts = cluster_db.list_cluster_hosts(cluster_id, user) hosts_id_list = [host['id'] for host in cluster_hosts] hosts_info = util.ActionHelper.get_hosts_info( cluster_id, hosts_id_list, user) deploy_successful = True try: deploy_manager = DeployManager( adapter_info, cluster_info, hosts_info) # deploy_manager.prepare_for_deploy() deploy_manager.redeploy() except Exception as error: logging.exception(error) deploy_successful = False if deploy_successful: util.ActionHelper.update_state( cluster_id, hosts_id_list, user, state='INSTALLING', ) else: util.ActionHelper.update_state( cluster_id, hosts_id_list, user, state='ERROR', message='failed to start redeployment', severity='ERROR' )
def delete_cluster( cluster_id, host_id_list, username=None, delete_underlying_host=False ): """Delete cluster and all clusterhosts on it. :param cluster_id: id of the cluster. :type cluster_id: int :param host_id_list: list of host id. :type host_id_list: list of int. If delete_underlying_host is set, all underlying hosts will be deleted. .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete cluster') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, host_id_list, user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) deploy_manager.remove_hosts( package_only=not delete_underlying_host, delete_cluster=True ) util.ActionHelper.delete_cluster( cluster_id, host_id_list, user, delete_underlying_host )
def delete_cluster(cluster_id, host_id_list, username=None, delete_underlying_host=False): """Delete cluster and all clusterhosts on it. :param cluster_id: id of the cluster. :type cluster_id: int :param host_id_list: list of host id. :type host_id_list: list of int. If delete_underlying_host is set, all underlying hosts will be deleted. .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete cluster') user = user_db.get_user_object(username) cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info(cluster_id, host_id_list, user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) deploy_manager.remove_hosts(package_only=not delete_underlying_host, delete_cluster=True) util.ActionHelper.delete_cluster(cluster_id, host_id_list, user, delete_underlying_host)
def delete_host( host_id, cluster_id_list, username=None ): """Delete host and all clusterhosts on it. :param host_id: id of the host. :type host_id: int .. note:: The function should be called out of database session. """ with util.lock('serialized_action', timeout=100) as lock: if not lock: raise Exception('failed to acquire lock to delete host') user = user_db.get_user_object(username) for cluster_id in cluster_id_list: cluster_info = util.ActionHelper.get_cluster_info( cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager( adapter_info, cluster_info, hosts_info) deploy_manager.remove_hosts( package_only=True, delete_cluster=False ) util.ActionHelper.delete_host( host_id, user )
def os_installed(host_id, clusterhosts_ready, clusters_os_ready, username=None): """Callback when os is installed. :param host_id: host that os is installed. :type host_id: integer :param clusterhosts_ready: the clusterhosts that should trigger ready. :param clusters_os_ready: the cluster that should trigger os ready. .. note:: The function should be called out of database session. """ with util.lock('serialized_action') as lock: if not lock: raise Exception('failed to acquire lock to ' 'do the post action after os installation') logging.info( 'os installed on host %s ' 'with cluster host ready %s cluster os ready %s', host_id, clusterhosts_ready, clusters_os_ready) if username: user = user_db.get_user_object(username) else: user = None os_installed_triggered = False for cluster_id, clusterhost_ready in clusterhosts_ready.items(): if not clusterhost_ready and os_installed_triggered: continue cluster_info = util.ActionHelper.get_cluster_info(cluster_id, user) adapter_id = cluster_info[const.ADAPTER_ID] adapter_info = util.ActionHelper.get_adapter_info( adapter_id, cluster_id, user) hosts_info = util.ActionHelper.get_hosts_info( cluster_id, [host_id], user) deploy_manager = DeployManager(adapter_info, cluster_info, hosts_info) if not os_installed_triggered: deploy_manager.os_installed() util.ActionHelper.host_ready(host_id, True, user) os_installed_triggered = True if clusterhost_ready: # deploy_manager.cluster_os_installed() util.ActionHelper.cluster_host_ready(cluster_id, host_id, False, user) if util.ActionHelper.is_cluster_os_ready(cluster_id, user): logging.info("deploy_manager begin cluster_os_installed") deploy_manager.cluster_os_installed()