def check_rabbitmq(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('rabbitmq') return LOG.info('%s%s Checking rabbitmq cluster status' % ('=' * 5, '>')) # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # get masters & slaves node list running_nodes = get_rabbitmq_nodes() if running_nodes is None: LOG.error('Can not get the running node list for rabbitmq cluster !') return # check all controller nodes in masters + slaves node list error_nodes = [] for node in controllers: if node.split('.')[0] not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s not in rabbitmq cluster !' % error_nodes) LOG.error('Rabbitmq cluster check faild !') else: LOG.info('Rabbitmq cluster check successfully !')
def check_ceph(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): if not NODE_ROLE.is_ceph_osd(): LOG.warn('This command can only run on fuel or controller or ceph-osd node !') return if NODE_ROLE.is_fuel(): check_all_nodes('ceph') return # get cluster status LOG.info('%s%s Checking ceph cluster status' %('='*5, '>')) ceph_check_health() # check osd status LOG.info('%s%s Checking ceph osd status' %('='*5, '>')) check_success = True osd_status = get_ceph_osd_status() if not osd_status: LOG.error('Can not get ceph osd status !') check_success = False else: for l in osd_status.split('\n'): if 'id' not in l and 'weigh' not in l and 'osd.' in l: osd = l.split()[2] status = l.split()[3] if status != 'up': LOG.error('%s status is not correct, please check it !' % osd) check_success = False if check_success: LOG.info('Ceph osd status check successfully !')
def check_ceph(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): if not NODE_ROLE.is_ceph_osd(): LOG.warn( 'This command can only run on fuel or controller or ceph-osd node !' ) return if NODE_ROLE.is_fuel(): check_all_nodes('ceph') return # get cluster status LOG.info('%s%s Checking ceph cluster status' % ('=' * 5, '>')) ceph_check_health() # check osd status LOG.info('%s%s Checking ceph osd status' % ('=' * 5, '>')) check_success = True osd_status = get_ceph_osd_status() if not osd_status: LOG.error('Can not get ceph osd status !') check_success = False else: for l in osd_status.split('\n'): if 'id' not in l and 'weigh' not in l and 'osd.' in l: osd = l.split()[2] status = l.split()[3] if status != 'up': LOG.error('%s status is not correct, please check it !' % osd) check_success = False if check_success: LOG.info('Ceph osd status check successfully !')
def check_rabbitmq(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('rabbitmq') return LOG.info('%s%s Checking rabbitmq cluster status' %('='*5, '>')) # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # get masters & slaves node list running_nodes = get_rabbitmq_nodes() if running_nodes is None: LOG.error('Can not get the running node list for rabbitmq cluster !') return # check all controller nodes in masters + slaves node list error_nodes = [] for node in controllers: if node.split('.')[0] not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s not in rabbitmq cluster !' % error_nodes) LOG.error('Rabbitmq cluster check faild !') else: LOG.info('Rabbitmq cluster check successfully !')
def check_mysql(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('mysql') return LOG.info('%s%s Checking mysql cluster status' %('='*5, '>')) # get running node list for mysql cluster running_nodes = get_mysql_nodes() if running_nodes is None: LOG.error('Can not get the running node list for mysql cluster !') return # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # check all controller node in mysql cluster error_nodes = [] for node in controllers: if node not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s is not running in mysql cluster !' % error_nodes) LOG.error('Mysql cluster check faild !') else: LOG.info('Mysql cluster check successfully !')
def check_mysql(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('mysql') return LOG.info('%s%s Checking mysql cluster status' % ('=' * 5, '>')) # get running node list for mysql cluster running_nodes = get_mysql_nodes() if running_nodes is None: LOG.error('Can not get the running node list for mysql cluster !') return # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # check all controller node in mysql cluster error_nodes = [] for node in controllers: if node not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s is not running in mysql cluster !' % error_nodes) LOG.error('Mysql cluster check faild !') else: LOG.info('Mysql cluster check successfully !')
def check_all(): '''Check All Cluster''' # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('all') else: check_rabbitmq() check_mysql() check_haproxy() check_ceph()
def check_all(): '''Check All Cluster''' # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('all') else: check_rabbitmq() check_mysql() check_haproxy() check_ceph() check_pacemaker() check_cephspace()
def setup(parser): """Set things up for the upgrade operation.""" if NODE_ROLE.is_fuel(): setup_rsyncd_config() setup_nodes(parser.MYIP) else: LOG.error('This command can only be run on the fuel node.')
def stack(parser): # if node role is "unknow", go back if NODE_ROLE.is_unknown(): LOG.error('Can not confirm the node role!') return if not NODE_ROLE.is_fuel(): if parser.CONTROLLER: if not NODE_ROLE.is_controller(): cmd_warn('controller') return if parser.COMPUTE: if not NODE_ROLE.is_compute(): cmd_warn('compute') return if parser.MONGO: if not NODE_ROLE.is_mongo(): cmd_warn('mongo') return if parser.CONTROLLER or parser.COMPUTE or parser.MONGO: if parser.PROFILE and not parser.SERVICE and not parser.CHECK_ALL: if parser.CONTROLLER: check('controller', 'profile') if parser.COMPUTE: check('compute', 'profile') if parser.MONGO: check('mongo', 'profile') if parser.SERVICE and not parser.PROFILE and not parser.CHECK_ALL: if parser.CONTROLLER: check('controller', 'service') if parser.COMPUTE: check('compute', 'service') if parser.MONGO: check('mongo', 'service') if parser.SERVICE and parser.PROFILE or parser.CHECK_ALL or not parser.PROFILE and not parser.SERVICE: if parser.CONTROLLER: check('controller', 'all') if parser.COMPUTE: check('compute', 'all') if parser.MONGO: check('mongo', 'all') return # check all if parser.CHECK_ALL and parser.PROFILE and parser.SERVICE: check_all() return elif parser.CHECK_ALL and parser.PROFILE: check_all_profile() return elif parser.CHECK_ALL and parser.SERVICE: check_all_service() return elif parser.CHECK_ALL: check_all() return # check profile or service if parser.PROFILE: check_all_profile() if parser.SERVICE: check_all_service()
def check_all(): '''Check All OpenStack Component''' if not NODE_ROLE.is_fuel(): check_all_profile() check_all_service() else: for role in all_roles: check_nodes(role, 'all')
def init(parser): if NODE_ROLE.is_unknown(): LOG.error("Can not confirm the node role!") if not NODE_ROLE.is_fuel(): LOG.warn("This command can only run on fuel node !") return init_node_list_file() init_node_role_file()
def init(parser): if NODE_ROLE.is_unknown(): LOG.error('Can not confirm the node role!') if not NODE_ROLE.is_fuel(): LOG.warn('This command can only run on fuel node !') return init_node_list_file() init_node_role_file()
def check_all_service(): if NODE_ROLE.is_fuel(): for role in all_roles: check_nodes(role, 'service', multi_role=True) else: for node_role in node_roles: if node_role != 'ceph_osd': eval('check_%s_service' % node_role)()
def check_all_service(): if NODE_ROLE.is_fuel(): for role in all_roles: if role == 'ceph_osd': role = 'ceph-osd' check_nodes(role, 'service', multi_role=True) else: for node_role in node_roles: eval('check_%s_service' % node_role)()
def deployment_monitor_plugins(parser): if not NODE_ROLE.is_fuel(): LOG.warn('This command can only run on fuel node !') return if parser.INFLUXDB: deployment_influxdb_grafana(parser.ENV) if parser.LMA_COLLECTOR: deployment_lma_collector(parser.ENV)
def go(parser): """Upgrade""" if NODE_ROLE.is_fuel(): if parser.CHECK_ONLY: check_upgrade_process() else: go_upgrade(parser.MYIP) else: LOG.error('This command can only be run on the fuel node.')
def check_all_profile(): if NODE_ROLE.is_fuel(): for role in all_roles: check_nodes(role, 'profile', multi_role=True) else: for node_role in node_roles: # print node_role if node_role != 'ceph_osd': eval('check_%s_profile' % node_role)()
def check_all_profile(): if NODE_ROLE.is_fuel(): for role in all_roles: if role == 'ceph_osd': role = 'ceph-osd' check_nodes(role, 'profile', multi_role=True) else: for node_role in node_roles: # print node_role eval('check_%s_profile' % node_role)()
def init(parser): if NODE_ROLE.is_unknown(): LOG.error('Can not confirm the node role!') if not NODE_ROLE.is_fuel(): LOG.warn('This command can only run on fuel node !') return if parser.UPDATE: update() return init_env() init_node_list_file() init_node_role_file()
def check_all(): '''Check All Environement Object''' if NODE_ROLE.is_fuel(): check_cmd = get_check_cmd('all') for role in ['controller','compute','mongo','ceph-osd']: node_list = get_node_list(role) proc_list = run_doctor_on_nodes(role, node_list, check_cmd) for proc in proc_list: proc.join() else: for i in register.all: eval(i)()
def check(role, obj): if NODE_ROLE.is_fuel(): check_nodes(role, obj) else: if not eval('NODE_ROLE.is_%s' % role)(): LOG.warn('This command can only run on fuel or %s node !' % role) else: if obj == 'all': eval('check_%s_%s' % (role, 'profile'))() eval('check_%s_%s' % (role, 'service'))() else: eval('check_%s_%s' % (role, obj))()
def check_all(): '''Check All Environement Object''' if NODE_ROLE.is_fuel(): check_cmd = get_check_cmd('all') for role in ['controller', 'compute', 'mongo', 'ceph-osd']: node_list = get_node_list(role) (proc_list, pipe) = run_doctor_on_nodes(role, node_list, check_cmd) for proc in proc_list: proc.join() LOG.info(pipe.recv(), remote=True) else: for i in register.all: eval(i)()
def check_all(): check_cmd = get_check_cmd() if NODE_ROLE.is_fuel(): controllers = get_controllers_hostname() if controllers: controller_node = controllers[0] result = run_doctor_cmd_on_node('controller', controller_node, check_cmd) logging.disable(logging.NOTSET) LOG.info(result, remote=True) else: LOG.error('Can not get controller node list') elif NODE_ROLE.is_controller(): print run_command(check_cmd)
def get_node_role(): node_roles = [] if NODE_ROLE.is_unknown(): return node_roles if NODE_ROLE.is_fuel(): node_roles.append('fuel') if NODE_ROLE.is_controller(): node_roles.append('controller') if NODE_ROLE.is_compute(): node_roles.append('compute') if NODE_ROLE.is_ceph_osd(): node_roles.append('ceph_osd') if NODE_ROLE.is_mongo(): node_roles.append('mongo') return node_roles
def check_network(): nic_status = _network_get_nic_status() if NODE_ROLE.is_fuel(): LOG.debug('Network card information:') for i in nic_status.keys(): LOG.debug('%s: %s' % (i, nic_status[i])) return cfg = yaml.load(file('/etc/astute.yaml')) # check node's nic status local_inf = _network_local_network_inf(cfg) _network_check_local(local_inf, nic_status) # check if node can connect to other node remote_inf = _network_remote_network_inf(cfg) _network_check_remote(remote_inf)
def check_all(): """Check All Environement Object""" if NODE_ROLE.is_fuel(): for role in ["controller", "compute", "mongo", "ceph-osd"]: node_list = get_node_list(role) for node in node_list: LOG.info("%s Role: %-10s Node: %-13s %s" % ("*" * 15, role, node, "*" * 15)) if LOG.enable_debug: out, err = ssh_connect2(node, "sudo eayunstack --debug doctor env -a") else: out, err = ssh_connect2(node, "sudo eayunstack doctor env -a") if err: LOG.error("Check failed !") else: for i in register.all: eval(i)()
def vrouter_check(parser): if NODE_ROLE.is_fuel(): # TODO: run on fuel node on future LOG.error('This check can be run only on network node') return # pick one controller to run controller_node = None for node in NODE_ROLE.nodes: if node['roles'] == 'controller': controller_node = node['host'] cmd = 'source /root/openrc;%s' % (' '.join(sys.argv)) ssh_connect2(controller_node, cmd) elif NODE_ROLE.is_controller(): _vrouter_check(parser) else: LOG.error('This check can be run only on network node')
def vrouter_check(parser): if NODE_ROLE.is_fuel(): # TODO: run on fuel node on future LOG.error("This check can be run only on network node") return # pick one controller to run controller_node = None for node in NODE_ROLE.nodes: if node["roles"] == "controller": controller_node = node["host"] cmd = "source /root/openrc;%s" % (" ".join(sys.argv)) ssh_connect2(controller_node, cmd) elif NODE_ROLE.is_controller(): _vrouter_check(parser) else: LOG.error("This check can be run only on network node")
def check_all(): '''Check All Environement Object''' if NODE_ROLE.is_fuel(): for role in ['controller','compute','mongo','ceph-osd']: node_list = get_node_list(role) for node in node_list: LOG.info('%s Role: %-10s Node: %-13s %s' % ('*'*15, role, node, '*'*15)) if LOG.enable_debug: out, err = ssh_connect2(node, 'sudo eayunstack --debug doctor env -a') else: out, err = ssh_connect2(node, 'sudo eayunstack doctor env -a') if err: LOG.error('Check failed !') else: for i in register.all: eval(i)()
def check_all(): '''Check All Environement Object''' if NODE_ROLE.is_fuel(): check_cmd = get_check_cmd('all') nodes = [] for role in ['controller', 'compute', 'mongo', 'ceph-osd']: node_list = get_node_list(role) for node in node_list: node_info = {} node_info['role'] = role node_info['name'] = node nodes.append(node_info) result = run_doctor_on_nodes(nodes, check_cmd) for res in result: LOG.info(res, remote=True) else: for i in register.all: eval(i)()
def check_all(): '''Check All Environement Object''' if NODE_ROLE.is_fuel(): check_cmd = get_check_cmd('all') nodes = [] for role in ['controller','compute','mongo','ceph-osd']: node_list = get_node_list(role) for node in node_list: node_info = {} node_info['role'] = role node_info['name'] = node nodes.append(node_info) result = run_doctor_on_nodes(nodes, check_cmd) for res in result: LOG.info(res, remote=True) else: for i in register.all: eval(i)()