Exemple #1
0
def main():
    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Poll for alerts', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        logger.log_or_print('Poll for alerts initiated.', lg, level='info')

        lck, err = lock.get_lock('poll_for_alerts')
        if err:
            raise Exception(err)
        if not lck:
            raise Exception('Could not acquire lock. Exiting.')

        active, err = grid_ops.is_active_admin_gridcell()
        if err:
            raise Exception(err)

        if not active:
            logger.log_or_print(
                'Not active admin GRIDCell so exiting.', lg, level='info')
            sys.exit(0)

        gluster_lck, err = lock.get_lock('gluster_commands')
        if err:
            raise Exception(err)

        si, err = system_info.load_system_config()
        if err:
            raise Exception(err)

        if not si:
            raise Exception('Could not load system information')

        alerts_list = []

        alerts_list, err = check_quotas()
        if err:
            raise Exception("Error getting quota information : %s" % err)

        lock.release_lock('gluster_commands')

        common_alerts, err = check_for_gridcell_errors(si)
        if err:
            raise Exception(err)

        alerts_list.extend(common_alerts)

        if alerts_list:
            alerts.raise_alert(alerts_list)
            str = ' | '.join(alerts_list)
            logger.log_or_print(str, lg, level='info')
        else:
            logger.log_or_print('No alerts to raise', lg, level='info')

        lock.release_lock('poll_for_alerts')
    except Exception, e:
        str = 'Error running poll for alerts  : %s' % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
def salt_start():
    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Conditional salt start completed', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        logger.log_or_print(
            'Conditional salt start initiated.', lg, level='info')
        pog, err = grid_ops.is_part_of_grid()
        logger.log_or_print('Part of grid : %s' % pog, lg, level='info')
        if err:
            raise Exception(err)
        if not pog:
            logger.log_or_print('Starting salt services.', lg, level='info')
            out, err = command.get_command_output(
                'service salt-minion start', False)
            if err:
                raise Exception(err)
            out, err = command.get_command_output(
                'service salt-master start', False)
            if err:
                raise Exception(err)
        else:
            logger.log_or_print(
                'Not starting salt services as I am part of the grid.', lg, level='info')
    except Exception, e:
        str = 'Error doing a conditional salt start : %s' % e
        logger.log_or_print(str, lg, level='critical')
        return False, str
def gen_status(path, lg=None):
    try:
        lck, err = lock.get_lock('generate_status')
        if err:
            raise Exception(err)
        if not lck:
            raise Exception('Generate Status : Could not acquire lock.')
        fullmanifestpath = os.path.normpath("%s/master.manifest" % path)
        ret, err = manifest_status.generate_status_info(fullmanifestpath)
        if not ret:
            if err:
                raise Exception(err)
            else:
                raise Exception('No status info obtained')
        fullpath = os.path.normpath("%s/master.status" % path)
        fulltmppath = "/tmp/master.status.tmp"
        # Generate into a tmp file
        with open(fulltmppath, 'w') as fd:
            json.dump(ret, fd, indent=2)
        # Now move the tmp to the actual manifest file name
        # print 'fullpath is ', fullpath
        shutil.move(fulltmppath, fullpath)
    except Exception, e:
        logger.log_or_print('Error generating status : %s' %
                            e, lg, level='critical')
        lock.release_lock('generate_status')
        return -1,  'Error generating status : %s' % e
Exemple #4
0
def gen_status(path, lg=None):
    try:
        lck, err = lock.get_lock('generate_status')
        if err:
            raise Exception(err)
        if not lck:
            raise Exception('Generate Status : Could not acquire lock.')
        fullmanifestpath = os.path.normpath("%s/master.manifest" % path)
        ret, err = manifest_status.generate_status_info(fullmanifestpath)
        if not ret:
            if err:
                raise Exception(err)
            else:
                raise Exception('No status info obtained')
        fullpath = os.path.normpath("%s/master.status" % path)
        fulltmppath = "/tmp/master.status.tmp"
        # Generate into a tmp file
        with open(fulltmppath, 'w') as fd:
            json.dump(ret, fd, indent=2)
        # Now move the tmp to the actual manifest file name
        # print 'fullpath is ', fullpath
        shutil.move(fulltmppath, fullpath)
    except Exception, e:
        logger.log_or_print('Error generating status : %s' % e,
                            lg,
                            level='critical')
        lock.release_lock('generate_status')
        return -1, 'Error generating status : %s' % e
Exemple #5
0
def salt_start():
    lg = None
    try:
        lg, err = logger.get_script_logger('Conditional salt start completed',
                                           '/var/log/integralstor/scripts.log',
                                           level=logging.DEBUG)

        logger.log_or_print('Conditional salt start initiated.',
                            lg,
                            level='info')
        pog, err = grid_ops.is_part_of_grid()
        logger.log_or_print('Part of grid : %s' % pog, lg, level='info')
        if err:
            raise Exception(err)
        if not pog:
            logger.log_or_print('Starting salt services.', lg, level='info')
            out, err = command.get_command_output('service salt-minion start',
                                                  False)
            if err:
                raise Exception(err)
            out, err = command.get_command_output('service salt-master start',
                                                  False)
            if err:
                raise Exception(err)
        else:
            logger.log_or_print(
                'Not starting salt services as I am part of the grid.',
                lg,
                level='info')
    except Exception, e:
        str = 'Error doing a conditional salt start : %s' % e
        logger.log_or_print(str, lg, level='critical')
        return False, str
Exemple #6
0
def main():
    lg = None
    action = 'Config backup'
    try:
        lg, err = logger.get_script_logger(
            'Config backup', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        if len(sys.argv) != 2 or sys.argv[1].strip() not in ['backup_gridcell_config', 'backup_grid_config']:
            raise Exception(
                'Usage: python config_backup.py [backup_gridcell_config|backup_grid_config]')

        if sys.argv[1].strip() == 'backup_gridcell_config':
            action = 'GRIDCell config backup'
        else:
            action = 'Grid config backup'
        str = '%s initiated.' % action
        logger.log_or_print(str, lg, level='info')

        if sys.argv[1].strip() == 'backup_gridcell_config':
            ret, err = zip_gridcell_gluster_config()
        else:
            ret, err = zip_grid_gluster_config()
        if err:
            raise Exception(err)
    except Exception, e:
        st = 'Error backing up config: %s' % e
        logger.log_or_print(st, lg, level='critical')
        sys.exit(-1)
Exemple #7
0
def main():

    lg = None
    try:
        lg, err = logger.get_script_logger('Generate status',
                                           '/var/log/integralstor/scripts.log',
                                           level=logging.DEBUG)

        logger.log_or_print('Generate status initiated.', lg, level='info')

        platform, err = config.get_platform()
        if err:
            raise Exception(err)

        default_path = False

        num_args = len(sys.argv)

        if num_args > 1:
            path = sys.argv[1]
        else:
            default_path = True
            path, err = config.get_system_status_path()
            if err:
                raise Exception(err)
            if not path:
                path = '/tmp'
        # print platform, path

        if platform == 'gridcell' and default_path:
            # This means that I must've been called from a cron script so need
            # to check if I really need to execute..
            from integralstor_gridcell import grid_ops
            active, err = grid_ops.is_active_admin_gridcell()
            if err:
                raise Exception(err)
            if not active:
                logger.log_or_print('Not active admin GRIDCell so exiting.',
                                    lg,
                                    level='info')
                sys.exit(0)

        logger.log_or_print("Generating the status in %s" % path,
                            lg,
                            level='info')
        rc, err = gen_status(path, lg)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating status file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
def main():

    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Task processor', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        logger.log_or_print(
            'Task processor execution initiated.', lg, level='info')

        db_path, err = config.get_db_path()
        if err:
            raise Exception(err)
        ret, err = scheduler_utils.process_tasks()
        if err:
            raise Exception(err)
    except Exception, e:
        str = 'Error running the task processor : %s' % e
        logger.log_or_print(str, lg, level='critical')
        return -1
def main():

    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Generate status', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        logger.log_or_print('Generate status initiated.', lg, level='info')

        platform, err = config.get_platform()
        if err:
            raise Exception(err)

        default_path = False

        num_args = len(sys.argv)

        if num_args > 1:
            path = sys.argv[1]
        else:
            default_path = True
            path, err = config.get_system_status_path()
            if err:
                raise Exception(err)
            if not path:
                path = '/tmp'
        # print platform, path

        if platform == 'gridcell' and default_path:
            # This means that I must've been called from a cron script so need
            # to check if I really need to execute..
            from integralstor_gridcell import grid_ops
            active, err = grid_ops.is_active_admin_gridcell()
            if err:
                raise Exception(err)
            if not active:
                logger.log_or_print(
                    'Not active admin GRIDCell so exiting.', lg, level='info')
                sys.exit(0)

        logger.log_or_print("Generating the status in %s" %
                            path, lg, level='info')
        rc, err = gen_status(path, lg)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating status file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
def main():

    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Generate manifest', '/var/log/integralstor/scripts.log', level=logging.DEBUG)
        logger.log_or_print('Generate manifest initiated.', lg, level='info')

        num_args = len(sys.argv)
        if num_args > 1:
            path = sys.argv[1]
        else:
            path, err = config.get_system_status_path()
            if err:
                raise Exception(err)
            if not path:
                path = '/tmp'
        logger.log_or_print("Generating the manifest in %s" %
                            path, lg, level='info')
        rc, err = gen_manifest(path)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating manifest file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        return -1
Exemple #11
0
def main():

    lg = None
    try:
        lg, err = logger.get_script_logger('Generate manifest',
                                           '/var/log/integralstor/scripts.log',
                                           level=logging.DEBUG)
        logger.log_or_print('Generate manifest initiated.', lg, level='info')

        num_args = len(sys.argv)
        if num_args > 1:
            path = sys.argv[1]
        else:
            path, err = config.get_system_status_path()
            if err:
                raise Exception(err)
            if not path:
                path = '/tmp'
        logger.log_or_print("Generating the manifest in %s" % path,
                            lg,
                            level='info')
        rc, err = gen_manifest(path)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating manifest file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        return -1
def main():
    lg = None
    action = 'Log backup'
    try:
        lg, err = logger.get_script_logger('Log backup',
                                           '/var/log/integralstor/scripts.log',
                                           level=logging.DEBUG)

        if len(sys.argv) != 2 or sys.argv[1].strip() not in [
                'backup_gridcell_logs', 'backup_grid_logs'
        ]:
            raise Exception(
                'Usage: python log_backup.py [backup_gridcell_logs|backup_grid_logs]'
            )

        if sys.argv[1].strip() == 'backup_gridcell_logs':
            action = 'GRIDCell Log backup'
        else:
            action = 'Grid Log backup'

        str = '%s initiated.' % action
        logger.log_or_print(str, lg, level='info')

        if sys.argv[1].strip() == 'backup_gridcell_logs':
            ret, err = zip_gridcell_logs()
        else:
            active, err = grid_ops.is_active_admin_gridcell()
            if err:
                raise Exception(err)

            if not active:
                logger.log_or_print('Not active admin GRIDCell so exiting.',
                                    lg,
                                    level='info')
                sys.exit(0)
            ret, err = zip_grid_logs()
        if err:
            raise Exception(err)
    except Exception, e:
        st = 'Error backing up logs: %s' % e
        logger.log_or_print(st, lg, level='critical')
        sys.exit(-1)
Exemple #13
0
            path = sys.argv[1]
        else:
            path, err = config.get_system_status_path()
            if err:
                raise Exception(err)
            if not path:
                path = '/tmp'
        logger.log_or_print("Generating the manifest in %s" % path,
                            lg,
                            level='info')
        rc, err = gen_manifest(path)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating manifest file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        return -1
    else:
        logger.log_or_print('Generate manifest completed successfully',
                            lg,
                            level='info')
        return 0


if __name__ == "__main__":
    ret = main()
    sys.exit(ret)

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
def sync_ctdb_files():
    """
    Syncs CTDB files on the localhost with the mounted admin vol.

    Input:
      ['None']
    Output:
      Returns three variables [ret1,ret2,ret3]:
        ret1 -- True if sync was sucessful, else False
        ret2 -- True if there was a difference, False if they were already in sync
        ret3 -- None if there were no errors/exceptions, 'Error string' otherwise
    """

    is_change = False
    try:
        config_dir, err = config.get_config_dir()
        if err:
            raise Exception(err)

        out, err = command.get_command_output(
            'diff "/etc/sysconfig/ctdb" "%s/lock/ctdb"' % str(config_dir), True, True)
        if err:
            shutil.copyfile('%s/lock/ctdb' %
                            str(config_dir), '/etc/sysconfig/ctdb')
            is_change = True

        out, err = command.get_command_output(
            'diff "/etc/ctdb/nodes" "%s/lock/nodes"' % str(config_dir), True, True)
        if err:
            shutil.copyfile('%s/lock/nodes' %
                            str(config_dir), '/etc/ctdb/nodes')
            is_change = True

        out, err = command.get_command_output(
            'diff "/etc/ctdb/public_addresses" "%s/lock/public_addresses"' % str(config_dir), True, True)
        if err:
            shutil.copyfile('%s/lock/public_addresses' %
                            str(config_dir), '/etc/ctdb/public_addresses')
            is_change = True

        lg, err = logger.get_script_logger(
            'Admin volume mounter: Sync CTDB config files', '/var/log/integralstor/scripts.log', level=logging.DEBUG)
        if is_change == True:
            logger.log_or_print(
                'ctdb related files were synced.', lg, level='info')
            logger.log_or_print(
                'Restarting ctdb.', lg, level='debug')
            out, err = command.get_command_output(
                'service ctdb restart', False, True)
            if not err and out:
                logger.log_or_print(
                    'Service ctdb: %s' % out, lg, level='debug')
            else:
                logger.log_or_print(
                    'Service ctdb error: %s' % err, lg, level='error')
        elif is_change == False:
            logger.log_or_print(
                'ctdb related files are in sync.', lg, level='info')

    except Exception, e:
        return False, is_change, "Couldn't sync ctdb files: %s" % str(e)
def mount_and_configure():
    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Admin volume mounter', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        logger.log_or_print(
            'Admin volume mounter initiated.', lg, level='info')

        pog, err = grid_ops.is_part_of_grid()
        if err:
            raise Exception(err)

        # If the localhost is part of the gridcell, proceed
        if pog:
            logger.log_or_print('Checking glusterd service', lg, level='debug')
            service = 'glusterd'
            status, err = services_management.get_service_status([service])
            if err:
                raise Exception(err)
            logger.log_or_print('Service %s status is %s' % (
                service, status['status_code']), lg, level='debug')
            if status['status_code'] != 0:
                logger.log_or_print(
                    'Service %s not started so restarting' % service, lg, level='error')
                out, err = command.get_command_output(
                    'service %s restart' % service, False, True)
                if not err and out:
                    logger.log_or_print('Service %s: %s' %
                                        (service, out), lg, level='debug')
                else:
                    logger.log_or_print('Service %s error : %s' % (
                        service, err), lg, level='error')

            admin_vol_name, err = config.get_admin_vol_name()
            if err:
                raise Exception(err)

            # Get the config dir - the mount point.
            config_dir, err = config.get_config_dir()
            if err:
                raise Exception(err)

            ag, err = grid_ops.is_admin_gridcell()
            if err:
                raise Exception(err)

            admin_gridcells, err = grid_ops.get_admin_gridcells()
            if err:
                raise Exception(err)

            is_pooled = False
            peer_list, err = gluster_trusted_pools.get_peer_list()
            if peer_list:
                is_pooled = True

            is_mounted = False
            # mount only if the localhost is pooled
            if is_pooled:
                is_mounted, err = grid_ops.is_admin_vol_mounted_local()
                if not is_mounted:
                    str = 'Admin volume is not mounted. Will attempt to mount now.' 
                    logger.log_or_print(str, lg, level='error')

                    # Try to mount
                    (ret, rc), err = command.execute_with_rc(
                        'mount -t glusterfs localhost:/%s %s' % (admin_vol_name, config_dir))
                    if err:
                        str = 'Mount from localhost failed.' 
                        logger.log_or_print(str, lg, level='error')
                    elif (not err) and (rc == 0):
                        is_access, err = assert_admin_vol_mount()
                        if err:
                            raise Exception(err)

                        sync, is_change, error = sync_ctdb_files()
                        if error:
                            # It's only a best-effort, it will try next
                            # minute again.
                            pass
                        if sync == False:
                            #raise Exception (err)
                            pass

                        # Restart nginx
                        out, err = command.get_command_output(
                            'service nginx restart', False, True)
                        if not err and out:
                            logger.log_or_print(
                                'Service nginx: %s' % out, lg, level='debug')
                        else:
                            logger.log_or_print(
                                'Service nginx error : %s' % err, lg, level='error')
                        # Restart uwsgi
                        out, err = command.get_command_output(
                            'service uwsgi restart', False, True)
                        if not err and out:
                            logger.log_or_print(
                                'Service uwsgi: %s' % out, lg, level='debug')
                        else:
                            logger.log_or_print(
                                'Service uwsgi error : %s' % err, lg, level='error')

                        if ag:
                            # Restart salt-master
                            out, err = command.get_command_output(
                                'service salt-master restart', False, True)
                            if not err and out:
                                logger.log_or_print(
                                    'Service salt-master: %s' % out, lg, level='debug')
                            else:
                                logger.log_or_print(
                                    'Service salt-master error : %s' % err, lg, level='error')
                            # Restart salt-minion
                        out, err = command.get_command_output(
                            'service salt-minion restart', False, True)
                        if not err and out:
                            logger.log_or_print(
                                'Service salt-minion: %s' % out, lg, level='debug')
                        else:
                            logger.log_or_print(
                                'Service salt-minion error : %s' % err, lg, level='error')

                    str = 'Admin vol is mounted'
                    logger.log_or_print(str, lg, level='info')

                # Admin volume is mounted, perform required checks
                else:
                    sync, is_change, err = sync_ctdb_files()
                    if err:
                        raise Exception(err)
                    if sync == False:
                        raise Exception(err)

                    logger.log_or_print('Checking services', lg, level='debug')
                    service_list = ['nginx','ctdb','salt-minion']
                    if ag:
                        service_list.append('salt-master')

                    for service in service_list:
                        status, err = services_management.get_service_status([
                                                                             service])
                        if err:
                            raise Exception(err)
                        logger.log_or_print('Service %s status is %s' % (
                            service, status['status_code']), lg, level='debug')
                        if status['status_code'] != 0:
                            logger.log_or_print(
                                'Service %s is not active, restarting' % service, lg, level='error')
                            out, err = command.get_command_output(
                                'service %s restart' % service, False, True)
                            if not err and out:
                                logger.log_or_print('Service %s: %s' % (
                                    service, out), lg, level='debug')
                            else:
                                logger.log_or_print('Service %s error : %s' % (
                                    service, err), lg, level='error')

                    # UWSGI service config not complete so need to check
                    # against the actual process name
                    (ret, rc), err = command.execute_with_rc(
                        'pidof uwsgi', shell=True)
                    if rc != 0:
                        logger.log_or_print(
                            'Service uwsgi is not active, restarting', lg, level='error')
                        out, err = command.get_command_output(
                            'service uwsgi restart', False, True)
                        if not err and out:
                            logger.log_or_print(
                                'Service uwsgi: %s' % out, lg, level='debug')
                        else:
                            logger.log_or_print(
                                'Service uwsgi error : %s' % err, lg, level='error')

                    str = 'Admin volume is already mounted'
                    logger.log_or_print(str, lg, level='info')

    except Exception, e:
        st = 'Error mounting admin volume : %s' % e
        logger.log_or_print(st, lg, level='critical')
        return False, st
                            logger.log_or_print(
                                'Service uwsgi: %s' % out, lg, level='debug')
                        else:
                            logger.log_or_print(
                                'Service uwsgi error : %s' % err, lg, level='error')

                    str = 'Admin volume is already mounted'
                    logger.log_or_print(str, lg, level='info')

    except Exception, e:
        st = 'Error mounting admin volume : %s' % e
        logger.log_or_print(st, lg, level='critical')
        return False, st
    else:
        str = 'Admin volume mounter completed.'
        logger.log_or_print(str, lg, level='info')
        return True, None


def main():
    ret, err = mount_and_configure()
    print ret, err
    if err:
        print err
        sys.exit(-1)


if __name__ == '__main__':
    main()

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
Exemple #17
0
            if err:
                raise Exception(err)
            if not active:
                logger.log_or_print('Not active admin GRIDCell so exiting.',
                                    lg,
                                    level='info')
                sys.exit(0)

        logger.log_or_print("Generating the status in %s" % path,
                            lg,
                            level='info')
        rc, err = gen_status(path, lg)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating status file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
    else:
        logger.log_or_print('Generate status completed successfully.',
                            lg,
                            level='info')
        sys.exit(0)


if __name__ == "__main__":
    main()

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
            logger.log_or_print('Starting salt services.', lg, level='info')
            out, err = command.get_command_output(
                'service salt-minion start', False)
            if err:
                raise Exception(err)
            out, err = command.get_command_output(
                'service salt-master start', False)
            if err:
                raise Exception(err)
        else:
            logger.log_or_print(
                'Not starting salt services as I am part of the grid.', lg, level='info')
    except Exception, e:
        str = 'Error doing a conditional salt start : %s' % e
        logger.log_or_print(str, lg, level='critical')
        return False, str
    else:
        logger.log_or_print(
            'Conditional salt start completed.', lg, level='info')
        return True, None


if __name__ == '__main__':
    ret, err = salt_start()
    if not ret:
        sys.exit(-1)
    else:
        sys.exit(0)

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
def sync_ctdb_files():
    """
    Syncs CTDB files on the localhost with the mounted admin vol.

    Input:
      ['None']
    Output:
      Returns three variables [ret1,ret2,ret3]:
        ret1 -- True if sync was sucessful, else False
        ret2 -- True if there was a difference, False if they were already in sync
        ret3 -- None if there were no errors/exceptions, 'Error string' otherwise
    """

    is_change = False
    try:
        config_dir, err = config.get_config_dir()
        if err:
            raise Exception(err)

        out, err = command.get_command_output(
            'diff "/etc/sysconfig/ctdb" "%s/lock/ctdb"' % str(config_dir),
            True, True)
        if err:
            shutil.copyfile('%s/lock/ctdb' % str(config_dir),
                            '/etc/sysconfig/ctdb')
            is_change = True

        out, err = command.get_command_output(
            'diff "/etc/ctdb/nodes" "%s/lock/nodes"' % str(config_dir), True,
            True)
        if err:
            shutil.copyfile('%s/lock/nodes' % str(config_dir),
                            '/etc/ctdb/nodes')
            is_change = True

        out, err = command.get_command_output(
            'diff "/etc/ctdb/public_addresses" "%s/lock/public_addresses"' %
            str(config_dir), True, True)
        if err:
            shutil.copyfile('%s/lock/public_addresses' % str(config_dir),
                            '/etc/ctdb/public_addresses')
            is_change = True

        lg, err = logger.get_script_logger(
            'Admin volume mounter: Sync CTDB config files',
            '/var/log/integralstor/scripts.log',
            level=logging.DEBUG)
        if is_change == True:
            logger.log_or_print('ctdb related files were synced.',
                                lg,
                                level='info')
            logger.log_or_print('Restarting ctdb.', lg, level='debug')
            out, err = command.get_command_output('service ctdb restart',
                                                  False, True)
            if not err and out:
                logger.log_or_print('Service ctdb: %s' % out,
                                    lg,
                                    level='debug')
            else:
                logger.log_or_print('Service ctdb error: %s' % err,
                                    lg,
                                    level='error')
        elif is_change == False:
            logger.log_or_print('ctdb related files are in sync.',
                                lg,
                                level='info')

    except Exception, e:
        return False, is_change, "Couldn't sync ctdb files: %s" % str(e)
def mount_and_configure():
    lg = None
    try:
        lg, err = logger.get_script_logger('Admin volume mounter',
                                           '/var/log/integralstor/scripts.log',
                                           level=logging.DEBUG)

        logger.log_or_print('Admin volume mounter initiated.',
                            lg,
                            level='info')

        pog, err = grid_ops.is_part_of_grid()
        if err:
            raise Exception(err)

        # If the localhost is part of the gridcell, proceed
        if pog:
            logger.log_or_print('Checking glusterd service', lg, level='debug')
            service = 'glusterd'
            status, err = services_management.get_service_status([service])
            if err:
                raise Exception(err)
            logger.log_or_print('Service %s status is %s' %
                                (service, status['status_code']),
                                lg,
                                level='debug')
            if status['status_code'] != 0:
                logger.log_or_print('Service %s not started so restarting' %
                                    service,
                                    lg,
                                    level='error')
                out, err = command.get_command_output(
                    'service %s restart' % service, False, True)
                if not err and out:
                    logger.log_or_print('Service %s: %s' % (service, out),
                                        lg,
                                        level='debug')
                else:
                    logger.log_or_print('Service %s error : %s' %
                                        (service, err),
                                        lg,
                                        level='error')

            admin_vol_name, err = config.get_admin_vol_name()
            if err:
                raise Exception(err)

            # Get the config dir - the mount point.
            config_dir, err = config.get_config_dir()
            if err:
                raise Exception(err)

            ag, err = grid_ops.is_admin_gridcell()
            if err:
                raise Exception(err)

            admin_gridcells, err = grid_ops.get_admin_gridcells()
            if err:
                raise Exception(err)

            is_pooled = False
            peer_list, err = gluster_trusted_pools.get_peer_list()
            if peer_list:
                is_pooled = True

            is_mounted = False
            # mount only if the localhost is pooled
            if is_pooled:
                is_mounted, err = grid_ops.is_admin_vol_mounted_local()
                if not is_mounted:
                    str = 'Admin volume is not mounted. Will attempt to mount now.'
                    logger.log_or_print(str, lg, level='error')

                    # Try to mount
                    (ret, rc), err = command.execute_with_rc(
                        'mount -t glusterfs localhost:/%s %s' %
                        (admin_vol_name, config_dir))
                    if err:
                        str = 'Mount from localhost failed.'
                        logger.log_or_print(str, lg, level='error')
                    elif (not err) and (rc == 0):
                        is_access, err = assert_admin_vol_mount()
                        if err:
                            raise Exception(err)

                        sync, is_change, error = sync_ctdb_files()
                        if error:
                            # It's only a best-effort, it will try next
                            # minute again.
                            pass
                        if sync == False:
                            #raise Exception (err)
                            pass

                        # Restart nginx
                        out, err = command.get_command_output(
                            'service nginx restart', False, True)
                        if not err and out:
                            logger.log_or_print('Service nginx: %s' % out,
                                                lg,
                                                level='debug')
                        else:
                            logger.log_or_print('Service nginx error : %s' %
                                                err,
                                                lg,
                                                level='error')
                        # Restart uwsgi
                        out, err = command.get_command_output(
                            'service uwsgi restart', False, True)
                        if not err and out:
                            logger.log_or_print('Service uwsgi: %s' % out,
                                                lg,
                                                level='debug')
                        else:
                            logger.log_or_print('Service uwsgi error : %s' %
                                                err,
                                                lg,
                                                level='error')

                        if ag:
                            # Restart salt-master
                            out, err = command.get_command_output(
                                'service salt-master restart', False, True)
                            if not err and out:
                                logger.log_or_print('Service salt-master: %s' %
                                                    out,
                                                    lg,
                                                    level='debug')
                            else:
                                logger.log_or_print(
                                    'Service salt-master error : %s' % err,
                                    lg,
                                    level='error')
                            # Restart salt-minion
                        out, err = command.get_command_output(
                            'service salt-minion restart', False, True)
                        if not err and out:
                            logger.log_or_print('Service salt-minion: %s' %
                                                out,
                                                lg,
                                                level='debug')
                        else:
                            logger.log_or_print(
                                'Service salt-minion error : %s' % err,
                                lg,
                                level='error')

                    str = 'Admin vol is mounted'
                    logger.log_or_print(str, lg, level='info')

                # Admin volume is mounted, perform required checks
                else:
                    sync, is_change, err = sync_ctdb_files()
                    if err:
                        raise Exception(err)
                    if sync == False:
                        raise Exception(err)

                    logger.log_or_print('Checking services', lg, level='debug')
                    service_list = ['nginx', 'ctdb', 'salt-minion']
                    if ag:
                        service_list.append('salt-master')

                    for service in service_list:
                        status, err = services_management.get_service_status(
                            [service])
                        if err:
                            raise Exception(err)
                        logger.log_or_print('Service %s status is %s' %
                                            (service, status['status_code']),
                                            lg,
                                            level='debug')
                        if status['status_code'] != 0:
                            logger.log_or_print(
                                'Service %s is not active, restarting' %
                                service,
                                lg,
                                level='error')
                            out, err = command.get_command_output(
                                'service %s restart' % service, False, True)
                            if not err and out:
                                logger.log_or_print('Service %s: %s' %
                                                    (service, out),
                                                    lg,
                                                    level='debug')
                            else:
                                logger.log_or_print('Service %s error : %s' %
                                                    (service, err),
                                                    lg,
                                                    level='error')

                    # UWSGI service config not complete so need to check
                    # against the actual process name
                    (ret, rc), err = command.execute_with_rc('pidof uwsgi',
                                                             shell=True)
                    if rc != 0:
                        logger.log_or_print(
                            'Service uwsgi is not active, restarting',
                            lg,
                            level='error')
                        out, err = command.get_command_output(
                            'service uwsgi restart', False, True)
                        if not err and out:
                            logger.log_or_print('Service uwsgi: %s' % out,
                                                lg,
                                                level='debug')
                        else:
                            logger.log_or_print('Service uwsgi error : %s' %
                                                err,
                                                lg,
                                                level='error')

                    str = 'Admin volume is already mounted'
                    logger.log_or_print(str, lg, level='info')

    except Exception, e:
        st = 'Error mounting admin volume : %s' % e
        logger.log_or_print(st, lg, level='critical')
        return False, st
                        else:
                            logger.log_or_print('Service uwsgi error : %s' %
                                                err,
                                                lg,
                                                level='error')

                    str = 'Admin volume is already mounted'
                    logger.log_or_print(str, lg, level='info')

    except Exception, e:
        st = 'Error mounting admin volume : %s' % e
        logger.log_or_print(st, lg, level='critical')
        return False, st
    else:
        str = 'Admin volume mounter completed.'
        logger.log_or_print(str, lg, level='info')
        return True, None


def main():
    ret, err = mount_and_configure()
    print ret, err
    if err:
        print err
        sys.exit(-1)


if __name__ == '__main__':
    main()

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
            # to check if I really need to execute..
            from integralstor_gridcell import grid_ops
            active, err = grid_ops.is_active_admin_gridcell()
            if err:
                raise Exception(err)
            if not active:
                logger.log_or_print(
                    'Not active admin GRIDCell so exiting.', lg, level='info')
                sys.exit(0)

        logger.log_or_print("Generating the status in %s" %
                            path, lg, level='info')
        rc, err = gen_status(path, lg)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating status file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
    else:
        logger.log_or_print(
            'Generate status completed successfully.', lg, level='info')
        sys.exit(0)


if __name__ == "__main__":
    main()

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
Exemple #23
0
            if err:
                raise Exception(err)
            out, err = command.get_command_output('service salt-master start',
                                                  False)
            if err:
                raise Exception(err)
        else:
            logger.log_or_print(
                'Not starting salt services as I am part of the grid.',
                lg,
                level='info')
    except Exception, e:
        str = 'Error doing a conditional salt start : %s' % e
        logger.log_or_print(str, lg, level='critical')
        return False, str
    else:
        logger.log_or_print('Conditional salt start completed.',
                            lg,
                            level='info')
        return True, None


if __name__ == '__main__':
    ret, err = salt_start()
    if not ret:
        sys.exit(-1)
    else:
        sys.exit(0)

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
        num_args = len(sys.argv)
        if num_args > 1:
            path = sys.argv[1]
        else:
            path, err = config.get_system_status_path()
            if err:
                raise Exception(err)
            if not path:
                path = '/tmp'
        logger.log_or_print("Generating the manifest in %s" %
                            path, lg, level='info')
        rc, err = gen_manifest(path)
        if err:
            raise Exception(err)
        # print rc
    except Exception, e:
        str = "Error generating manifest file : %s" % e
        logger.log_or_print(str, lg, level='critical')
        return -1
    else:
        logger.log_or_print(
            'Generate manifest completed successfully', lg, level='info')
        return 0


if __name__ == "__main__":
    ret = main()
    sys.exit(ret)

# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
Exemple #25
0
        lock.release_lock('gluster_commands')

        common_alerts, err = check_for_gridcell_errors(si)
        if err:
            raise Exception(err)

        alerts_list.extend(common_alerts)

        if alerts_list:
            alerts.raise_alert(alerts_list)
            str = ' | '.join(alerts_list)
            logger.log_or_print(str, lg, level='info')
        else:
            logger.log_or_print('No alerts to raise', lg, level='info')

        lock.release_lock('poll_for_alerts')
    except Exception, e:
        str = 'Error running poll for alerts  : %s' % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
    else:
        logger.log_or_print('Poll for alerts completed.', lg, level='info')
        sys.exit(0)


if __name__ == "__main__":
    main()


# vim: tabstop=8 softtabstop=0 expandtab ai shiftwidth=4 smarttab
Exemple #26
0
def main():
    lg = None
    try:
        lg, err = logger.get_script_logger('Gluster batch processing',
                                           '/var/log/integralstor/scripts.log',
                                           level=logging.DEBUG)

        logger.log_or_print('Batch processing initiated.', lg, level='info')

        active, err = grid_ops.is_active_admin_gridcell()
        if err:
            raise Exception(err)

        if not active:
            logger.log_or_print('Not active admin GRIDCell so exiting.',
                                lg,
                                level='info')
            sys.exit(0)

        batch_files_path, err = config.get_batch_files_path()
        if err:
            raise Exception(err)

        ret, err = lock.get_lock('batch_process')
        if err:
            raise Exception(err)

        if not ret:
            raise Exception('Could not acquire batch lock. Exiting.')

        gluster_lck, err = lock.get_lock('gluster_commands')
        if err:
            raise Exception(err)

        if not gluster_lck:
            raise Exception('Could not acquire gluster lock. Exiting.')

        fl = os.listdir(os.path.normpath(batch_files_path))
        if fl:
            for file in fl:
                if not file.startswith("bp_"):
                    # unknown file type so ignore
                    continue
                else:
                    logger.log_or_print('Processing file %s/%s' %
                                        (batch_files_path, file),
                                        lg,
                                        level='info')
                    with open(
                            os.path.normpath(
                                "%s/%s" % (batch_files_path, file)), "r") as f:
                        # print 'a'
                        # print
                        # os.path.normpath("%s/%s"%(batch_files_path,file))
                        d = json.load(f)
                        # print 'a1'
                        ret, err = process_batch(d, file, logger)
                        if err:
                            str = "Error loading json content for %s/%s : %s" % (
                                batch_files_path, file, err)
                            logger.log_or_print(str, lg, level='error')
                            continue
        else:
            logger.log_or_print('No batch processes pending.',
                                lg,
                                level='info')

        ret, err = lock.release_lock('gluster_commands')
        if err:
            raise Exception(err)

        ret, err = lock.release_lock('batch_process')
        if err:
            raise Exception(err)

    except Exception, e:
        str = "Error processing batch files : %s" % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)
def main():
    lg = None
    try:
        lg, err = logger.get_script_logger(
            'Gluster batch processing', '/var/log/integralstor/scripts.log', level=logging.DEBUG)

        logger.log_or_print('Batch processing initiated.', lg, level='info')

        active, err = grid_ops.is_active_admin_gridcell()
        if err:
            raise Exception(err)

        if not active:
            logger.log_or_print(
                'Not active admin GRIDCell so exiting.', lg, level='info')
            sys.exit(0)

        batch_files_path, err = config.get_batch_files_path()
        if err:
            raise Exception(err)

        ret, err = lock.get_lock('batch_process')
        if err:
            raise Exception(err)

        if not ret:
            raise Exception('Could not acquire batch lock. Exiting.')

        gluster_lck, err = lock.get_lock('gluster_commands')
        if err:
            raise Exception(err)

        if not gluster_lck:
            raise Exception('Could not acquire gluster lock. Exiting.')

        fl = os.listdir(os.path.normpath(batch_files_path))
        if fl:
            for file in fl:
                if not file.startswith("bp_"):
                    # unknown file type so ignore
                    continue
                else:
                    logger.log_or_print('Processing file %s/%s' %
                                        (batch_files_path, file), lg, level='info')
                    with open(os.path.normpath("%s/%s" % (batch_files_path, file)), "r") as f:
                        # print 'a'
                        # print
                        # os.path.normpath("%s/%s"%(batch_files_path,file))
                        d = json.load(f)
                        # print 'a1'
                        ret, err = process_batch(d, file, logger)
                        if err:
                            str = "Error loading json content for %s/%s : %s" % (
                                batch_files_path, file, err)
                            logger.log_or_print(str, lg, level='error')
                            continue
        else:
            logger.log_or_print(
                'No batch processes pending.', lg, level='info')

        ret, err = lock.release_lock('gluster_commands')
        if err:
            raise Exception(err)

        ret, err = lock.release_lock('batch_process')
        if err:
            raise Exception(err)

    except Exception, e:
        str = "Error processing batch files : %s" % e
        logger.log_or_print(str, lg, level='critical')
        sys.exit(-1)