Exemplo n.º 1
0
def get_instance_log_conf_file(instance_id):
    """
    Args:
    instance_id is an instance identifier.
    Returns the Logentries-RSyslog configuration file deployed on the instance or None if the configuration does not exist or could not be retrieved. 
    """
    # Retrieve current log config file
    log_conf_file = None
    host_name='%s_%s'%(constants.get_group_name(), instance_id)
    filename = 'logentries_%s.conf'%host_name
    rsyslog_conf_name = '/etc/rsyslog.d/%s'%filename
    local_conf_name = '/tmp/%s'%filename
    
    # Remove local version of the file if already present as it may be obsolete
    try:
        local('rm %s'%local_conf_name, capture=True)
    except:
        logger.debug('No version of the file present locally. host_name=%s, remote_filename=%s, local_filename=%s', host_name, rsyslog_conf_name, local_conf_name)
    # Get remote conf file or return None if it cannot be retrieved
    if not files.exists(rsyslog_conf_name, use_sudo=True):
        return None
    try:
        get(rsyslog_conf_name,local_conf_name)
    except:
        logger.debug('No version of the file present remotely. host_name=%s, remote_filename=%s, local_filename=%s', host_name, rsyslog_conf_name, local_conf_name)
        return None
    # Open conf file or return None if it cannot be opened
    try:
        log_conf_file = open(local_conf_name,'r')
    except:
        logger.error('Cannot open Logentries-Rsyslog configuration file. host_name=%s, local_filename=%s', host_name, local_conf_name)
        return None
    logger.debug('Remote Logentries-Rsyslog configuration file successfully retrieved and opened. filename=%s, hostname=%s', rsyslog_conf_name, host_name)
    return log_conf_file
Exemplo n.º 2
0
def deploy_log_conf(instance_id, log_conf):
    """
    Args:
    instance_id is an not None instance identifier.
    log_conf is a Logentries-RSyslog configuration.
    Deploys Logentries-RSyslog configuration file 'log_conf' and restart RSyslog so that this file is taken into account.
    Returns True if and only if log_conf was successfully deployed and RSyslog was successfully restarted. 
    """
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)
    if log_conf is None:
        logger.warning('No RSyslog Configuration file is provided. hostname=%s', host_name)
        return False

    # Get current instance information
    local_conf_name = log_conf.get_name()

    # Save configuration in a file
    log_conf_file = log_conf.save()

    filename = os.path.basename(log_conf_file.name)
    log_conf_file.close()

    remote_conf_name = '/etc/rsyslog.d/%s'%filename
    
    try:
        put(local_conf_name,remote_conf_name,use_sudo=True)
    except:
        logger.error('File could not be transfer to instance. local_filename=%s, remote_filename=%s, hostname=%s',local_conf_name, remote_conf_name, host_name)
        return False
    logger.debug('Configuration file successfully deployed. local_filename=%s, remote_filename=%s, hostanme=%s', local_conf_name, remote_conf_name, host_name)
    return True
Exemplo n.º 3
0
def remove_log_conf(instance_id):
    """
    Args:
    instance_id is an instance identifier.
    Returns True if and only if the Logentries-RSyslog configuration file is no longer present on the remote the instance.
    """
    host_name='%s_%s'%(constants.get_group_name(), instance_id)
    remote_conf_filename = '/etc/rsyslog.d/logentries_%s.conf'%host_name


    try:
        # Remove logentries rsyslog conf file
        command = 'rm %s'%remote_conf_filename
        output = sudo(command, warn_only=True)
        if output.succeeded:
            logger.debug('File successfully removed. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name)
        else:
            logger.warning('Could not remove file.  remote_filename=%s, hostname=%s.', remote_conf_filename, host_name)
    except:
        logger.error('Could not remove file. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name)

    present = files.exists(remote_conf_filename)
    if not present:
            logger.debug('File is not present on the system. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name)
    return present
Exemplo n.º 4
0
def get_instance_log_paths(instance_id, log_filter):
    """
    Args:
    instance_id is an instance identifier.
    ssh_config is the ssh configuration associated to the instance with id 'instance_id'.
    Returns the list of log paths for the instance with id 'instance_id'.
    Returns an empty list if no log file paths could be retrieved from the instance.
    """
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)
    log_paths = []

    # Retrieve log file paths that match the log filter
    command = "find / -type f -regex '%s'"%(log_filter)
    try:
        output = sudo(command, warn_only=True)
    except:
        logger.warning('Could not retrieve log paths. hostname=%s, log_filter=%s, command=%s', host_name, log_filter, command)
        return []

    if not output.succeeded:
        logger.warning('Could not retrieve log paths. hostname=%s, log_filter=%s, message=%s', host_name, log_filter, output.stdout.replace('\n',' \\ '))
        return log_paths
    # Clean output
    log_paths.extend([logpath.replace('\r','') for logpath in output.stdout.split('\n')])
    logger.info('Log Paths: %s',log_paths)
    return log_paths
Exemplo n.º 5
0
def restart_rsyslog(instance_id):
    """
    Restarts RSyslog service.
    Args:
    instance_id is an instance identifier.
    Returns True if and only if RSyslog was successfully restarted. 
    """
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)
    output = None

    if files.exists('/etc/rsyslog.d/', use_sudo=True):
        command = 'service rsyslog restart'
        try:
            output = sudo(command, warn_only=True)
            logger.warning('Could not restart syslog. hostname=%s, command=\'%s\'', host_name, command)
        except:
            command = '/etc/init.d/rsyslog restart'
            try:
                sudo(command, warn_only=True)
            except:
                logger.error('Rsyslog could not be restarted. hostname=%s, command=\'%s\'', host_name, command)
    else:
        logger.warning('Instance does not support RSyslog. hostname=%s', host_name)
        return False

    if output is None:
        return False
    
    if output.succeeded:
        logger.info('RSyslog restarted successfully. hostname=%s', host_name)
    else:
        logger.error('Error restarting RSyslog. hostname=%s, message=%s', host_name, output.stdout)
    return output.succeeded
Exemplo n.º 6
0
def update_instance_conf(instance_id, log_paths, log_conf):
    """
    Args:
    instance_id is an instance identifier.
    log_paths is a not None list of log file paths.
    log_conf is a not None Logentries-RSyslog configuration object.
    Returns the updated log_conf, taking into account new log files present on the instance as well as modifications made to the corresponding logentries host.
    
    """
    log_client = logclient.Client(constants.ACCOUNT_KEY)
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)

    # Creation of a host if no configuration file exists
    if log_conf is None and len(log_paths)>0:
        host = utils.create_host_and_logs(log_client,instance_id,log_paths)
        log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host)
    # if a configuration file exists for the instance, look for the differences between the log lists
    elif log_conf is not None:
        conf_host = log_conf.get_host()
        if conf_host is None:
            logger.error('This instance configuration is missing the corresponding model!! hostname=%s', host_name)
            host = utils.create_host_and_logs(log_client,instance_id,log_paths)
            log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host)
            return log_conf

        if conf_host.get_key() is None:
            logger.warning('Instance has a logentries-rsyslog config file but no account key!! hostname=%s', host.get_name())
            logger.warning('Instance is re-provisioned. hostname=%s', host.get_name())
            host = utils.create_host_and_logs(log_client,instance_id,log_paths)
            log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host)
            return log_conf
        
        logentries_host = get_logentries_host(log_client,conf_host)
        # If there is no matching host, then it is assumed that it was deleted from Logentries and that no configuration should be associated to this instance.
        if logentries_host is None:
            logger.info('Instance has an logentries-rsyslog config file but no matching host in logentries!! hostname=%s', host_name)
            logger.info('Instance will be deprovisioned hostname=%s', host_name)            
            #host = utils.create_host_and_logs(log_client,instance_id,log_paths)
            #log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host)
            return None

        logentries_logs = logentries_host.get_logs()
        logentries_log_paths = [log.get_filename() for log in logentries_logs]
        logger.info('Logs are already followed on this instance. hostname=%s, log_paths=%s',log_conf.get_host().get_name(), logentries_log_paths)
        new_log_paths = [log_path for log_path in log_paths if log_path not in logentries_log_paths]
        logger.info('New logs detected. hostname=%s, new_log_paths=%s',log_conf.get_host().get_name(), new_log_paths)


        for new_log_name in new_log_paths:
            logentries_host, log_key = log_client.create_log_token(host=logentries_host, log_name=new_log_name)
            logger.info('Log Created. hostname=%s, log_path=%s, key=%s', logentries_host.get_name(), new_log_name, log_key)

        removed_logs = [removed_log for removed_log in logentries_logs if removed_log.get_filename() not in log_paths]
        for removed_log in removed_logs:
            if removed_log is not None and log_client.remove_log(host=logentries_host, log=removed_log):
                logger.info('Log Removed. hostname=%s, log=%s', logentries_host.get_name(), removed_log)
        log_conf.set_host(logentries_host)
    return log_conf
Exemplo n.º 7
0
def get_instance_log_conf(instance_id):
    """
    Args:
    instance_id is an not None instance identifier.
    Returns the Logentries-RSyslog configuration deployed on the instance or None if no such configuration is deployed.
    """
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)
    log_conf_file = get_instance_log_conf_file(instance_id)
    if log_conf_file is None:
        logger.debug('No existing logentries rsyslog configuration file was found. hostname=%s', host_name)

    log_conf = load_conf_file(log_conf_file,instance_id)

    if log_conf is None:
        logger.info('Logentries rsyslog configuration file could not be read. hostname=%s', host_name)

    return log_conf
Exemplo n.º 8
0
def set_instance_host_keys():
    """
    Args:
    Collects host keys associated to instance in fabric host_list.
    """
    instance_id, log_filter = utils.get_log_filter(env.host)
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)
    log_client = logclient.Client(constants.get_account_key())

    global _GROUP_HOST_LIST
    log_conf = get_instance_log_conf(instance_id)
    if log_conf is None:
        return
    conf_host = log_conf.get_host()
    logger.debug('Checking if host should be kept. host=%s', conf_host)
    if conf_host is not None:
        logger.info('Host found in ssh configuration. host=%s', conf_host.to_json())
        _GROUP_HOST_LIST.append(conf_host.get_key())
Exemplo n.º 9
0
def deprovision():
    """
    Deprovisions the instance by removing the logentries rsyslog config file from it, restarting rsyslog and removing the corresponding host from the logentries system.
    """
    instance_id, log_filter = utils.get_log_filter(env.host)
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)

    log_conf_file = get_instance_log_conf_file(instance_id)
    if log_conf_file is None:
        logger.debug('Cannot deprovision instance as it has not been provisioned. hostname=%s', host_name)
        return
    log_conf = load_conf_file(log_conf_file, instance_id)

    if log_conf is None:
        logger.info('No existing logentries rsyslog configuration file was found on instance %s',instance_id)
        return

    if remove_log_conf(instance_id):
        restart_rsyslog(instance_id)

    conf_host = log_conf.get_host()
    if conf_host is None:
        logger.error('Error. This instance configuration is missing the corresponding model!! instance_id=%s',instance_id)
        return
    
    if conf_host.get_key() is None:
        logger.error('Host has a logentries-rsyslog config file but no account key, host=%s!!',host.to_json())
    else:
        log_client = logclient.Client(constants.get_account_key())
        logentries_host = get_logentries_host(log_client,conf_host)

        # If there is no matching host, then it is assumed that it was deleted from Logentries and that no configuration should be associated to this instance.
        if logentries_host is not None:
                succeeded = log_client.remove_host(logentries_host)
                if succeeded:
                    logger.warning('Host removed from Logentries. host=%s', logentries_host.to_json())
                else:
                    logger.error('Could not remove host from Logentries. host=%s', logentries_host.to_json())
        else:
            logger.error('Could not remove host from Logentries. hostname=%s', host_name)
    return
Exemplo n.º 10
0
def sync():
    """
    Syncs the logentries account with each instance and logs defined in the ssh config file.
    """
    # Get current instance information
    instance_id, log_filter = utils.get_log_filter(env.host)
    host_name = '%s_%s'%(constants.get_group_name(), instance_id)

    log_paths = get_instance_log_paths(instance_id, log_filter)
    if not files.exists('/etc/rsyslog.d/', use_sudo=True):
        logger.info('Instance does not support rsyslog. hostname=%s', host_name)
        return
    log_conf = get_instance_log_conf(instance_id)
    log_conf = update_instance_conf(instance_id, log_paths, log_conf)
    if log_conf is None:
        logger.info('No new rsyslog configuration was detected. hostname=%s', host_name)
        return

    deploy_log_conf(instance_id, log_conf)
    # Restart RSyslog
    restart_rsyslog(instance_id)  
    return