def get_instance_log_conf_file(instance_id): """ Args: instance_id is an instance identifier. Returns the Logentries-RSyslog configuration file deployed on the instance or None if the configuration does not exist or could not be retrieved. """ # Retrieve current log config file log_conf_file = None host_name='%s_%s'%(constants.get_group_name(), instance_id) filename = 'logentries_%s.conf'%host_name rsyslog_conf_name = '/etc/rsyslog.d/%s'%filename local_conf_name = '/tmp/%s'%filename # Remove local version of the file if already present as it may be obsolete try: local('rm %s'%local_conf_name, capture=True) except: logger.debug('No version of the file present locally. host_name=%s, remote_filename=%s, local_filename=%s', host_name, rsyslog_conf_name, local_conf_name) # Get remote conf file or return None if it cannot be retrieved if not files.exists(rsyslog_conf_name, use_sudo=True): return None try: get(rsyslog_conf_name,local_conf_name) except: logger.debug('No version of the file present remotely. host_name=%s, remote_filename=%s, local_filename=%s', host_name, rsyslog_conf_name, local_conf_name) return None # Open conf file or return None if it cannot be opened try: log_conf_file = open(local_conf_name,'r') except: logger.error('Cannot open Logentries-Rsyslog configuration file. host_name=%s, local_filename=%s', host_name, local_conf_name) return None logger.debug('Remote Logentries-Rsyslog configuration file successfully retrieved and opened. filename=%s, hostname=%s', rsyslog_conf_name, host_name) return log_conf_file
def deploy_log_conf(instance_id, log_conf): """ Args: instance_id is an not None instance identifier. log_conf is a Logentries-RSyslog configuration. Deploys Logentries-RSyslog configuration file 'log_conf' and restart RSyslog so that this file is taken into account. Returns True if and only if log_conf was successfully deployed and RSyslog was successfully restarted. """ host_name = '%s_%s'%(constants.get_group_name(), instance_id) if log_conf is None: logger.warning('No RSyslog Configuration file is provided. hostname=%s', host_name) return False # Get current instance information local_conf_name = log_conf.get_name() # Save configuration in a file log_conf_file = log_conf.save() filename = os.path.basename(log_conf_file.name) log_conf_file.close() remote_conf_name = '/etc/rsyslog.d/%s'%filename try: put(local_conf_name,remote_conf_name,use_sudo=True) except: logger.error('File could not be transfer to instance. local_filename=%s, remote_filename=%s, hostname=%s',local_conf_name, remote_conf_name, host_name) return False logger.debug('Configuration file successfully deployed. local_filename=%s, remote_filename=%s, hostanme=%s', local_conf_name, remote_conf_name, host_name) return True
def remove_log_conf(instance_id): """ Args: instance_id is an instance identifier. Returns True if and only if the Logentries-RSyslog configuration file is no longer present on the remote the instance. """ host_name='%s_%s'%(constants.get_group_name(), instance_id) remote_conf_filename = '/etc/rsyslog.d/logentries_%s.conf'%host_name try: # Remove logentries rsyslog conf file command = 'rm %s'%remote_conf_filename output = sudo(command, warn_only=True) if output.succeeded: logger.debug('File successfully removed. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name) else: logger.warning('Could not remove file. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name) except: logger.error('Could not remove file. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name) present = files.exists(remote_conf_filename) if not present: logger.debug('File is not present on the system. remote_filename=%s, hostname=%s.', remote_conf_filename, host_name) return present
def get_instance_log_paths(instance_id, log_filter): """ Args: instance_id is an instance identifier. ssh_config is the ssh configuration associated to the instance with id 'instance_id'. Returns the list of log paths for the instance with id 'instance_id'. Returns an empty list if no log file paths could be retrieved from the instance. """ host_name = '%s_%s'%(constants.get_group_name(), instance_id) log_paths = [] # Retrieve log file paths that match the log filter command = "find / -type f -regex '%s'"%(log_filter) try: output = sudo(command, warn_only=True) except: logger.warning('Could not retrieve log paths. hostname=%s, log_filter=%s, command=%s', host_name, log_filter, command) return [] if not output.succeeded: logger.warning('Could not retrieve log paths. hostname=%s, log_filter=%s, message=%s', host_name, log_filter, output.stdout.replace('\n',' \\ ')) return log_paths # Clean output log_paths.extend([logpath.replace('\r','') for logpath in output.stdout.split('\n')]) logger.info('Log Paths: %s',log_paths) return log_paths
def restart_rsyslog(instance_id): """ Restarts RSyslog service. Args: instance_id is an instance identifier. Returns True if and only if RSyslog was successfully restarted. """ host_name = '%s_%s'%(constants.get_group_name(), instance_id) output = None if files.exists('/etc/rsyslog.d/', use_sudo=True): command = 'service rsyslog restart' try: output = sudo(command, warn_only=True) logger.warning('Could not restart syslog. hostname=%s, command=\'%s\'', host_name, command) except: command = '/etc/init.d/rsyslog restart' try: sudo(command, warn_only=True) except: logger.error('Rsyslog could not be restarted. hostname=%s, command=\'%s\'', host_name, command) else: logger.warning('Instance does not support RSyslog. hostname=%s', host_name) return False if output is None: return False if output.succeeded: logger.info('RSyslog restarted successfully. hostname=%s', host_name) else: logger.error('Error restarting RSyslog. hostname=%s, message=%s', host_name, output.stdout) return output.succeeded
def update_instance_conf(instance_id, log_paths, log_conf): """ Args: instance_id is an instance identifier. log_paths is a not None list of log file paths. log_conf is a not None Logentries-RSyslog configuration object. Returns the updated log_conf, taking into account new log files present on the instance as well as modifications made to the corresponding logentries host. """ log_client = logclient.Client(constants.ACCOUNT_KEY) host_name = '%s_%s'%(constants.get_group_name(), instance_id) # Creation of a host if no configuration file exists if log_conf is None and len(log_paths)>0: host = utils.create_host_and_logs(log_client,instance_id,log_paths) log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host) # if a configuration file exists for the instance, look for the differences between the log lists elif log_conf is not None: conf_host = log_conf.get_host() if conf_host is None: logger.error('This instance configuration is missing the corresponding model!! hostname=%s', host_name) host = utils.create_host_and_logs(log_client,instance_id,log_paths) log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host) return log_conf if conf_host.get_key() is None: logger.warning('Instance has a logentries-rsyslog config file but no account key!! hostname=%s', host.get_name()) logger.warning('Instance is re-provisioned. hostname=%s', host.get_name()) host = utils.create_host_and_logs(log_client,instance_id,log_paths) log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host) return log_conf logentries_host = get_logentries_host(log_client,conf_host) # If there is no matching host, then it is assumed that it was deleted from Logentries and that no configuration should be associated to this instance. if logentries_host is None: logger.info('Instance has an logentries-rsyslog config file but no matching host in logentries!! hostname=%s', host_name) logger.info('Instance will be deprovisioned hostname=%s', host_name) #host = utils.create_host_and_logs(log_client,instance_id,log_paths) #log_conf = configfile.LoggingConfFile(name='logentries_%s.conf'%host.get_name(),host=host) return None logentries_logs = logentries_host.get_logs() logentries_log_paths = [log.get_filename() for log in logentries_logs] logger.info('Logs are already followed on this instance. hostname=%s, log_paths=%s',log_conf.get_host().get_name(), logentries_log_paths) new_log_paths = [log_path for log_path in log_paths if log_path not in logentries_log_paths] logger.info('New logs detected. hostname=%s, new_log_paths=%s',log_conf.get_host().get_name(), new_log_paths) for new_log_name in new_log_paths: logentries_host, log_key = log_client.create_log_token(host=logentries_host, log_name=new_log_name) logger.info('Log Created. hostname=%s, log_path=%s, key=%s', logentries_host.get_name(), new_log_name, log_key) removed_logs = [removed_log for removed_log in logentries_logs if removed_log.get_filename() not in log_paths] for removed_log in removed_logs: if removed_log is not None and log_client.remove_log(host=logentries_host, log=removed_log): logger.info('Log Removed. hostname=%s, log=%s', logentries_host.get_name(), removed_log) log_conf.set_host(logentries_host) return log_conf
def get_instance_log_conf(instance_id): """ Args: instance_id is an not None instance identifier. Returns the Logentries-RSyslog configuration deployed on the instance or None if no such configuration is deployed. """ host_name = '%s_%s'%(constants.get_group_name(), instance_id) log_conf_file = get_instance_log_conf_file(instance_id) if log_conf_file is None: logger.debug('No existing logentries rsyslog configuration file was found. hostname=%s', host_name) log_conf = load_conf_file(log_conf_file,instance_id) if log_conf is None: logger.info('Logentries rsyslog configuration file could not be read. hostname=%s', host_name) return log_conf
def set_instance_host_keys(): """ Args: Collects host keys associated to instance in fabric host_list. """ instance_id, log_filter = utils.get_log_filter(env.host) host_name = '%s_%s'%(constants.get_group_name(), instance_id) log_client = logclient.Client(constants.get_account_key()) global _GROUP_HOST_LIST log_conf = get_instance_log_conf(instance_id) if log_conf is None: return conf_host = log_conf.get_host() logger.debug('Checking if host should be kept. host=%s', conf_host) if conf_host is not None: logger.info('Host found in ssh configuration. host=%s', conf_host.to_json()) _GROUP_HOST_LIST.append(conf_host.get_key())
def deprovision(): """ Deprovisions the instance by removing the logentries rsyslog config file from it, restarting rsyslog and removing the corresponding host from the logentries system. """ instance_id, log_filter = utils.get_log_filter(env.host) host_name = '%s_%s'%(constants.get_group_name(), instance_id) log_conf_file = get_instance_log_conf_file(instance_id) if log_conf_file is None: logger.debug('Cannot deprovision instance as it has not been provisioned. hostname=%s', host_name) return log_conf = load_conf_file(log_conf_file, instance_id) if log_conf is None: logger.info('No existing logentries rsyslog configuration file was found on instance %s',instance_id) return if remove_log_conf(instance_id): restart_rsyslog(instance_id) conf_host = log_conf.get_host() if conf_host is None: logger.error('Error. This instance configuration is missing the corresponding model!! instance_id=%s',instance_id) return if conf_host.get_key() is None: logger.error('Host has a logentries-rsyslog config file but no account key, host=%s!!',host.to_json()) else: log_client = logclient.Client(constants.get_account_key()) logentries_host = get_logentries_host(log_client,conf_host) # If there is no matching host, then it is assumed that it was deleted from Logentries and that no configuration should be associated to this instance. if logentries_host is not None: succeeded = log_client.remove_host(logentries_host) if succeeded: logger.warning('Host removed from Logentries. host=%s', logentries_host.to_json()) else: logger.error('Could not remove host from Logentries. host=%s', logentries_host.to_json()) else: logger.error('Could not remove host from Logentries. hostname=%s', host_name) return
def sync(): """ Syncs the logentries account with each instance and logs defined in the ssh config file. """ # Get current instance information instance_id, log_filter = utils.get_log_filter(env.host) host_name = '%s_%s'%(constants.get_group_name(), instance_id) log_paths = get_instance_log_paths(instance_id, log_filter) if not files.exists('/etc/rsyslog.d/', use_sudo=True): logger.info('Instance does not support rsyslog. hostname=%s', host_name) return log_conf = get_instance_log_conf(instance_id) log_conf = update_instance_conf(instance_id, log_paths, log_conf) if log_conf is None: logger.info('No new rsyslog configuration was detected. hostname=%s', host_name) return deploy_log_conf(instance_id, log_conf) # Restart RSyslog restart_rsyslog(instance_id) return