def populate(self, data_in):
"""Populate data for agent to eventually send to server.
Args:
data_in: dict of datapoint values from agent
timeseries: TimeSeries data if True
Returns:
None
"""
# Initialize data
data = deepcopy(data_in)
# Validate base_type
if len(data) != 1 or isinstance(data, defaultdict) is False:
log_message = ('Agent data "%s" is invalid') % (data)
log.log2die(1005, log_message)
######################################################################
# Get a description to use for label value. You could do a lookup in
# a table based on the spoken language of the environment based on the
# label and assign the translated result to data[label]['description']
######################################################################
for label in data.keys():
data[label]['description'] = label
break
# Add data to appropriate self.data key
if data[label]['base_type'] is not None:
self.data['timeseries'].update(data)
else:
self.data['timefixed'].update(data)
def commit(self, session, error_code):
"""Do a database modification.
Args:
session: Session
error_code: Error number to use if one occurs
Returns:
None
"""
# Do commit
try:
# Commit change
session.commit()
except Exception as exception_error:
session.rollback()
log_message = ('Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
self.close()
def log_directory(self):
"""Determine the log_directory.
Args:
None
Returns:
value: configured log_directory
"""
# Initialize key variables
key = 'main'
sub_key = 'log_directory'
# Process configuration
value = _key_sub_key(key, sub_key, self.config_dict)
# Check if value exists
if os.path.isdir(value) is False:
log_message = ('log_directory: "%s" '
'in configuration doesn\'t exist!') % (value)
log.log2die(1030, log_message)
# Return
return value
def add(self, record, error_code):
"""Add a record to the database.
Args:
record: Record object
error_code: Error number to use if one occurs
Returns:
None
"""
# Initialize key variables
session = self.session()
# Do add
try:
# Commit change
session.add(record)
session.commit()
except Exception as exception_error:
session.rollback()
log_message = ('Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
self.close()
def start(self):
"""Start the daemon.
Args:
None
Returns:
"""
# Check for a pidfile to see if the daemon already runs
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if pid:
log_message = (
'PID file: %s already exists. Daemon already running?'
'') % (self.pidfile)
log.log2die(1062, log_message)
# Start the daemon
self.daemonize()
# Log success
log_message = ('Daemon Started - PID file: %s') % (self.pidfile)
log.log2info(1070, log_message)
# Run code for daemon
self.run()
def stop(self):
"""Stop the daemon.
Args:
None
Returns:
"""
# Get the pid from the pidfile
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if not pid:
log_message = (
'PID file: %s does not exist. Daemon not running?'
'') % (self.pidfile)
log.log2warning(1063, log_message)
# Not an error in a restart
return
# Try killing the daemon process
try:
while 1:
# Sleep a while
time.sleep(0.3)
# Process lockfile state when trying to stop
if self.lockfile is None:
os.kill(pid, signal.SIGTERM)
else:
if os.path.exists(self.lockfile) is True:
continue
else:
os.kill(pid, signal.SIGTERM)
except OSError as err:
error = str(err.args)
if error.find("No such process") > 0:
self.delpid()
self.dellock()
else:
log_message = (str(err.args))
log_message = (
'%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1068, log_message)
except:
log_message = (
'Unknown daemon "stop" error for PID file: %s'
'') % (self.pidfile)
log.log2die(1066, log_message)
# Log success
self.delpid()
self.dellock()
log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
log.log2info(1071, log_message)
def validate(self):
"""Make sure we are using a test database."""
# Only work on test databases
if self.config.db_name().startswith('test_') is False:
log_message = ('Test database not found in configuration. '
'Try setting your "MDL_CONFIGDIR" environment '
'variable to a directory with a test configuration')
log.log2die(1017, log_message)
def daemonize(self):
"""Deamonize class. UNIX double fork mechanism.
Args:
None
Returns:
None
"""
# Create a parent process that will manage the child
# when the code using this class is done.
try:
pid = os.fork()
if pid > 0:
# Exit first parent
sys.exit(0)
except OSError as err:
log_message = ('Daemon fork #1 failed: %s') % (err)
log_message = ('%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1060, log_message)
# Decouple from parent environment
os.chdir('/')
os.setsid()
os.umask(0)
# Do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError as err:
log_message = ('Daemon fork #2 failed: %s') % (err)
log_message = ('%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1061, log_message)
# Redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
f_handle_si = open(os.devnull, 'r')
# f_handle_so = open(os.devnull, 'a+')
f_handle_so = open(os.devnull, 'a+')
f_handle_se = open(os.devnull, 'a+')
os.dup2(f_handle_si.fileno(), sys.stdin.fileno())
# os.dup2(f_handle_so.fileno(), sys.stdout.fileno())
os.dup2(f_handle_so.fileno(), sys.stdout.fileno())
os.dup2(f_handle_se.fileno(), sys.stderr.fileno())
# write pidfile
atexit.register(self.delpid)
pid = str(os.getpid())
with open(self.pidfile, 'w+') as f_handle:
f_handle.write(pid + '\n')
def purge(self):
"""Purge data from cache by posting to central server.
Args:
None
Returns:
success: "True: if successful
"""
# Initialize key variables
id_agent = self.data['id_agent']
# Add files in cache directory to list only if they match the
# cache suffix
all_filenames = [
filename for filename in os.listdir(self.cache_dir)
if os.path.isfile(os.path.join(self.cache_dir, filename))
]
filenames = [
filename for filename in all_filenames
if filename.endswith(self.cache_suffix)
]
# Read cache file in sorted order.
# NOTE: We must post data in timestamp sorted order.
for filename in filenames.sorted():
# Only post files for our own UID value
if id_agent not in filename:
continue
# Get the full filepath for the cache file and post
filepath = os.path.join(self.cache_dir, filename)
with open(filepath, 'r') as f_handle:
try:
data = json.load(f_handle)
except:
# Log removal
log_message = (
'Error reading previously cached agent data file %s '
'for agent %s. May be corrupted.'
'') % (filepath, self.name())
log.log2die(1058, log_message)
# Post file
success = self.post(save=False, data=data)
# Delete file if successful
if success is True:
os.remove(filepath)
# Log removal
log_message = ('Purging cache file %s after successfully '
'contacting server'
'') % (filepath)
log.log2info(1053, log_message)
def add_all(self, data_list, error_code, die=True):
"""Do a database modification.
Args:
data_list: List of sqlalchemy table objects
error_code: Error number to use if one occurs
die: Don't die if False, just return success
Returns:
success: True is successful
"""
# Initialize key variables
success = False
# Open database connection. Prepare cursor
session = self.session()
try:
# Update the database cache
session.add_all(data_list)
# Commit change
session.commit()
# disconnect from server
self.close()
# Update success
success = True
except Exception as exception_error:
success = False
session.rollback()
log_message = ('Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warning(error_code, log_message)
except:
success = False
session.rollback()
log_message = ('Unexpected database exception')
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warning(error_code, log_message)
# Return
return success
def setup(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(URL,
echo=True,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size,
pool_recycle=3600)
# Try to create the database
print_ok('Attempting to create database tables')
try:
sql_string = ('ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (
config.db_name())
engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1036, log_message)
# Apply schemas
print_ok('Applying Schemas')
BASE.metadata.create_all(engine)
def _mkdir(directory):
"""Create a directory if it doesn't already exist.
Args:
directory: Directory name
Returns:
None
"""
# Do work
if os.path.exists(directory) is False:
os.makedirs(directory, mode=0o755)
else:
if os.path.isfile(directory) is True:
log_message = (
'%s is not a directory.'
'') % (directory)
log.log2die(1043, log_message)
def run_script(cli_string, shell=False, die=True):
"""Run the cli_string UNIX CLI command and record output.
Args:
cli_string: Command to run on the CLI
die: Die if command runs with an error
Returns:
None
"""
# Initialize key variables
encoding = locale.getdefaultlocale()[1]
header_returncode = ('[Return Code]')
header_stdout = ('[Output]')
header_stderr = ('[Error Message]')
header_bad_cmd = ('[ERROR: Bad Command]')
log_message = ''
# Create the subprocess object
if shell is False:
do_command_list = list(cli_string.split(' '))
process = subprocess.Popen(do_command_list,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
process = subprocess.Popen(cli_string,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdoutdata, stderrdata = process.communicate()
returncode = process.returncode
# Crash if the return code is not 0
if die is True:
if returncode != 0:
# Print the Return Code header, Return Code, STDOUT header
string2print = ('%s %s %s %s') % (header_bad_cmd, cli_string,
header_returncode, returncode)
log_message = ('%s%s') % (log_message, string2print)
# Print the STDERR
string2print = ('%s') % (header_stderr)
log_message = ('%s %s') % (log_message, string2print)
for line in stderrdata.decode(encoding).split('\n'):
string2print = ('%s') % (line)
log_message = ('%s %s') % (log_message, string2print)
# Print the STDOUT
string2print = ('%s') % (header_stdout)
log_message = ('%s %s') % (log_message, string2print)
for line in stdoutdata.decode(encoding).split('\n'):
string2print = ('%s') % (line)
log_message = ('%s %s') % (log_message, string2print)
# All done
log.log2die(1074, log_message)
# Return
return stdoutdata