def add_all(self, data_list, error_code):
"""Do a database modification.
Args:
data_list: List of sqlalchemy table objects
error_code: Error number to use if one occurs
Returns:
None
"""
# Open database connection. Prepare cursor
session = self.session()
try:
# Update the database cache
session.add_all(data_list)
# Commit change
session.commit()
except Exception as exception_error:
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
session.close()
def __init__(self, idx_agent):
"""Function for intializing the class.
Args:
idx_agent: Agent idx
Returns:
None
"""
# Initialize important variables
self.data_dict = defaultdict(dict)
# Get the result
database = db.Database()
session = database.session()
result = session.query(Agent).filter(
Agent.idx == idx_agent)
# Massage data
if result.count() == 1:
for instance in result:
self.data_dict['uid'] = jm_general.decode(instance.id)
self.data_dict['name'] = jm_general.decode(instance.name)
self.data_dict['enabled'] = instance.enabled
self.data_dict['last_timestamp'] = instance.last_timestamp
break
else:
log_message = ('Agent IDX %s not found.') % (idx_agent)
log.log2die(1035, log_message)
# Return the session to the database pool after processing
session.close()
def add(self, record, error_code):
"""Add a record to the database.
Args:
record: Record object
error_code: Error number to use if one occurs
Returns:
None
"""
# Initialize key variables
session = self.session()
# Do add
try:
# Commit change
session.add(record)
session.commit()
except Exception as exception_error:
session.rollback()
log_message = ('Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
self.close()
def populate(self, data_in):
"""Populate data for agent to eventually send to server.
Args:
data_in: dict of datapoint values from agent
chartable: Chartable data if True
Returns:
None
"""
# Initialize data
data = deepcopy(data_in)
# Validate base_type
if len(data) != 1 or isinstance(data, defaultdict) is False:
log_message = ('Agent data "%s" is invalid') % (data)
log.log2die(1025, log_message)
# Get a description to use for label value
for label in data.keys():
description = self.lang.label_description(label)
data[label]["description"] = description
break
# Add data to appropriate self.data key
if data[label]["base_type"] is not None:
self.data["chartable"].update(data)
else:
self.data["other"].update(data)
def __init__(self, code):
"""Function for intializing the class.
Args:
code: Department code
Returns:
None
"""
# Initialize important variables
self.data_dict = defaultdict(dict)
value = code.encode()
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(Department).filter(Department.code == value)
# Return the session to the database pool after processing
database.close()
# Massage data
if result.count() == 1:
for instance in result:
self.data_dict['idx'] = instance.idx
self.data_dict[
'code'] = jm_general.decode(instance.code)
self.data_dict[
'name'] = jm_general.decode(instance.name)
break
else:
log_message = ('Department %s not found.') % (code)
log.log2die(1096, log_message)
def data_directory(self):
"""Determine the data_directory.
Args:
None
Returns:
value: configured data_directory
"""
# Get parameter
key = 'server'
sub_key = 'data_directory'
# Get result
value = _key_sub_key(key, sub_key, self.config_dict)
# Determine whether path exists
if os.path.isdir(value) is False:
log_message = (
'data_directory: "%s" '
'in configuration doesn\'t exist!') % (value)
log.log2die(1007, log_message)
# Return
return value
def ingest_cache_directory(self):
"""Determine the ingest_cache_directory.
Args:
None
Returns:
value: configured ingest_cache_directory
"""
# Initialize key variables
key = 'server'
sub_key = 'ingest_cache_directory'
# Process configuration
value = _key_sub_key(key, sub_key, self.config_dict)
# Check if value exists
if os.path.isdir(value) is False:
log_message = (
'ingest_cache_directory: "%s" '
'in configuration doesn\'t exist!') % (value)
log.log2die(1030, log_message)
# Return
return value
def agent_cache_directory(self):
"""Determine the agent_cache_directory.
Args:
None
Returns:
value: configured agent_cache_directory
"""
# Initialize key variables
key = 'agents_common'
sub_key = 'agent_cache_directory'
# Get result
value = _key_sub_key(key, sub_key, self.config_dict)
# Check if value exists
if os.path.isdir(value) is False:
log_message = (
'agent_cache_directory: "%s" '
'in configuration doesn\'t exist!') % (value)
log.log2die(1031, log_message)
# Return
return value
def start(self):
"""Start the daemon.
Args:
None
Returns:
"""
# Check for a pidfile to see if the daemon already runs
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if pid:
log_message = (
'PID file: %s already exists. Daemon already running?'
'') % (self.pidfile)
log.log2die(1062, log_message)
# Start the daemon
self.daemonize()
# Log success
log_message = ('Daemon Started - PID file: %s') % (self.pidfile)
log.log2info(1070, log_message)
# Run code for daemon
self.run()
def __init__(self, idx_host, idx_agent):
"""Method initializing the class.
Args:
idx_host: Host idx
idx_agent: Agent idx
Returns:
None
"""
# Initialize key variables
self.data_dict = defaultdict(dict)
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(HostAgent).filter(and_(
HostAgent.idx_host == idx_host,
HostAgent.idx_agent == idx_agent))
# Return the session to the database pool after processing
database.close()
# Massage data
if result.count() == 1:
for instance in result:
self.data_dict['last_timestamp'] = instance.last_timestamp
break
else:
log_message = (
'Host IDX %s Agent IDX %s not found in iset_hostagent table.'
'') % (idx_host, idx_agent)
log.log2die(1105, log_message)
def move_files(source_dir, target_dir):
"""Delete files in a directory.
Args:
source_dir: Directory where files are currently
target_dir: Directory where files need to be
Returns:
Nothing
"""
# Make sure source directory exists
if os.path.exists(source_dir) is False:
log_message = ('Directory %s does not exist.') % (
source_dir)
log.log2die(1011, log_message)
# Make sure target directory exists
if os.path.exists(target_dir) is False:
log_message = ('Directory %s does not exist.') % (
target_dir)
log.log2die(1012, log_message)
source_files = os.listdir(source_dir)
for filename in source_files:
full_path = ('%s/%s') % (source_dir, filename)
shutil.move(full_path, target_dir)
def __init__(self, idx_agent):
"""Function for intializing the class.
Args:
idx: idx of agent
Returns:
None
"""
# Initialize important variables
self.data_point_dict = defaultdict(dict)
# Get the result
database = db.Database()
session = database.session()
result = session.query(Datapoint).filter(
Datapoint.idx_agent == idx_agent)
# Massage data
if result.count() > 0:
for instance in result:
agent_label = jm_general.decode(instance.agent_label)
idx = instance.idx
uncharted_value = jm_general.decode(instance.uncharted_value)
self.data_point_dict[agent_label] = (idx, uncharted_value)
else:
log_message = ('Agent idx %s not found.') % (idx_agent)
log.log2die(1050, log_message)
# Return the session to the database pool after processing
session.close()
def _agent_config(agent_name, config_dict):
"""Get agent config parameter from YAML.
Args:
agent_name: Agent Name
config_dict: Dictionary to explore
die: Die if true and the result encountered is None
Returns:
result: result
"""
# Get result
key = 'agents'
result = None
# Get new result
if key in config_dict:
configurations = config_dict[key]
for configuration in configurations:
if 'agent_name' in configuration:
if configuration['agent_name'] == agent_name:
result = configuration
break
# Error if not configured
if result is None:
log_message = (
'Agent %s not defined in configuration in '
'agents:%s section') % (key, key)
log.log2die(1094, log_message)
# Return
return result
def add(self, record, error_code):
"""Add a record to the database.
Args:
record: Record object
error_code: Error number to use if one occurs
Returns:
None
"""
# Initialize key variables
session = self.session()
# Do add
try:
# Commit change
session.add(record)
session.commit()
except Exception as exception_error:
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
self.close()
def __init__(self, uid):
"""Function for intializing the class.
Args:
uid: UID of agent
Returns:
None
"""
# Initialize important variables
self.data_dict = defaultdict(dict)
value = uid.encode()
self.uid = value
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(Agent).filter(Agent.id == value)
# Massage data
if result.count() == 1:
for instance in result:
self.data_dict['idx'] = instance.idx
self.data_dict['name'] = jm_general.decode(instance.name)
self.data_dict['enabled'] = instance.enabled
self.data_dict['last_timestamp'] = instance.last_timestamp
break
else:
log_message = ('uid %s not found.') % (value)
log.log2die(1035, log_message)
# Return the session to the database pool after processing
session.close()
def commit(self, session, error_code):
"""Do a database modification.
Args:
session: Session
error_code: Error number to use if one occurs
Returns:
None
"""
# Do commit
try:
# Commit change
session.commit()
except Exception as exception_error:
session.rollback()
log_message = ('Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
self.close()
def commit(self, session, error_code):
"""Do a database modification.
Args:
session: Session
error_code: Error number to use if one occurs
Returns:
None
"""
# Do commit
try:
# Commit change
session.commit()
except Exception as exception_error:
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
log.log2die(error_code, log_message)
except:
session.rollback()
log_message = ('Unexpected database exception')
log.log2die(error_code, log_message)
# disconnect from server
self.close()
def populate(self, data_in):
"""Populate data for agent to eventually send to server.
Args:
data_in: dict of datapoint values from agent
timeseries: TimeSeries data if True
Returns:
None
"""
# Initialize data
data = deepcopy(data_in)
# Validate base_type
if len(data) != 1 or isinstance(data, defaultdict) is False:
log_message = ('Agent data "%s" is invalid') % (data)
log.log2die(1025, log_message)
######################################################################
# Get a description to use for label value. You could do a lookup in
# a table based on the spoken language of the environment based on the
# label and assign the translated result to data[label]['description']
######################################################################
for label in data.keys():
data[label]['description'] = label
break
# Add data to appropriate self.data key
if data[label]['base_type'] is not None:
self.data['timeseries'].update(data)
else:
self.data['timefixed'].update(data)
def main():
"""Process agent data.
Args:
None
Returns:
None
"""
# Get configuration
config = jm_configuration.Config()
# Run server setup if required
if config.server() is True:
server_setup()
# Install required PIP packages
print('Installing required pip3 packages')
pip3 = infoset.utils.jm_general.search_file('pip3')
if pip3 is None:
log_message = ('Cannot find python "pip3". Please install.')
log.log2die(1052, log_message)
utils_directory = infoset.utils.__path__[0]
requirements_file = ('%s/requirements.txt') % (
Path(utils_directory).parents[1])
script_name = (
'pip3 install --user --upgrade --requirement %s'
'') % (requirements_file)
infoset.utils.jm_general.run_script(script_name)
def __init__(self, config_key):
"""Function for intializing the class.
Args:
config_key: Configuration config_key
Returns:
None
"""
# Initialize important variables
self.data_dict = defaultdict(dict)
value = config_key.encode()
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(Configuration).filter(Configuration.config_key == value)
# Massage data
if result.count() == 1:
for instance in result:
self.data_dict['idx'] = instance.idx
self.data_dict[
'config_key'] = jm_general.decode_key(instance.config_key)
self.data_dict[
'name'] = jm_general.decode_key(instance.name)
break
else:
log_message = ('Configuration %s not found.') % (config_key)
log.log2die(1048, log_message)
# Return the session to the database pool after processing
session.close()
def contactable(self):
"""Check if device is contactable.
Args:
device_id: Device ID
Returns:
contactable: True if a contactable
"""
# Define key variables
contactable = False
result = None
# Try to reach device
try:
# If we can poll the SNMP sysObjectID,
# then the device is contactable
result = self.sysobjectid(connectivity_check=True)
if bool(result) is True:
contactable = True
except Exception as _:
# Not contactable
contactable = False
except:
# Log a message
log_message = ('Unexpected SNMP error for device %s') % (
self.snmp_params['snmp_hostname'])
log.log2die(1008, log_message)
# Return
return contactable
def start(self):
"""Start the daemon.
Args:
None
Returns:
"""
# Check for a pidfile to see if the daemon already runs
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if pid:
log_message = (
'PID file: %s already exists. Daemon already running?'
'') % (self.pidfile)
log.log2die(1062, log_message)
# Start the daemon
self.daemonize()
# Log success
log_message = ('Daemon Started - PID file: %s') % (self.pidfile)
log.log2quiet(1070, log_message)
# Run code for daemon
self.run()
def __init__(self, idx):
"""Function for intializing the class.
Args:
idx: BillType Index
Returns:
None
"""
# Initialize important variables
self.data_dict = defaultdict(dict)
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(BillType).filter(BillType.idx == idx)
# Return the session to the database pool after processing
database.close()
# Massage data
if result.count() == 1:
for instance in result:
self.data_dict['idx'] = instance.idx
self.data_dict[
'code'] = jm_general.decode(instance.code)
self.data_dict[
'name'] = jm_general.decode(instance.name)
break
else:
log_message = ('BillType idx %s not found.') % (idx)
log.log2die(1099, log_message)
def ingest_cache_directory(self):
"""Determine the ingest_cache_directory.
Args:
None
Returns:
value: configured ingest_cache_directory
"""
# Initialize key variables
key = 'main'
sub_key = 'ingest_cache_directory'
# Process configuration
value = _key_sub_key(key, sub_key, self.config_dict)
# Check if value exists
if os.path.isdir(value) is False:
log_message = ('ingest_cache_directory: "%s" '
'in configuration doesn\'t exist!') % (value)
log.log2die(1011, log_message)
# Return
return value
def do_test(cli_args, config):
"""Process 'test' CLI option.
Args:
connectivity_check: Set if testing for connectivity
Returns:
None
"""
# Show host information
validate = snmp_manager.Validate(cli_args.host, config.snmp_auth())
snmp_params = validate.credentials()
snmp_object = snmp_manager.Interact(snmp_params)
if bool(snmp_params) is True:
print('\nValid credentials found:\n')
print(yaml.dump(snmp_params, default_flow_style=False))
print('\n')
# Get SNMP data and print
status = snmp_info.Query(snmp_object)
yaml_data = status.everything()
print(yaml_data)
else:
# Error, host problems
log_message = (
'Uncontactable host %s or no valid SNMP '
'credentials found for it.') % (cli_args.host)
log.log2die(1039, log_message)
def _key_sub_key(key, sub_key, config_dict, die=True):
"""Get config parameter from YAML.
Args:
key: Primary key
sub_key: Secondary key
config_dict: Dictionary to explore
die: Die if true and the result encountered is None
Returns:
result: result
"""
# Get result
result = None
# Get new result
if key in config_dict:
if sub_key in config_dict[key]:
result = config_dict[key][sub_key]
# Error if not configured
if result is None and die is True:
log_message = (
'%s:%s not defined in configuration') % (key, sub_key)
log.log2die(1016, log_message)
# Return
return result
def stop(self):
"""Stop the daemon.
Args:
None
Returns:
"""
# Get the pid from the pidfile
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if not pid:
log_message = (
'PID file: %s does not exist. Daemon not running?'
'') % (self.pidfile)
log.log2warning(1063, log_message)
# Not an error in a restart
return
# Try killing the daemon process
try:
while 1:
# Sleep a while
time.sleep(0.3)
# Process lockfile state when trying to stop
if self.lockfile is None:
os.kill(pid, signal.SIGTERM)
else:
if os.path.exists(self.lockfile) is True:
continue
else:
os.kill(pid, signal.SIGTERM)
except OSError as err:
error = str(err.args)
if error.find("No such process") > 0:
self.delpid()
self.dellock()
else:
log_message = (str(err.args))
log_message = (
'%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1068, log_message)
except:
log_message = (
'Unknown daemon "stop" error for PID file: %s'
'') % (self.pidfile)
log.log2die(1066, log_message)
# Log success
self.delpid()
self.dellock()
log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
log.log2info(1071, log_message)
def validate(self):
"""Make sure we are using a test database."""
# Only work on test databases
if self.config.db_name().startswith('test_') is False:
log_message = ('Test database not found in configuration. '
'Try setting your "INFOSET_CONFIGDIR" environment '
'variable to a directory with a test configuration')
log.log2die(1017, log_message)
def run(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
mappings = [Agent, Department, Device, Billcode, DeviceAgent, Datapoint, AgentName]
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(
URL, echo=False,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size, pool_recycle=3600)
# Try to create the database
shared.print_ok('Attempting to create database tables')
try:
sql_string = (
'ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (config.db_name())
engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1046, log_message)
# Apply schemas
shared.print_ok('Generating Schemas.')
with open('infoset.sql', 'w') as infoset_mysql:
for mapping in mappings:
print(CreateTable(mapping.__table__))
infoset_mysql.write(str(CreateTable(mapping.__table__)))
infoset_mysql.close()
# Insert database entries
self._insert_agent_device()
self._insert_billcode()
self._insert_department()
self._insert_datapoint()
self._insert_config()
def daemonize(self):
"""Deamonize class. UNIX double fork mechanism.
Args:
None
Returns:
None
"""
# Create a parent process that will manage the child
# when the code using this class is done.
try:
pid = os.fork()
if pid > 0:
# Exit first parent
sys.exit(0)
except OSError as err:
log_message = ('Daemon fork #1 failed: %s') % (err)
log_message = ('%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1060, log_message)
# Decouple from parent environment
os.chdir('/')
os.setsid()
os.umask(0)
# Do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError as err:
log_message = ('Daemon fork #2 failed: %s') % (err)
log_message = ('%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1061, log_message)
# Redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
f_handle_si = open(os.devnull, 'r')
# f_handle_so = open(os.devnull, 'a+')
f_handle_so = open(os.devnull, 'a+')
f_handle_se = open(os.devnull, 'a+')
os.dup2(f_handle_si.fileno(), sys.stdin.fileno())
# os.dup2(f_handle_so.fileno(), sys.stdout.fileno())
os.dup2(f_handle_so.fileno(), sys.stdout.fileno())
os.dup2(f_handle_se.fileno(), sys.stderr.fileno())
# write pidfile
atexit.register(self.delpid)
pid = str(os.getpid())
with open(self.pidfile, 'w+') as f_handle:
f_handle.write(pid + '\n')
def daemonize(self):
"""Deamonize class. UNIX double fork mechanism.
Args:
None
Returns:
None
"""
# Create a parent process that will manage the child
# when the code using this class is done.
try:
pid = os.fork()
if pid > 0:
# Exit first parent
sys.exit(0)
except OSError as err:
log_message = ('Daemon fork #1 failed: %s') % (err)
log_message = ('%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1060, log_message)
# Decouple from parent environment
os.chdir('/')
os.setsid()
os.umask(0)
# Do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError as err:
log_message = ('Daemon fork #2 failed: %s') % (err)
log_message = ('%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1061, log_message)
# Redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
f_handle_si = open(os.devnull, 'r')
# f_handle_so = open(os.devnull, 'a+')
f_handle_so = open(os.devnull, 'a+')
f_handle_se = open(os.devnull, 'a+')
os.dup2(f_handle_si.fileno(), sys.stdin.fileno())
# os.dup2(f_handle_so.fileno(), sys.stdout.fileno())
os.dup2(f_handle_so.fileno(), sys.stdout.fileno())
os.dup2(f_handle_se.fileno(), sys.stderr.fileno())
# write pidfile
atexit.register(self.delpid)
pid = str(os.getpid())
with open(self.pidfile, 'w+') as f_handle:
f_handle.write(pid + '\n')
def stop(self):
"""Stop the daemon.
Args:
None
Returns:
"""
# Get the pid from the pidfile
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if not pid:
log_message = (
'PID file: %s does not exist. Daemon not running?'
'') % (self.pidfile)
log.log2warn(1063, log_message)
# Not an error in a restart
return
# Try killing the daemon process
try:
while 1:
if self.lockfile is None:
os.kill(pid, signal.SIGTERM)
else:
time.sleep(0.3)
if os.path.exists(self.lockfile) is True:
continue
else:
os.kill(pid, signal.SIGTERM)
time.sleep(0.3)
except OSError as err:
error = str(err.args)
if error.find("No such process") > 0:
self.delpid()
self.dellock()
else:
log_message = (str(err.args))
log_message = (
'%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1068, log_message)
except:
log_message = (
'Unknown daemon "stop" error for PID file: %s'
'') % (self.pidfile)
log.log2die(1066, log_message)
# Log success
self.delpid()
self.dellock()
log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
log.log2quiet(1071, log_message)
def purge(self):
"""Purge data from cache by posting to central server.
Args:
None
Returns:
success: "True: if successful
"""
# Initialize key variables
id_agent = self.data['id_agent']
# Add files in cache directory to list only if they match the
# cache suffix
all_filenames = [
filename for filename in os.listdir(self.cache_dir)
if os.path.isfile(os.path.join(self.cache_dir, filename))
]
filenames = [
filename for filename in all_filenames
if filename.endswith(self.cache_suffix)
]
# Read cache file in sorted order.
# NOTE: We must post data in timestamp sorted order.
for filename in filenames.sorted():
# Only post files for our own UID value
if id_agent not in filename:
continue
# Get the full filepath for the cache file and post
filepath = os.path.join(self.cache_dir, filename)
with open(filepath, 'r') as f_handle:
try:
data = json.load(f_handle)
except:
# Log removal
log_message = (
'Error reading previously cached agent data file %s '
'for agent %s. May be corrupted.'
'') % (filepath, self.name())
log.log2die(1058, log_message)
# Post file
success = self.post(save=False, data=data)
# Delete file if successful
if success is True:
os.remove(filepath)
# Log removal
log_message = ('Purging cache file %s after successfully '
'contacting server'
'') % (filepath)
log.log2info(1110, log_message)
def run(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Needed to run install subcommand
# https://stackoverflow.com/questions/1321270/how-to-extend-distutils-with-a-simple-post-install-script/1321345#1321345
BuildCommand.run(self)
self.reserved = '_SYSTEM_RESERVED_'
self.config = configuration.Config()
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
URL = ('sqlite:///%s') % (config.db_file())
engine = create_engine(URL, echo=True, encoding='utf8')
# Try to create the database
shared.print_ok('Attempting to create database tables')
try:
sql_string = ('ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (
config.db_name())
# engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1046, log_message)
# Apply schemas
shared.print_ok('Applying Schemas.')
BASE.metadata.create_all(engine)
# Insert database entries
self._insert_agent_device()
self._insert_billcode()
self._insert_department()
self._insert_datapoint()
self._insert_config()
def server_setup():
"""Setup server.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
# Get configuration
config = jm_configuration.Config()
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(
DBURL, echo=True,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size, pool_recycle=3600)
# Try to create the database
print('Attempting to create database tables')
try:
sql_string = (
'ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (config.db_name())
engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1036, log_message)
# Apply schemas
print('Applying Schemas')
BASE.metadata.create_all(engine)
# Insert database entries
insert_agent_host()
insert_billtype()
insert_department()
insert_config()
# Try some additional statements
metadata.insert_oids()
def _process_error(
connectivity_check=False, session_error_status=None,
session_error_index=None, get=False,
log_message=None):
"""Process errors received.
Args:
connectivity_check:
Set if testing for connectivity. Some session
errors are ignored so that a null result is returned
session_error_status: Error status
session_error_index: Error index
get: True if formatting the results of an SNMP get
Returns:
results:
"""
# Initialize key variables
results = {}
# Timeout contacting device
# (Timeout as OID requested does not exist, not because the
# device is uncontactable)
# The device must be checked for connectivity before hand. If it
# can connect but some additional OID is unavailable, then this is
# invoked. This is used to determine whether a device has 64 bit
# IFMIB octet counters
if connectivity_check is False:
if (session_error_status == 0) and (
session_error_index == -24):
# Return blank results
return results
if connectivity_check is True:
# Bad SNMP authentication during authentication check
if (session_error_status == 0) and (
session_error_index == -4):
# Return blank results
return results
# Device completely off the air (SNMP timeout)
if (session_error_status == 0) and (
session_error_index == 0):
# Return blank results
return results
# Otherwise Fail
if get is True:
action_taken = 'SNMPget'
else:
action_taken = 'SNMPwalk'
log_message = ('%s - %s') % (action_taken, log_message)
log.log2die(1003, log_message)
def main():
"""Process agent data.
Args:
None
Returns:
None
"""
# Get configuration
config = reference.ReferenceSampleConfig()
api = reference.ReferenceSampleAPI(config)
agent_name = config.agent_name()
devicename = config.prefix
id_agent = reference.get_id_agent(agent_name, test=True)
# Instantiate an agent
report = reference.ReferenceSampleAgent(config, devicename, test=True)
# Populate data and post
report.populate_dict(data2post())
success = report.post()
# Posting success
if success is True:
# Log success
log_message = ('Successfully posted test data for agent ID %s'
'') % (id_agent)
log.log2see(1015, log_message)
# Try to retrieve data
uri = ('db/agent/getidagent/%s') % (id_agent)
results = api.get(uri)
# print results
if results['exists'] is True:
log_message = ('Successfully retrieved test data for agent ID %s'
'') % (id_agent)
log.log2see(1034, log_message)
print('\nOK\n')
else:
log_message = ('WARNING: Contacted this infoset server. '
'The data for the test agent ID %s is not present '
'in the database. Ingester has not added agent to '
'the database'
'') % (id_agent)
log.log2see(1035, log_message)
print('\nOK - Ingester not running\n')
else:
log_message = ('Failed to post data to the local infoset server. '
'Review the installation steps '
'and verify whether the API is running.')
log.log2die(1039, log_message)
print('\nFail\n')
def __init__(self, idx, start=None, stop=None):
"""Function for intializing the class.
Args:
idx: idx of datapoint
start: Starting timestamp
stop: Ending timestamp
Returns:
None
"""
# Initialize important variables
self.data = defaultdict(dict)
# Get the datapoint's base_type
datapointer = db_datapoint.GetIDX(idx)
self.base_type = datapointer.base_type()
self.agent_label = datapointer.agent_label()
# Redefine start times
if start is None:
self.ts_start = jm_general.normalized_timestamp() - (3600 * 24)
else:
self.ts_start = jm_general.normalized_timestamp(start)
# Redefine stop times
if stop is None:
self.ts_stop = jm_general.normalized_timestamp()
else:
self.ts_stop = jm_general.normalized_timestamp(stop)
# Fix edge cases
if self.ts_start > self.ts_stop:
self.ts_start = self.ts_stop
# Make sure datapoint exists
if db_datapoint.idx_exists(idx) is False:
log_message = ('idx %s not found.') % (idx)
log.log2die(1049, log_message)
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(Data.timestamp, Data.value).filter(and_(
Data.timestamp >= self.ts_start,
Data.timestamp <= self.ts_stop,
Data.idx_datapoint == idx))
# Massage data
for instance in result:
self.data[instance.timestamp] = instance.value
# Return the session to the database pool after processing
session.close()
def add_all(self, data_list, error_code, die=True):
"""Do a database modification.
Args:
data_list: List of sqlalchemy table objects
error_code: Error number to use if one occurs
die: Don't die if False, just return success
Returns:
success: True is successful
"""
# Initialize key variables
success = False
# Open database connection. Prepare cursor
session = self.session()
try:
# Update the database cache
session.add_all(data_list)
# Commit change
session.commit()
# Update success
success = True
except Exception as exception_error:
success = False
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warn(error_code, log_message)
except:
success = False
session.rollback()
log_message = ('Unexpected database exception')
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warn(error_code, log_message)
# disconnect from server
self.close()
# Return
return success
def add_all(self, data_list, error_code, die=True):
"""Do a database modification.
Args:
data_list: List of sqlalchemy table objects
error_code: Error number to use if one occurs
die: Don't die if False, just return success
Returns:
success: True is successful
"""
# Initialize key variables
success = False
# Open database connection. Prepare cursor
session = self.session()
try:
# Update the database cache
session.add_all(data_list)
# Commit change
session.commit()
# disconnect from server
self.close()
# Update success
success = True
except Exception as exception_error:
success = False
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warning(error_code, log_message)
except:
success = False
session.rollback()
log_message = ('Unexpected database exception')
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warning(error_code, log_message)
# Return
return success
def setup(self):
"""Setup database.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
config = self.config
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(URL,
echo=True,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size,
pool_recycle=3600)
# Try to create the database
print('Attempting to create database tables')
try:
sql_string = ('ALTER DATABASE %s CHARACTER SET utf8mb4 '
'COLLATE utf8mb4_general_ci') % (
config.db_name())
engine.execute(sql_string)
except:
log_message = (
'Cannot connect to database %s. '
'Verify database server is started. '
'Verify database is created. '
'Verify that the configured database authentication '
'is correct.') % (config.db_name())
log.log2die(1036, log_message)
# Apply schemas
print('Applying Schemas')
BASE.metadata.create_all(engine)
# Insert database entries
self._insert_agent_device()
self._insert_billcode()
self._insert_department()
self._insert_datapoint()
self._insert_config()
def purge(self):
"""Purge data from cache by posting to central server.
Args:
None
Returns:
success: "True: if successful
"""
# Initialize key variables
uid = self.data["uid"]
# Add files in cache directory to list
filenames = [
filename
for filename in os.listdir(self.cache_dir)
if os.path.isfile(os.path.join(self.cache_dir, filename))
]
# Read cache file
for filename in filenames:
# Only post files for our own UID value
if uid not in filename:
continue
# Get the full filepath for the cache file and post
filepath = os.path.join(self.cache_dir, filename)
with open(filepath, "r") as f_handle:
try:
data = json.load(f_handle)
except:
# Log removal
log_message = (
"Error reading previously cached agent data file %s " "for agent %s. May be corrupted." ""
) % (filepath, self.name())
log.log2die(1064, log_message)
# Post file
success = self.post(save=False, data=data)
# Delete file if successful
if success is True:
os.remove(filepath)
# Log removal
log_message = ("Purging cache file %s after successfully " "contacting server %s" "") % (
filepath,
self.url,
)
log.log2quiet(1029, log_message)
def __init__(self, did, config):
"""Function for intializing the class.
Args:
did: Datapoint id
config: Config object
Returns:
None
"""
# Initialize important variables
self.data_dict = defaultdict(dict)
# Prepare SQL query to read a record from the database.
# Only active oids
sql_query = (
'SELECT '
'idx, '
'idx_agent, '
'agent_label, '
'agent_source, '
'enabled, '
'base_type, '
'multiplier, '
'last_timestamp '
'FROM iset_datapoint '
'WHERE '
'(iset_datapoint.id=\'%s\') LIMIT 1') % (
did)
# Do query and get results
database = db.Database(config)
query_results = database.query(sql_query, 1052)
# Massage data
for row in query_results:
# uid found?
if not row[0]:
log_message = ('did %s not found.') % (did)
log.log2die(1047, log_message)
# Assign values
self.data_dict['idx'] = row[0]
self.data_dict['idx_agent,'] = row[1]
self.data_dict['agent_label'] = row[2]
self.data_dict['agent_source'] = row[3]
self.data_dict['enabled'] = row[4]
self.data_dict['base_type'] = row[5]
self.data_dict['multiplier'] = row[6]
self.data_dict['last_timestamp'] = row[7]
break
def check_sudo():
"""Check user isn't running as sudo.
Args:
None
Returns:
None
"""
# Prevent running as sudo user
if 'SUDO_UID' in os.environ:
log_message = ('Cannot run script using "sudo".')
log.log2die(1132, log_message)
def read_yaml_files(directories):
"""Read the contents of all yaml files in a directory.
Args:
directories: List of directory names with configuration files
Returns:
config_dict: Dict of yaml read
"""
# Initialize key variables
yaml_found = False
yaml_from_file = ''
all_yaml_read = ''
# Check each directory in sequence
for config_directory in directories:
# Check if config_directory exists
if os.path.isdir(config_directory) is False:
log_message = (
'Configuration directory "%s" '
'doesn\'t exist!' % config_directory)
log.log2die(1009, log_message)
# Cycle through list of files in directory
for filename in os.listdir(config_directory):
# Examine all the '.yaml' files in directory
if filename.endswith('.yaml'):
# YAML files found
yaml_found = True
# Read file and add to string
file_path = ('%s/%s') % (config_directory, filename)
with open(file_path, 'r') as file_handle:
yaml_from_file = file_handle.read()
# Append yaml from file to all yaml previously read
all_yaml_read = ('%s\n%s') % (all_yaml_read, yaml_from_file)
# Verify YAML files found in directory
if yaml_found is False:
log_message = (
'No files found in directory "%s" with ".yaml" '
'extension.') % (config_directory)
log.log2die(1010, log_message)
# Return
config_dict = yaml.load(all_yaml_read)
return config_dict
def _mkdir(directory):
"""Create a directory if it doesn't already exist.
Args:
directory: Directory name
Returns:
None
"""
# Do work
if os.path.exists(directory) is False:
os.makedirs(directory, mode=0o755)
else:
if os.path.isfile(directory) is True:
log_message = ('%s is not a directory.' '') % (directory)
log.log2die(1043, log_message)
def read_yaml_files(directories):
"""Read the contents of all yaml files in a directory.
Args:
directories: List of directory names with configuration files
Returns:
config_dict: Dict of yaml read
"""
# Initialize key variables
yaml_found = False
yaml_from_file = ''
all_yaml_read = ''
# Check each directory in sequence
for config_directory in directories:
# Check if config_directory exists
if os.path.isdir(config_directory) is False:
log_message = ('Configuration directory "%s" '
'doesn\'t exist!' % config_directory)
log.log2die(1009, log_message)
# Cycle through list of files in directory
for filename in os.listdir(config_directory):
# Examine all the '.yaml' files in directory
if filename.endswith('.yaml'):
# YAML files found
yaml_found = True
# Read file and add to string
file_path = ('%s/%s') % (config_directory, filename)
try:
with open(file_path, 'r') as file_handle:
yaml_from_file = file_handle.read()
except:
log_message = ('Error reading file %s. Check permissions, '
'existence and file syntax.'
'') % (file_path)
log.log2die(1065, log_message)
# Append yaml from file to all yaml previously read
all_yaml_read = ('%s\n%s') % (all_yaml_read, yaml_from_file)
# Verify YAML files found in directory
if yaml_found is False:
log_message = ('No files found in directory "%s" with ".yaml" '
'extension.') % (config_directory)
log.log2die(1010, log_message)
# Return
config_dict = yaml.load(all_yaml_read)
return config_dict
def systemd_daemon(agent_name, action=None):
"""Manage systemd daemon for agent.
Args:
agent_name: Name of agent
action: Action to occur
Returns:
None
"""
# Initialize key variables
executable = '/bin/systemctl'
options = ['start', 'stop', 'restart']
fixed_action = action.lower()
# Check user is root
running_username = getpass.getuser()
if running_username != 'root':
log_message = 'You can only run this command as the \'root\' user.'
log.log2die(1134, log_message)
# Check if agent exists
if systemd_exists(agent_name) is False:
log_message = 'systemd not configured for daemon {}'.format(agent_name)
log.log2die(1122, log_message)
# Process request
if fixed_action in options:
command = '{} {} {}.service'.format(
executable, fixed_action, agent_name)
run_script(command)
else:
log_message = (
'Invalid action "{}" for systemd daemon {}'
''.format(action, agent_name))
log.log2die(1037, log_message)
def run_script(cli_string, shell=False, die=True):
"""Run the cli_string UNIX CLI command and record output.
Args:
cli_string: Command to run on the CLI
die: Die if command runs with an error
Returns:
None
"""
# Initialize key variables
encoding = locale.getdefaultlocale()[1]
header_returncode = ('[Return Code]')
header_stdout = ('[Output]')
header_stderr = ('[Error Message]')
header_bad_cmd = ('[ERROR: Bad Command]')
log_message = ''
# Create the subprocess object
if shell is False:
do_command_list = list(cli_string.split(' '))
process = subprocess.Popen(do_command_list,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
process = subprocess.Popen(cli_string,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdoutdata, stderrdata = process.communicate()
returncode = process.returncode
# Crash if the return code is not 0
if die is True:
if returncode != 0:
# Print the Return Code header, Return Code, STDOUT header
string2print = ('%s %s %s %s') % (header_bad_cmd, cli_string,
header_returncode, returncode)
log_message = ('%s%s') % (log_message, string2print)
# Print the STDERR
string2print = ('%s') % (header_stderr)
log_message = ('%s %s') % (log_message, string2print)
for line in stderrdata.decode(encoding).split('\n'):
string2print = ('%s') % (line)
log_message = ('%s %s') % (log_message, string2print)
# Print the STDOUT
string2print = ('%s') % (header_stdout)
log_message = ('%s %s') % (log_message, string2print)
for line in stdoutdata.decode(encoding).split('\n'):
string2print = ('%s') % (line)
log_message = ('%s %s') % (log_message, string2print)
# All done
log.log2die(1074, log_message)
# Return
return stdoutdata
def main():
"""Process agent data.
Args:
None
Returns:
None
"""
# Get configuration
config = reference.ReferenceSampleConfig()
api = reference.ReferenceSampleAPI(config)
agent_name = config.agent_name()
devicename = config.prefix
id_agent = reference.get_id_agent(agent_name, test=True)
# Instantiate an agent
report = reference.ReferenceSampleAgent(config, devicename, test=True)
# Populate data and post
report.populate_dict(data2post())
success = report.post()
# Posting success
if success is True:
# Log success
log_message = (
'Successfully posted test data for agent ID %s'
'') % (id_agent)
log.log2see(1015, log_message)
# Try to retrieve data
uri = ('/agents?id_agent=%s') % (id_agent)
results = api.get(uri)
if bool(results) is True:
if isinstance(results, dict) is True:
# print results
if results['exists'] is True:
log_message = (
'Successfully retrieved test data for agent ID %s'
'') % (id_agent)
log.log2see(1132, log_message)
print('\nOK\n')
else:
log_message = (
'WARNING: Contacted this infoset server. '
'The data for the test agent ID %s is not present '
'in the database. Ingester has not added agent to '
'the database'
'') % (id_agent)
log.log2see(1133, log_message)
print("""\
OK, but Ingester has not updated the database yet. \
Run test in a minute and this message should change. \
If not, the Ingester may not be running.
""")
else:
log_message = (
'Failed to retrieve posted data to the local infoset '
'server. Review the installation steps '
'and verify whether the API is running.')
log.log2die(1140, log_message)
print('\nFail\n')
else:
log_message = (
'Failed to retrieve posted data to the local infoset '
'server. Review the installation steps '
'and verify whether the API is running.')
log.log2die(1141, log_message)
print('\nFail\n')
else:
log_message = (
'Failed to post data to the local infoset server. '
'Review the installation steps '
'and verify whether the API is running.')
log.log2die(1142, log_message)
print('\nFail\n')