def _read_data_from_file(filepath):
"""Provide validated information when valid.
Args:
filepath: Path to file
Returns:
data: Data
"""
# Initialize key variables
data = {}
# Try reading file if filename format is OK
if _valid_filename(filepath) is True:
# Ingest data
try:
with open(filepath, 'r') as f_handle:
data = json.load(f_handle)
except:
# Log status
log_message = (
'File %s does not contain JSON data, does not exist, '
'or is unreadable.') % (filepath)
log.log2warn(1006, log_message)
else:
# Log status
log_message = (
'File %s does has incorrect filename format.'
'') % (filepath)
log.log2warn(1026, log_message)
# Return
return data
def _check_timestamp_key(information):
"""Check whether timestamp key is an integer value.
Args:
information: Dict of JSON data
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
key = 'timestamp'
# Verify we have a dictionary
if isinstance(information, dict) is False:
log_message = ('Ingest data is not a dictionary')
log.log2warn(1110, log_message)
valid = False
else:
# Timestamp must be an integer
try:
if key in information:
int(information[key])
except:
log_message = ('Ingest data has no valid timestamp key')
log.log2warn(1047, log_message)
valid = False
# Return
return valid
def _check_primary_keys_exist(information):
"""Check whether primary keys exist in json.
Args:
information: Data dictionary to check
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
agent_meta_keys = ['timestamp', 'uid', 'agent', 'hostname']
# Verify we have a dictionary
if isinstance(information, dict) is False:
log_message = ('Ingest data is not a dictionary')
log.log2warn(1093, log_message)
valid = False
else:
# Verify keys in information
for key in agent_meta_keys:
if key not in information:
# Log status and stop processing
valid = False
log_message = ('Ingest data does not have a %s key.') % (key)
log.log2warn(1039, log_message)
break
# Return
return valid
def purge(self):
"""Purge cache file that was read.
Args:
None
Returns:
success: "True" if successful
"""
# Initialize key variables
success = True
try:
os.remove(self.filename)
except:
success = False
# Report success
if success is True:
log_message = (
'Ingest cache file %s deleted') % (self.filename)
log.log2quiet(1046, log_message)
else:
log_message = (
'Failed to delete ingest cache file %s') % (self.filename)
log.log2warn(1087, log_message)
# Return
return success
def _poll(self):
"""Query all remote hosts for data.
Args:
None
Returns:
None
"""
# Initialize key variables
pollers = []
# Create a list of polling objects
hostnames = self.config.agent_hostnames()
for hostname in hostnames:
# Only poll hosts that exist in the database
if db_host.hostname_exists(hostname) is False:
log_message = (
'Agent "%s": Hostname %s in the configuration file '
'does not exist in the database. '
'Run the snmp_evaluate_hosts.py script.'
'') % (self.agent_name, hostname)
log.log2warn(1095, log_message)
continue
# Add poller
poller = Poller(hostname, self.agent_name)
pollers.append(poller)
# Start threaded polling
if bool(pollers) is True:
Agent.threads(self.agent_name, pollers)
def valid(self):
"""Master method that defines whether data is OK.
Args:
None
Returns:
all_ok:
"""
# Initialize key variables
validity = [self.validated]
# Append results of tests
validity.append(self._check_meta())
validity.append(self._check_data_types())
validity.append(self._check_duplicates())
# Do final check
if False in validity:
all_ok = False
# Error message
if self.filepath is not None:
log_message = (
'Cache file %s is invalid'
'') % (self.filepath)
log.log2warn(1021, log_message)
else:
log_message = ('Cache data is invalid')
log.log2warn(1059, log_message)
else:
all_ok = True
# Return
return all_ok
def __init__(self, agent_name):
"""Method initializing the class.
Args:
agent_name
Returns:
None
"""
# Initialize key variables
self.agent_name = agent_name
self.agent_yaml = {}
# Get the language used
config = jm_configuration.Config()
lang = config.language()
# Determine the agent's language yaml file
root_directory = jm_general.root_directory()
yaml_file = (
'%s/infoset/metadata/%s/agents/%s.yaml') % (
root_directory, lang, self.agent_name)
# Read the agent's language yaml file
if os.path.exists(yaml_file) is True:
with open(yaml_file, 'r') as file_handle:
yaml_from_file = file_handle.read()
self.agent_yaml = yaml.load(yaml_from_file)
else:
log_message = ('Agent language file %s does not exist.') % (
yaml_file)
log.log2warn(1034, log_message)
def run(self):
"""Update the database using threads."""
while True:
# Initialize key variables
hostname = None
# Get the data_dict
data_dict = self.queue.get()
uid = data_dict['uid']
metadata = data_dict['metadata']
config = data_dict['config']
# Initialize other values
max_timestamp = 0
# Sort metadata by timestamp
metadata.sort()
# Process file for each timestamp, starting from the oldes file
for (timestamp, filepath) in metadata:
# Read in data
ingest = drain.Drain(filepath)
# Make sure file is OK
# Move it to a directory for further analysis
# by administrators
if ingest.valid() is False:
log_message = (
'Cache ingest file %s is invalid. Moving.'
'') % (filepath)
log.log2warn(1054, log_message)
shutil.copy(
filepath, config.ingest_failures_directory())
os.remove(filepath)
continue
# Update database
dbase = UpdateDB(ingest)
dbase.update()
# Get the max timestamp
max_timestamp = max(timestamp, max_timestamp)
# Get hostname
hostname = ingest.hostname()
# Purge source file
ingest.purge()
# Update the last time the agent was contacted
_update_agent_last_update(uid, max_timestamp)
# Update the host / agent table timestamp if hostname was processed
if hostname is not None:
_host_agent_last_update(hostname, uid, max_timestamp)
# All done!
self.queue.task_done()
def stop(self):
"""Stop the daemon.
Args:
None
Returns:
"""
# Get the pid from the pidfile
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if not pid:
log_message = (
'PID file: %s does not exist. Daemon not running?'
'') % (self.pidfile)
log.log2warn(1063, log_message)
# Not an error in a restart
return
# Try killing the daemon process
try:
while 1:
if self.lockfile is None:
os.kill(pid, signal.SIGTERM)
else:
time.sleep(0.3)
if os.path.exists(self.lockfile) is True:
continue
else:
os.kill(pid, signal.SIGTERM)
time.sleep(0.3)
except OSError as err:
error = str(err.args)
if error.find("No such process") > 0:
self.delpid()
self.dellock()
else:
log_message = (str(err.args))
log_message = (
'%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1068, log_message)
except:
log_message = (
'Unknown daemon "stop" error for PID file: %s'
'') % (self.pidfile)
log.log2die(1066, log_message)
# Log success
self.delpid()
self.dellock()
log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
log.log2quiet(1071, log_message)
def _check_duplicates(information):
"""Check whether reported data reported is already in the database.
Args:
None
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
# Check that we are evaluating a dict
if isinstance(information, dict) is False:
log_message = ('Ingest data is not a dictionary')
log.log2warn(1116, log_message)
valid = False
return valid
# Check that we have the correct keys in the dict
if _check_primary_keys_exist(information) is False:
valid = False
return valid
# Get values
timestamp = int(information['timestamp'])
uid = information['uid']
hostname = information['hostname']
# Check if there is a duplicate entry for this UID
if db_agent.uid_exists(uid) is not False:
idx_agent = db_agent.GetUID(uid).idx()
# Check if host exists
if db_host.hostname_exists(hostname) is True:
idx_host = db_host.GetHost(hostname).idx()
# Check for host / agent entry existence
if db_hostagent.host_agent_exists(
idx_host, idx_agent) is True:
# Check if this host / agent has been updated before
last_timesamp = db_hostagent.GetHostAgent(
idx_host, idx_agent).last_timestamp()
# Validate
if timestamp <= last_timesamp:
log_message = (
'Data for UID %s, hostname %s at timestamp %s '
'is already found in database.'
'') % (uid, hostname, timestamp)
log.log2warn(1113, log_message)
valid = False
# Return
return valid
def post(self, save=True, data=None):
"""Post data to central server.
Args:
save: When True, save data to cache directory if postinf fails
data: Data to post. If None, then uses self.data
Returns:
success: "True: if successful
"""
# Initialize key variables
success = False
response = False
timestamp = self.data['timestamp']
uid = self.data['uid']
# Create data to post
if data is None:
data = self.data
# Post data save to cache if this fails
try:
result = requests.post(self.url, json=data)
response = True
except:
if save is True:
# Create a unique very long filename to reduce risk of
hosthash = jm_general.hashstring(self.data['hostname'], sha=1)
filename = ('%s/%s_%s_%s.json') % (
self.cache_dir, timestamp, uid, hosthash)
# Save data
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
# Define success
if response is True:
if result.status_code == 200:
success = True
# Log message
if success is True:
log_message = (
'Agent "%s" successfully contacted server %s'
'') % (self.name(), self.url)
log.log2quiet(1027, log_message)
else:
log_message = (
'Agent "%s" failed to contact server %s'
'') % (self.name(), self.url)
log.log2warn(1028, log_message)
# Return
return success
def add_all(self, data_list, error_code, die=True):
"""Do a database modification.
Args:
data_list: List of sqlalchemy table objects
error_code: Error number to use if one occurs
die: Don't die if False, just return success
Returns:
success: True is successful
"""
# Initialize key variables
success = False
# Open database connection. Prepare cursor
session = self.session()
try:
# Update the database cache
session.add_all(data_list)
# Commit change
session.commit()
# Update success
success = True
except Exception as exception_error:
success = False
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warn(error_code, log_message)
except:
success = False
session.rollback()
log_message = ('Unexpected database exception')
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warn(error_code, log_message)
# disconnect from server
self.close()
# Return
return success
def query(self):
"""Query all remote hosts for data.
Args:
None
Returns:
None
"""
# Check each hostname
hostnames = self.config.agent_snmp_hostnames()
for hostname in hostnames:
# Get valid SNMP credentials
validate = snmp_manager.Validate(
hostname, self.snmp_config.snmp_auth())
snmp_params = validate.credentials()
# Log message
if snmp_params is None:
log_message = (
'No valid SNMP configuration found '
'for host "%s" ') % (hostname)
log.log2warn(1006, log_message)
continue
# Create Query make sure MIB is supported
snmp_object = snmp_manager.Interact(snmp_params)
snmp_query = mib_sentry3.init_query(snmp_object)
if snmp_query.supported() is False:
log_message = (
'The Sentry3 MIB is not supported by host "%s"'
'') % (hostname)
log.log2warn(1001, log_message)
continue
# Get the UID for the agent after all preliminary checks are OK
uid_env = agent.get_uid(hostname)
# Post data to the remote server
self.upload(uid_env, hostname, snmp_query)
def query(self):
"""Query all remote hosts for data.
Args:
None
Returns:
None
"""
# Initialize key variables
port = self.config.agent_port()
# Set logging
log_file = self.config.log_file()
logging.basicConfig(filename=log_file, level=logging.DEBUG)
# Post data to the remote server
log.log2warn(101010101010101010101010101010101, 'boo')
infoset.run(
debug=True, host='0.0.0.0',
threaded=True, port=port)
def _check_primary_keys_in_file(self):
"""Validate the values of the primary JSON keys in the ingest file.
Args:
None
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
filepath = self.filepath
# Parse filename for information
if self.filepath is not None:
if _valid_filename(filepath) is True:
filename = os.path.basename(filepath)
(name, _) = filename.split('.')
(tstamp, uid, _) = name.split('_')
timestamp = int(tstamp)
# Double check that the UID and timestamp in the
# filename matches that in the file.
# Ignore invalid files as a safety measure.
# Don't try to delete. They could be owned by some
# one else and the daemon could crash
if uid != self.information['uid']:
log_message = (
'UID %s in file %s does not match UID %s in filename.'
'') % (self.information['uid'], uid, filepath)
log.log2warn(1123, log_message)
valid = False
# Check timestamp
if timestamp != self.information['timestamp']:
log_message = (
'Timestamp %s in file %s does not match timestamp '
'%s in filename.'
'') % (
self.information['timestamp'],
timestamp, filepath)
log.log2warn(1111, log_message)
valid = False
# Check timestamp validity
if jm_general.validate_timestamp(timestamp) is False:
log_message = (
'Timestamp %s in file %s is not normalized'
'') % (self.information['timestamp'], filepath)
log.log2warn(1112, log_message)
valid = False
else:
valid = False
# Return
return valid
def _check_reported_data(information):
"""Check the data types being reported by the agent.
Args:
information: Data to analyze
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
data_types = ['chartable', 'other']
# Check that we are evaluating a dict
if isinstance(information, dict) is False:
log_message = ('Ingest data is not a dictionary')
log.log2warn(1121, log_message)
valid = False
return valid
# Process chartable data
for data_type in data_types:
# Skip if data type isn't in the data
if data_type not in information:
continue
# Process the data type
for _, reported_data in sorted(
information[data_type].items()):
# Process keys
for key in ['base_type', 'description', 'data']:
if key not in reported_data:
log_message = (
'"%s" data type does not contain a "%s" key.'
'') % (data_type, key)
log.log2warn(1115, log_message)
valid = False
# Process data
if 'data' in reported_data:
for datapoint in reported_data['data']:
if len(datapoint) != 3:
log_message = (
'"%s" data type does not contain valid '
'datapoints in it\'s "data" key.'
'') % (data_type)
log.log2warn(1114, log_message)
valid = False
# Return
return valid
def threads(agent_name, pollers):
"""Function where agents poll devices using multithreading.
Args:
agent_name: Agent name
pollers: List of polling objects
Returns:
None
"""
# Get configuration
config = jm_configuration.Config()
threads_in_pool = config.agent_threads()
# Spawn processes only if we have files to process
if bool(pollers) is True:
# Process lock file
f_obj = hidden.File()
lockfile = f_obj.lock(agent_name)
if os.path.exists(lockfile) is True:
# Return if lock file is present
log_message = (
"Agent lock file %s exists. Multiple agent daemons "
"running or the daemon may have died "
"catastrophically in the past, in which case the lockfile "
"should be deleted. Exiting agent process. "
"Will try again later."
""
) % (lockfile)
log.log2warn(1044, log_message)
return
else:
# Create lockfile
open(lockfile, "a").close()
# Spawn a pool of threads, and pass them queue instance
for _ in range(min(threads_in_pool, len(pollers))):
update_thread = AgentThread(THREAD_QUEUE)
update_thread.daemon = True
# Sometimes we exhaust the thread abilities of the OS
# even with the "threads_in_pool" limit. This is because
# there could be a backlog of files to cache files process
# and we have overlapping ingests due to a deleted lockfile.
# This code ensures we don't exceed the limits.
try:
update_thread.start()
except RuntimeError:
log_message = (
'Too many threads created for agent "%s". ' "Verify that agent lock file is present."
) % (agent_name)
# Remove the lockfile so we can restart later then die
os.remove(lockfile)
log.log2die(1078, log_message)
except:
log_message = ("Unknown error occurred when trying to " 'create threads for agent "%s"') % (agent_name)
# Remove the lockfile so we can restart later then die
os.remove(lockfile)
log.log2die(1079, log_message)
# Start polling
for poller in pollers:
##############################################################
#
# Define variables that will be required for the threading
# We have to initialize the dict during every loop to prevent
# data corruption
#
##############################################################
THREAD_QUEUE.put(poller)
# Wait on the queue until everything has been processed
THREAD_QUEUE.join()
# PYTHON BUG. Join can occur while threads are still shutting down.
# This can create spurious "Exception in thread (most likely raised
# during interpreter shutdown)" errors.
# The "time.sleep(1)" adds a delay to make sure things really terminate
# properly. This seems to be an issue on virtual machines in Dev only
time.sleep(1)
# Return if lock file is present
if os.path.exists(lockfile) is True:
os.remove(lockfile)
def process(agent_name):
"""Method initializing the class.
Args:
agent_name: agent name
Returns:
None
"""
# Initialize key variables
argument_list = []
uid_metadata = defaultdict(lambda: defaultdict(dict))
# Configuration setup
config = jm_configuration.Config()
threads_in_pool = config.ingest_threads()
# Make sure we have database connectivity
if db.connectivity() is False:
log_message = (
'No connectivity to database. Check if running. '
'Check database authentication parameters.'
'')
log.log2warn(1053, log_message)
return
# Get meta data on files
uid_metadata = validate_cache_files()
# Spawn processes only if we have files to process
if bool(uid_metadata.keys()) is True:
# Process lock file
f_obj = hidden.File()
lockfile = f_obj.lock(agent_name)
if os.path.exists(lockfile) is True:
# Return if lock file is present
log_message = (
'Ingest lock file %s exists. Multiple ingest daemons running '
'or lots of cache files to ingest. Ingester may have died '
'catastrophically in the past, in which case the lockfile '
'should be deleted. Exiting ingest process. '
'Will try again later.'
'') % (lockfile)
log.log2warn(1069, log_message)
return
else:
# Create lockfile
open(lockfile, 'a').close()
# Read each cache file
for hosthash in uid_metadata.keys():
for uid in uid_metadata[hosthash].keys():
# Create a list of arguments to process
argument_list.append(
(config, uid_metadata[hosthash][uid])
)
# Create a pool of sub process resources
with Pool(processes=threads_in_pool) as pool:
# Create sub processes from the pool
pool.map(_wrapper_process, argument_list)
# Wait for all the processes to end
# pool.join()
# Return if lock file is present
if os.path.exists(lockfile) is True:
os.remove(lockfile)
def _check_chartable_data(information):
"""Check the data types being reported by the agent.
Args:
information: Data to analyze
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
data_type = 'chartable'
# Check that we are evaluating a dict
if isinstance(information, dict) is False:
log_message = ('Ingest data is not a dictionary')
log.log2warn(1122, log_message)
valid = False
return valid
# Check for chartable data
if data_type in information:
# Process the data type
for _, reported_data in sorted(
information[data_type].items()):
# Make sure the base types are numeric
if 'base_type' in reported_data:
try:
float(reported_data['base_type'])
except:
log_message = (
'Chartable "base_type" key is non numeric.')
log.log2warn(1120, log_message)
valid = False
else:
log_message = (
'Chartable data has no "base_type" key.')
log.log2warn(1117, log_message)
valid = False
# Process data
if 'data' in reported_data:
for datapoint in reported_data['data']:
# Check to make sure value is numeric
value = datapoint[1]
try:
float(value)
except:
log_message = (
'Chartable data has non numeric data values.')
log.log2warn(1119, log_message)
valid = False
break
else:
log_message = (
'Chartable data has no "data" key.')
log.log2warn(1118, log_message)
valid = False
# Return
return valid
def _datapoints(self):
"""Create the master dictionary for the host.
Args:
None
Returns:
value: Index value
"""
# Initialize key variables
snmp_params = self.snmp_params
master = self._master()
# Get sources
snmp_object = snmp_manager.Interact(snmp_params)
for labels_oid in master.keys():
sources = {}
oid_results = snmp_object.swalk(labels_oid)
# Return if there is an error
if bool(oid_results) is False:
log_message = (
'Failed to contact SNMP host %s. '
'Will collect data on next poll.'
'') % (self.hostname)
log.log2warn(1024, log_message)
return
for key, value in oid_results.items():
sources[_index(labels_oid, key)] = jm_general.decode(value)
# Get values
for labels_oid in master.keys():
for agent_label in master[labels_oid].keys():
# Initialize datapoints
datapoints = defaultdict(lambda: defaultdict(dict))
# Information about the OID
values_oid = master[labels_oid][agent_label]['values_oid']
base_type = master[labels_oid][agent_label]['base_type']
multiplier = master[labels_oid][agent_label]['multiplier']
# Get OID values
values = {}
oid_results = snmp_object.swalk(values_oid)
# Return if there is an error
if bool(oid_results) is False:
log_message = (
'Failed to contact SNMP host %s. '
'Will collect data on next poll.'
'') % (self.hostname)
log.log2warn(1022, log_message)
return
# Only process floating point values
for key, value in oid_results.items():
try:
_ = float(value)
except:
continue
values[_index(labels_oid, key)] = value * multiplier
# Create list of data for json
data = []
for index, value in values.items():
data.append([index, value, sources[index]])
# Finish up dict for json
datapoints[agent_label]['data'] = data
datapoints[agent_label]['base_type'] = base_type
# Populate agent
self.agent.populate(datapoints)
# Post data
self.agent.post()
def __init__(self, filename):
"""Method initializing the class.
Args:
filename: Cache filename
Returns:
None
"""
# Initialize key variables
self.filename = filename
self.data = defaultdict(lambda: defaultdict(dict))
self.metadata = []
self.validated = False
self.agent_meta = {}
data_types = ['chartable', 'other']
agent_meta_keys = ['timestamp', 'uid', 'agent', 'hostname']
# Ingest data
validator = validate.ValidateCache(filename)
information = validator.getinfo()
# Log if data is bad
if information is False:
log_message = (
'Cache ingest file %s is invalid.') % (filename)
log.log2warn(1051, log_message)
return
else:
self.validated = True
if self.validated is True:
# Get universal parameters from file. Convert to unicode
for key in agent_meta_keys:
if key == 'timestamp':
self.agent_meta[key] = int(information[key])
else:
self.agent_meta[key] = information[key]
timestamp = self.agent_meta['timestamp']
uid = self.agent_meta['uid']
# Process chartable data
for data_type in data_types:
# Skip if data type isn't in the data
if data_type not in information:
continue
# Process the data type
for label, group in sorted(
information[data_type].items()):
# Get universal parameters for group
base_type = _base_type(group['base_type'])
description = group['description']
# Initialize base type
if base_type not in self.data[data_type]:
self.data[data_type][base_type] = []
# Process data
for datapoint in group['data']:
index = datapoint[0]
value = datapoint[1]
source = datapoint[2]
did = _did(
uid, label, index,
self.agent_meta['agent'],
self.agent_meta['hostname'])
# Update data
if base_type != 0:
self.data[data_type][base_type].append(
(uid, did, value, timestamp)
)
else:
self.data[data_type][base_type].append(
(uid, did, value, timestamp)
)
# Update sources after fixing encoding
self.metadata.append(
(uid, did, label, source,
description, base_type)
)
def valid(self):
"""Master method that defines whether data is OK.
Args:
None
Returns:
all_ok:
"""
# Initialize key variables
validity = []
ts_start = time.time()
# Check primary keys
validity.append(
_check_primary_keys_exist(self.information))
# Check timestamp key
if False not in validity:
validity.append(
_check_timestamp_key(self.information))
# Check validity of primary keys in file
if False not in validity:
validity.append(
self._check_primary_keys_in_file())
# Check chartable and unchartable data in the data
if False not in validity:
validity.append(_check_reported_data(self.information))
if False not in validity:
validity.append(_check_chartable_data(self.information))
# Check if data to be validated is already in the database
if False not in validity:
validity.append(_check_duplicates(self.information))
# Do final check
if False in validity:
# Log failure
if self.filepath is None:
mid_string = ''
else:
mid_string = ('in %s') % (self.filepath)
log_message = ('Cache data %s is invalid') % (mid_string)
log.log2warn(1059, log_message)
all_ok = False
else:
# Log success
ts_stop = time.time()
duration = ts_stop - ts_start
if self.filepath is None:
mid_string = ''
else:
mid_string = ('of %s') % (self.filepath)
log_message = (
'Data validation %s took %s seconds.'
'') % (mid_string, round(duration, 4))
log.log2quiet(1126, log_message)
all_ok = True
# Return
return all_ok
def run(self):
"""Update the database using threads."""
while True:
# Get the data_dict
data_dict = self.queue.get()
uid = data_dict['uid']
metadata = data_dict['metadata']
config = data_dict['config']
agents = data_dict['agents']
datapoints = data_dict['datapoints']
# Initialize other values
max_timestamp = 0
# Sort metadata by timestamp
metadata.sort()
# Process file for each timestamp
for (timestamp, filepath) in metadata:
# Read in data
ingest = drain.Drain(filepath)
# Make sure file is OK
# Move it to a directory for further analysis
# by administrators
if ingest.valid() is False:
log_message = (
'Cache ingest file %s is invalid. Moving.'
'') % (filepath)
log.log2warn(1054, log_message)
shutil.move(
filepath, config.ingest_failures_directory())
continue
# Update agent table if not there
if ingest.uid() not in agents:
_insert_agent(
ingest.uid(),
ingest.agent(),
ingest.hostname(),
config
)
# Append the new insertion to the list
agents.append(ingest.uid())
# Update datapoint metadata if not there
for item in ingest.sources():
did = item[1]
if did not in datapoints:
_insert_datapoint(item, config)
# Append the new insertion to the list
datapoints.append(did)
# Create map of DIDs to database row index values
mapping = _datapoints_by_did(config)
# Update chartable data
_update_chartable(mapping, ingest, config)
_update_unchartable(mapping, ingest, config)
# Get the max timestamp
max_timestamp = max(timestamp, max_timestamp)
# Purge source file
ingest.purge()
# Update the last time the agent was contacted
_update_agent_last_update(uid, max_timestamp, config)
# All done!
self.queue.task_done()
def process(agent_name):
"""Method initializing the class.
Args:
agent_name: agent name
Returns:
None
"""
# Initialize key variables
uid_metadata = defaultdict(lambda: defaultdict(dict))
# Configuration setup
config = jm_configuration.Config()
threads_in_pool = config.ingest_threads()
# Make sure we have database connectivity
if db.connectivity() is False:
log_message = (
'No connectivity to database. Check if running. '
'Check database authentication parameters.'
'')
log.log2warn(1053, log_message)
return
# Get meta data on files
uid_metadata = validate_cache_files()
# Spawn processes only if we have files to process
if bool(uid_metadata.keys()) is True:
# Process lock file
f_obj = hidden.File()
lockfile = f_obj.lock(agent_name)
if os.path.exists(lockfile) is True:
# Return if lock file is present
log_message = (
'Ingest lock file %s exists. Multiple ingest daemons running '
'or lots of cache files to ingest. Ingester may have died '
'catastrophically in the past, in which case the lockfile '
'should be deleted. Exiting ingest process. '
'Will try again later.'
'') % (lockfile)
log.log2warn(1069, log_message)
return
else:
# Create lockfile
open(lockfile, 'a').close()
# Spawn a pool of threads, and pass them queue instance
# Only create the required number of threads up to the
# threads_in_pool maximum
for _ in range(
min(threads_in_pool, len(uid_metadata))):
update_thread = ProcessUID(THREAD_QUEUE)
update_thread.daemon = True
# Sometimes we exhaust the thread abilities of the OS
# even with the "threads_in_pool" limit. This is because
# there could be a backlog of files to cache files process
# and we have overlapping ingests due to a deleted lockfile.
# This code ensures we don't exceed the limits.
try:
update_thread.start()
except RuntimeError:
log_message = (
'Too many threads created for cache ingest. '
'Verify that ingest lock file is present.')
# Remove the lockfile so we can restart later then die
os.remove(lockfile)
log.log2die(1067, log_message)
except:
log_message = (
'Unknown error occurred when trying to '
'create cache ingest threads')
# Remove the lockfile so we can restart later then die
os.remove(lockfile)
log.log2die(1072, log_message)
# Read each cache file
for hosthash in uid_metadata.keys():
for uid in uid_metadata[hosthash].keys():
##############################################################
#
# Define variables that will be required for the threading
# We have to initialize the dict during every loop to prevent
# data corruption
#
##############################################################
data_dict = {}
data_dict['uid'] = uid
data_dict['metadata'] = uid_metadata[hosthash][uid]
data_dict['config'] = config
THREAD_QUEUE.put(data_dict)
# Wait on the queue until everything has been processed
THREAD_QUEUE.join()
# PYTHON BUG. Join can occur while threads are still shutting down.
# This can create spurious "Exception in thread (most likely raised
# during interpreter shutdown)" errors.
# The "time.sleep(1)" adds a delay to make sure things really terminate
# properly. This seems to be an issue on virtual machines in Dev only
time.sleep(1)
# Return if lock file is present
if os.path.exists(lockfile) is True:
os.remove(lockfile)
def process(self):
"""Update the database using threads."""
# Initialize key variables
updated = False
hostnames = []
uids = []
ingests = []
agent_names = []
agent_data = {
'hostname': None,
'uid': None,
'sources': [],
'chartable': [],
'unchartable': []
}
# Get the data_dict
metadata = self.metadata
config = self.config
# Initialize other values
max_timestamp = 0
# Get start time for activity
start_ts = time.time()
# Sort metadata by timestamp
metadata.sort()
# Process file for each timestamp, starting from the oldes file
for (timestamp, filepath) in metadata:
# Read in data
ingest = drain.Drain(filepath)
# Make sure file is OK
# Move it to a directory for further analysis
# by administrators
if ingest.valid() is False:
log_message = (
'Cache ingest file %s is invalid. Moving.'
'') % (filepath)
log.log2warn(1054, log_message)
shutil.copy(
filepath, config.ingest_failures_directory())
os.remove(filepath)
continue
# Append data
agent_data['chartable'].extend(ingest.chartable())
agent_data['unchartable'].extend(ingest.other())
agent_data['sources'].extend(ingest.sources())
hostnames.append(ingest.hostname())
uids.append(ingest.uid())
ingests.append(ingest)
agent_names.append(ingest.agent())
# Purge source file
# ingest.purge()
# Get the max timestamp
max_timestamp = max(timestamp, max_timestamp)
# Update update flag
updated = True
# Verify that we have only processed data for the same hostname
# UID and agent name
if (jm_general.all_same(hostnames) is False) or (
jm_general.all_same(uids) is False) or (
jm_general.all_same(agent_names) is False):
log_message = (
'Cache ingest files error for hostname %s,'
'agent name %s, UID %s.'
'') % (hostnames[0], agent_names[0], uids[0])
log.log2quiet(1083, log_message)
# Process the rest
if updated is True:
# Update remaining agent data
agent_data['hostname'] = hostnames[0]
agent_data['uid'] = uids[0]
agent_data['agent_name'] = agent_names[0]
# Update database
dbase = UpdateDB(agent_data)
dbase.update()
# Update the last time the agent was contacted
_update_agent_last_update(agent_data['uid'], max_timestamp)
# Update the host / agent table timestamp if
# hostname was processed
_host_agent_last_update(
agent_data['hostname'], agent_data['uid'], max_timestamp)
# Purge source files. Only done after complete
# success of database updates. If not we could lose data in the
# event of an ingester crash. Ingester would re-read the files
# and process the non-duplicates, while deleting the duplicates.
for ingest in ingests:
ingest.purge()
# Log duration of activity
duration = time.time() - start_ts
log_message = (
'UID %s was processed in %s seconds.'
'') % (agent_data['uid'], duration)
log.log2quiet(1127, log_message)
