def __init__(self, filepath):
"""Method initializing the class.
Args:
filepath: Cache filename
Returns:
None
"""
# Initialize key variables
self.filepath = filepath
self.data = None
self._valid = False
name_ok = _valid_filename(filepath)
# Read data from file
if name_ok is True:
self.data = _read_data_from_file(filepath)
else:
# Log status
log_message = ('File %s does has incorrect filename format.'
'') % (filepath)
log.log2warning(1026, log_message)
# Check main keys in data.
contents = _CheckMainKeys(self.data)
if contents.valid() is True:
if name_ok is True:
self._valid = True
def __init__(self, data):
do_update = False
success = None
agent_data = {
'devicename': None,
'id_agent': None,
'sources': [],
'timeseries': [],
'timefixed': [],
'max_timestamp': 0
}
ingest = drain.Drain(filename=None, data=data)
if ingest.valid() is False:
log_message = (
'Cache data at timestamp %s is invalid. Moving.'
'') % (ingest.timestamp())
log.log2warning(1054, log_message)
else:
agent_data['timeseries'] = ingest.timeseries()
agent_data['timefixed'] = ingest.timefixed()
agent_data['sources'] = ingest.sources()
agent_data['devicename'] = ingest.devicename()
agent_data['id_agent'] = ingest.id_agent()
agent_data['agent_name'] = ingest.agent()
agent_data['max_timestamp'] = ingest.timestamp()
# Upadate and note success
(success, datapoints_processed) = self._do_update(
agent_data)
def _read_data_from_file(filepath):
"""Provide validated information when valid.
Args:
filepath: Path to file
Returns:
data: Data
"""
# Initialize key variables
data = {}
# Ingest data
try:
with open(filepath, 'r') as f_handle:
data = json.load(f_handle)
except:
# Log status
log_message = ('File %s does not contain JSON data, does not exist, '
'or is unreadable.') % (filepath)
log.log2warning(1006, log_message)
# Return
return data
def _data_keys_ok(self):
"""Check if data keys are OK.
Args:
None
Returns:
valid: True if valid
"""
# Initialize key variables
valid = False
count = 0
data_types = ['timeseries', 'timefixed']
# Process data
if self._valid is True:
for data_type in data_types:
# Skip if data type isn't in the data
if data_type in self.data:
count += 1
# Verify validity
if count > 0:
valid = True
# Log error
if valid is False:
log_message = ('Ingest data does not contain all data keys.')
log.log2warning(1003, log_message)
# Return
return valid
def valid(self):
"""Validate main keys.
Args:
None
Returns:
valid: Valid if True
"""
# Initialize key variables
valid = False
valid_list = [self._valid]
valid_list.append(self._timestamp())
valid_list.append(self._agent())
valid_list.append(self._id_agent())
valid_list.append(self._devicename())
# Define validity
if len(valid_list) == valid_list.count(True):
valid = True
else:
log_message = ('Ingest data does not have all main keys')
log.log2warning(1000, log_message)
# Return
return valid
def connectivity():
"""Check connectivity to the database.
Args:
None
Returns:
valid: True if connectivity is OK
"""
# Initialize key variables
valid = False
# Do test
database = Database()
session = database.session()
try:
result = session.query(Agent.id_agent).filter(
and_(Agent.id_agent == '-1'.encode(), Agent.idx_agent == -1))
for _ in result:
break
valid = True
except Exception as e:
log_message = str(e)
log.log2warning(1053, log_message)
database.close()
# Return
return valid
def valid(self):
"""Validate main keys in filename.
Args:
None
Returns:
valid: Valid if True
"""
# Initialize key variables
valid = False
valid_list = [self._valid]
# Check keys
if len(valid_list) == valid_list.count(True):
valid_list.append(self._keys_in_filename())
# Return
if len(valid_list) == valid_list.count(True):
valid = True
else:
log_message = ('File %s failed validity testing.') % (
self.filepath)
log.log2warning(1001, log_message)
return valid
def stop(self):
"""Stop the daemon.
Args:
None
Returns:
"""
# Get the pid from the pidfile
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if not pid:
log_message = (
'PID file: %s does not exist. Daemon not running?'
'') % (self.pidfile)
log.log2warning(1063, log_message)
# Not an error in a restart
return
# Try killing the daemon process
try:
while 1:
# Sleep a while
time.sleep(0.3)
# Process lockfile state when trying to stop
if self.lockfile is None:
os.kill(pid, signal.SIGTERM)
else:
if os.path.exists(self.lockfile) is True:
continue
else:
os.kill(pid, signal.SIGTERM)
except OSError as err:
error = str(err.args)
if error.find("No such process") > 0:
self.delpid()
self.dellock()
else:
log_message = (str(err.args))
log_message = (
'%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1068, log_message)
except:
log_message = (
'Unknown daemon "stop" error for PID file: %s'
'') % (self.pidfile)
log.log2die(1066, log_message)
# Log success
self.delpid()
self.dellock()
log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
log.log2info(1071, log_message)
def post(self, save=True, data=None):
"""Post data to central server.
Args:
save: When True, save data to cache directory if postinf fails
data: Data to post. If None, then uses self.data
Returns:
success: "True: if successful
"""
# Initialize key variables
success = False
response = False
timestamp = self.data['timestamp']
id_agent = self.data['id_agent']
# Create data to post
if data is None:
data = self.data
# Post data save to cache if this fails
try:
result = requests.post(self.url, json=data)
response = True
except:
if save is True:
# Create a unique very long filename to reduce risk of
devicehash = general.hashstring(self.data['devicename'], sha=1)
filename = ('%s/%s_%s_%s.json') % (self.cache_dir, timestamp,
id_agent, devicehash)
# Save data
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
# Define success
if response is True:
if result.status_code == 200:
success = True
# Log message
if success is True:
log_message = ('Agent "%s" successfully contacted server %s'
'') % (self.name(), self.url)
log.log2info(1027, log_message)
else:
log_message = ('Agent "%s" failed to contact server %s'
'') % (self.name(), self.url)
log.log2warning(1028, log_message)
# Return
return success
def add_all(self, data_list, error_code, die=True):
"""Do a database modification.
Args:
data_list: List of sqlalchemy table objects
error_code: Error number to use if one occurs
die: Don't die if False, just return success
Returns:
success: True is successful
"""
# Initialize key variables
success = False
# Open database connection. Prepare cursor
session = self.session()
try:
# Update the database cache
session.add_all(data_list)
# Commit change
session.commit()
# disconnect from server
self.close()
# Update success
success = True
except Exception as exception_error:
success = False
session.rollback()
log_message = (
'Unable to modify database connection. '
'Error: \"%s\"') % (exception_error)
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warning(error_code, log_message)
except:
success = False
session.rollback()
log_message = ('Unexpected database exception')
if die is True:
log.log2die(error_code, log_message)
else:
log.log2warning(error_code, log_message)
# Return
return success
def valid(self):
"""Check if data keys are OK.
Args:
None
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
# Return if instantiation tests have failed
if self._valid is False:
valid = False
return valid
# Assign other values
timestamp = int(self.data['timestamp'])
id_agent = self.data['id_agent']
devicename = self.data['devicename']
# Check if there is a duplicate entry for this id_agent
if db_agent.id_agent_exists(id_agent) is not False:
idx_agent = db_agent.GetIDAgent(id_agent).idx_agent()
# Check if device exists
if db_device.devicename_exists(devicename) is True:
idx_device = db_device.GetDevice(devicename).idx_device()
# Check for device / agent entry existence
if db_deviceagent.device_agent_exists(
idx_device, idx_agent) is True:
# Check if this device / agent has been updated before
last_timesamp = db_deviceagent.GetDeviceAgent(
idx_device, idx_agent).last_timestamp()
# Validate
if timestamp <= last_timesamp:
log_message = (
'Data for id_agent %s, devicename %s '
'at timestamp %s '
'is already found in database.'
'') % (id_agent, devicename, timestamp)
log.log2warning(1113, log_message)
valid = False
# Return
return valid
def post(self, save=True, data=None):
"""Post data to central server.
Args:
save: When True, save data to cache directory if postinf fails
data: Data to post. If None, then uses self.data
Returns:
success: "True: if successful
"""
# Initialize key variables
success = False
timestamp = self.data['timestamp']
id_agent = self.data['id_agent']
# Create data to post
if data is None:
data = self.data
# Post data save to cache if this fails
uri = ('/receive/%s') % (id_agent)
success = self._api.post(uri, data)
# Log message
if success is True:
log_message = ('Agent "%s" successfully contacted server'
'') % (self.name())
log.log2info(1012, log_message)
else:
# Save data if requested
if save is True:
# Create a unique very long filename to reduce risk of
filename = ('%s/%s_%s.json') % (self.cache_dir, timestamp,
self.cache_suffix)
# Save data
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
# Log message
log_message = ('Agent "%s" failed to contact server'
'') % (self.name())
log.log2warning(1013, log_message)
# Return
return success
def __init__(self, data):
"""Method initializing the class.
Args:
data: Ingested data to validate
Returns:
None
"""
# Initialize key variables
self.data = data
self._valid = True
# Check if data is a dict
if isinstance(data, dict) is False:
log_message = ('Ingest data is not a dictionary')
log.log2warning(1093, log_message)
self._valid = False
def valid(self):
"""Master method that defines whether data is OK.
Args:
None
Returns:
all_ok:
"""
# Initialize key variables
valid_list = [self._valid]
ts_start = time.time()
# Check timeseries and timefixed data in the data
if len(valid_list) == valid_list.count(True):
check = _CheckData(self.information)
valid_list.append(check.valid())
# Check if data to be validated is already in the database
if len(valid_list) == valid_list.count(True):
check = _CheckDuplicates(self.information)
valid_list.append(check.valid())
# Do final check
if len(valid_list) == valid_list.count(True):
# Log success
ts_stop = time.time()
duration = ts_stop - ts_start
log_message = (
'Data validation of %s took %s seconds.'
'') % (self.filepath, round(duration, 4))
log.log2debug(1126, log_message)
all_ok = True
else:
# Log failure
log_message = ('Cache data in %s is invalid') % (self.filepath)
log.log2warning(1059, log_message)
all_ok = False
# Return
return all_ok
def _agent_label_keys_ok(self):
"""Check if agent label keys are OK.
Args:
None
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
data_types = ['timeseries', 'timefixed']
# Check major keys expected under each ageng label
if self._data_keys_ok() is False:
valid = False
return valid
# Check each data_type
for data_type in data_types:
# Skip if key is not present
if data_type not in self.data:
continue
# Process next major key
# The "_" in this case is the agent label in the dict
for _, agent_items in sorted(
self.data[data_type].items()):
# Process keys in data reported by agents
for key in ['base_type', 'description', 'data']:
if key not in agent_items:
log_message = (
'"%s" data type does not contain a "%s" key.'
'') % (data_type, key)
log.log2warning(1115, log_message)
valid = False
return valid
# Process data
if 'data' in agent_items:
for datapoint in agent_items['data']:
if len(datapoint) != 3:
log_message = (
'"%s" data type does not contain valid '
'datapoints in it\'s "data" key.'
'') % (data_type)
log.log2warning(1114, log_message)
valid = False
else:
# If there is no data, then it must be invalid
log_message = (
'Ingest data has no "data" label '
'values for database.')
log.log2warning(1004, log_message)
valid = False
return valid
# Return
return valid
def valid(self):
"""Validate Data.
Args:
None
Returns:
valid: Valid if True
"""
# Initialize key variables
valid = False
valid_list = [self._valid]
# All other tests need to pass for _timeseries_data_ok to pass
valid_list.append(self._timeseries_data_ok())
# Return
if len(valid_list) == valid_list.count(True):
valid = True
else:
log_message = ('Failed validity testing. %s') % (valid_list)
log.log2warning(1002, log_message)
return valid
def _keys_in_filename(self):
"""Validate main keys contained in the file are in the filename.
Args:
None
Returns:
valid: Valid if True
"""
# Initialize key variables
valid = True
# Get timestamp and id_agent from filename
filename = os.path.basename(self.filepath)
(name, _) = filename.split('.')
(tstamp, id_agent, _) = name.split('_')
timestamp = int(tstamp)
# Double check that the id_agent and timestamp in the
# filename matches that in the file.
# Ignore invalid files as a safety measure.
# Don't try to delete. They could be owned by some
# one else and the daemon could crash
if id_agent != self.data['id_agent']:
log_message = (
'id_agent %s in file %s does not match '
'id_agent %s in filename.'
'') % (
self.data['id_agent'],
id_agent, self.filepath)
log.log2warning(1123, log_message)
valid = False
# Check timestamp
if timestamp != self.data['timestamp']:
log_message = (
'Timestamp %s in file %s does not match timestamp '
'%s in filename.'
'') % (
self.data['timestamp'],
timestamp, self.filepath)
log.log2warning(1111, log_message)
valid = False
# Check timestamp validity
if general.validate_timestamp(timestamp) is False:
log_message = (
'Timestamp %s in file %s is not normalized'
'') % (self.data['timestamp'], self.filepath)
log.log2warning(1112, log_message)
valid = False
# Return
return valid
def process(ingester_agent_name):
"""Process cache data by adding it to the database using subprocesses.
Args:
ingester_agent_name: Ingester agent name
Returns:
None
"""
# Initialize key variables
argument_list = []
id_agent_metadata = defaultdict(lambda: defaultdict(dict))
# Configuration setup
config = configuration.Config()
configured_pool_size = config.ingest_pool_size()
# Make sure we have database connectivity
if db.connectivity() is False:
log_message = ('No connectivity to database. Check if running. '
'Check database authentication parameters.'
'')
log.log2warning(1053, log_message)
return
# Get meta data on files
id_agent_metadata = validate_cache_files()
# Spawn processes only if we have files to process
if bool(id_agent_metadata.keys()) is True:
# Process lock file
lockfile = daemon.lock_file(ingester_agent_name)
if os.path.exists(lockfile) is True:
# Return if lock file is present
log_message = (
'Ingest lock file %s exists. Multiple ingest daemons running '
'or lots of cache files to ingest. Ingester may have died '
'catastrophically in the past, in which case the lockfile '
'should be deleted. Exiting ingest process. '
'Will try again later.'
'') % (lockfile)
log.log2warning(1069, log_message)
return
else:
# Create lockfile
open(lockfile, 'a').close()
# Read each cache file
for devicehash in id_agent_metadata.keys():
for id_agent in id_agent_metadata[devicehash].keys():
# Create a list of arguments to process
argument_list.append(
(config, id_agent_metadata[devicehash][id_agent],
ingester_agent_name))
# Create a pool of sub process resources
pool_size = int(min(configured_pool_size, len(id_agent_metadata)))
with Pool(processes=pool_size) as pool:
# Create sub processes from the pool
pool.map(_wrapper_process, argument_list)
# Wait for all the processes to end
# pool.join()
# Return if lock file is present
if os.path.exists(lockfile) is True:
os.remove(lockfile)
def process(self):
"""Update the database using threads."""
# Initialize key variables
do_update = False
success = None
ingests = []
agent_data = {
'devicename': None,
'id_agent': None,
'sources': [],
'timeseries': [],
'timefixed': []
}
# Get the directory to which failed files will be moved
failure_directory = self.config.ingest_failures_directory()
# Initialize other values
max_timestamp = 0
# Get start time for activity
start_ts = time.time()
# Process file for each timestamp, starting from the oldes file
for data_dict in self.metadata:
# Initialize key variables
timestamp = data_dict['timestamp']
filepath = data_dict['filepath']
# Read in data
ingest = drain.Drain(filepath)
# Make sure file is OK
# Move it to a directory for further analysis
# by administrators
if ingest.valid() is False:
log_message = ('Cache ingest file %s is invalid. Moving.'
'') % (filepath)
log.log2warning(1054, log_message)
shutil.copy(filepath, failure_directory)
os.remove(filepath)
continue
# Append data
agent_data['timeseries'].extend(ingest.timeseries())
agent_data['timefixed'].extend(ingest.timefixed())
agent_data['sources'].extend(ingest.sources())
# Append ingest object to a list for later processing
ingests.append(ingest)
# Get the max timestamp
max_timestamp = max(timestamp, max_timestamp)
# Update information that doesn't change
if do_update is False:
agent_data['devicename'] = ingest.devicename()
agent_data['id_agent'] = ingest.id_agent()
agent_data['agent_name'] = ingest.agent()
# Get the PID file for the agent
pid_file = daemon.pid_file(self.ingester_agent_name)
# Update the PID file for the agent to ensure agentd.py
# doesn't kill the ingest while processing a long stream
# of files. If we are running this using __main__ = process()
# then the pid file wouldn't have been created, hence the logic.
if os.path.isfile(pid_file) is True:
daemon.update_pid(self.ingester_agent_name)
# Update update flag
do_update = True
# Process the rest
if do_update is True:
# Update remaining agent data
agent_data['max_timestamp'] = max_timestamp
# Add datapoints to the database
db_prepare = _PrepareDatabase(agent_data)
db_prepare.add_datapoints()
# Get the latest datapoints
datapoints = db_prepare.get_datapoints()
# Get the assigned index values for the device and agent
idx_device = db_prepare.idx_device()
idx_agent = db_prepare.idx_agent()
# Update database with data
db_update = _UpdateDB(agent_data, datapoints)
success = db_update.update()
# Update database table timestamps
update_timestamps = _UpdateLastTimestamp(idx_device, idx_agent,
max_timestamp)
update_timestamps.agent()
update_timestamps.deviceagent()
update_timestamps.datapoint()
# Purge source files. Only done after complete
# success of database updates. If not we could lose data in the
# event of an ingester crash. Ingester would re-read the files
# and process the non-duplicates, while deleting the duplicates.
for ingest in ingests:
ingest.purge()
# Log duration of activity
duration = time.time() - start_ts
if success is True:
log_message = (
'Agent %s was processed from %s cache files in %s '
'seconds (%s seconds/file, %s seconds/datapoint)'
'') % (agent_data['id_agent'], len(ingests),
round(duration, 4), round(
duration / len(ingests),
4), round(duration / len(datapoints), 6))
log.log2info(1007, log_message)
else:
log_message = (
'Failed to process all cache files for agent %s. '
'Investigate.') % (agent_data['id_agent'])
log.log2info(1008, log_message)
def process(self):
"""Update the database using threads."""
# Initialize key variables
do_update = False
success = None
ingests = []
agent_data = {
'devicename': None,
'id_agent': None,
'sources': [],
'timeseries': [],
'timefixed': [],
'max_timestamp': 0
}
# Get the directory to which failed files will be moved
failure_directory = self.config.ingest_failures_directory()
# Get start time for activity
start_ts = time.time()
# Process file for each timestamp, starting from the oldes file
for data_dict in self.metadata:
# Initialize key variables
timestamp = data_dict['timestamp']
filepath = data_dict['filepath']
# Read in data
ingest = drain.Drain(filepath)
# Make sure file is OK
# Move it to a directory for further analysis
# by administrators
if ingest.valid() is False:
log_message = ('Cache ingest file %s is invalid. Moving.'
'') % (filepath)
log.log2warning(1054, log_message)
shutil.copy(filepath, failure_directory)
os.remove(filepath)
continue
# Append data
agent_data['timeseries'].extend(ingest.timeseries())
agent_data['timefixed'].extend(ingest.timefixed())
agent_data['sources'].extend(ingest.sources())
# Append ingest object to a list for later processing
ingests.append(ingest)
# Update information that doesn't change
if do_update is False:
agent_data['devicename'] = ingest.devicename()
agent_data['id_agent'] = ingest.id_agent()
agent_data['agent_name'] = ingest.agent()
# Get the PID file for the agent
pid_file = daemon.pid_file(self.ingester_agent_name)
else:
# Get the max timestamp
agent_data['max_timestamp'] = max(timestamp,
agent_data['max_timestamp'])
# Update the PID file for the agent to ensure agentd.py
# doesn't kill the ingest while processing a long stream
# of files. If we are running this using __main__ = process()
# then the pid file wouldn't have been created, hence the logic.
if os.path.isfile(pid_file) is True:
daemon.update_pid(self.ingester_agent_name)
# Update update flag
do_update = True
# Process the rest
if do_update is True:
# Upadate and note success
(success,
datapoints_processed) = self._do_update(agent_data, ingests)
# Log duration of activity
duration = time.time() - start_ts
if success is True:
log_message = (
'Agent %s was processed from %s cache files in %s '
'seconds (%s seconds/file, %s seconds/datapoint)'
'') % (agent_data['id_agent'], len(ingests),
round(duration, 4), round(
duration / len(ingests),
4), round(duration / datapoints_processed, 6))
log.log2info(1007, log_message)
else:
log_message = (
'Failed to process all cache files for agent %s. '
'Investigate.') % (agent_data['id_agent'])
log.log2info(1008, log_message)
def __init__(self, filename):
"""Method initializing the class.
Args:
filename: Cache filename
Returns:
None
"""
# Initialize key variables
self.filename = filename
self._information = defaultdict(lambda: defaultdict(dict))
self._sources = []
self.validated = False
self.agent_meta = {}
data_types = ['timeseries', 'timefixed']
# Ingest data
validator = validate.ValidateCache(filename)
information = validator.getinfo()
# Log if data is bad
if information is False:
log_message = ('Cache ingest file %s is invalid.') % (filename)
log.log2warning(1051, log_message)
return
else:
self.validated = True
# Process validated data
if self.validated is True:
# Get main keys
self.agent_meta = _main_keys(information)
timestamp = self.agent_meta['timestamp']
id_agent = self.agent_meta['id_agent']
# Process timeseries data
for data_type in data_types:
# Skip if data type isn't in the data
if data_type not in information:
continue
# Process the data type
for agent_label, label_dict in sorted(
information[data_type].items()):
# Get universal parameters for label_dict
base_type = _base_type(label_dict['base_type'])
description = label_dict['description']
# Create a key in the data based on the base_type
if base_type not in self._information[data_type]:
self._information[data_type][base_type] = []
# Process the data associated with the agent_label
for datapoint in label_dict['data']:
# Create a unique, unchangeable id_datapoint for data
index = datapoint[0]
value = datapoint[1]
source = datapoint[2]
id_datapoint = _id_datapoint(
id_agent, agent_label, index,
self.agent_meta['agent'],
self.agent_meta['devicename'])
# Convert values to float if this is
# data that could be charted
if base_type is not None:
value = float(value)
# Update the data
self._information[data_type][base_type].append({
'id_agent':
id_agent,
'id_datapoint':
id_datapoint,
'value':
value,
'timestamp':
timestamp
})
# Update sources after fixing encoding
self._sources.append({
'id_agent': id_agent,
'id_datapoint': id_datapoint,
'agent_label': agent_label,
'agent_source': source,
'description': description,
'base_type': base_type
})
def _timeseries_data_ok(self):
"""Check if timeseries data is OK.
Args:
None
Returns:
valid: True if valid
"""
# Initialize key variables
valid = True
data_type = 'timeseries'
# Check major keys expected under each ageng label
if self._agent_label_keys_ok() is False:
valid = False
return valid
# Check for timeseries data
if data_type in self.data:
# Process the data type
for _, reported_data in sorted(self.data[data_type].items()):
# Make sure the base types are numeric
if 'base_type' in reported_data:
try:
float(reported_data['base_type'])
except:
log_message = (
'TimeSeries "base_type" key is non numeric.')
log.log2warning(1120, log_message)
valid = False
return valid
else:
log_message = ('TimeSeries data has no "base_type" key.')
log.log2warning(1117, log_message)
valid = False
return valid
# Process data
if 'data' in reported_data:
for datapoint in reported_data['data']:
# Check to make sure value is numeric
value = datapoint[1]
try:
float(value)
except:
log_message = (
'TimeSeries data has non numeric data values.')
log.log2warning(1119, log_message)
valid = False
return valid
else:
log_message = ('TimeSeries data has no "data" key.')
log.log2warning(1118, log_message)
valid = False
return valid
# Return
return valid