def checkout(dbapi_connection, connection_record, connection_proxy):
"""Checkout sub-processes connection for sub-processing if needed.
Checkout is called when a connection is retrieved from the Pool.
Args:
dbapi_connection: Connection object
connection_record: Connection record object
connection_proxy: Connection proxy object
Returns:
None
"""
# Get PID of main process
pid = os.getpid()
# Detect if this is a sub-process
if connection_record.info['pid'] != pid:
# substitute log.debug() or similar here as desired
log_message = (
'Parent process %s forked (%s) with an open '
'database connection, '
'which is being discarded and recreated.'
'') % (
connection_record.info['pid'], pid)
log.log2quiet(1067, log_message)
connection_record.connection = connection_proxy.connection = None
raise exc.DisconnectionError(
"Connection record belongs to pid %s, "
"attempting to check out in pid %s" %
(connection_record.info['pid'], pid)
)
def start(self):
"""Start the daemon.
Args:
None
Returns:
"""
# Check for a pidfile to see if the daemon already runs
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if pid:
log_message = (
'PID file: %s already exists. Daemon already running?'
'') % (self.pidfile)
log.log2die(1062, log_message)
# Start the daemon
self.daemonize()
# Log success
log_message = ('Daemon Started - PID file: %s') % (self.pidfile)
log.log2quiet(1070, log_message)
# Run code for daemon
self.run()
def purge(self):
"""Purge cache file that was read.
Args:
None
Returns:
success: "True" if successful
"""
# Initialize key variables
success = True
try:
os.remove(self.filename)
except:
success = False
# Report success
if success is True:
log_message = (
'Ingest cache file %s deleted') % (self.filename)
log.log2quiet(1046, log_message)
else:
log_message = (
'Failed to delete ingest cache file %s') % (self.filename)
log.log2warn(1087, log_message)
# Return
return success
def _check_when_disabled(agent_name):
"""Stop agent.
Args:
agent_filepath: Filepath of agent to be restarted.
agent_name: Agent name
Returns:
None
"""
# Initialize key variables
agentconfig = jm_configuration.ConfigAgent(agent_name)
agent_filename = agentconfig.agent_filename()
# Get agent status variables
root_dir = jm_general.root_directory()
agent_filepath = ('%s/%s') % (root_dir, agent_filename)
pid = hidden.File()
pidfile = pid.pid(agent_name)
# Shutdown agent if running
if os.path.isfile(pidfile) is True:
with open(pidfile, 'r') as f_handle:
pidvalue = int(f_handle.readline().strip())
if psutil.pid_exists(pidvalue) is True:
log_message = (
'Agent "%s" is alive, but should be disabled. '
'Attempting to stop.'
'') % (agent_name)
log.log2quiet(1032, log_message)
_stop(agent_filepath, agent_name)
def stop(self):
"""Stop the daemon.
Args:
None
Returns:
"""
# Get the pid from the pidfile
try:
with open(self.pidfile, 'r') as pf_handle:
pid = int(pf_handle.read().strip())
except IOError:
pid = None
if not pid:
log_message = (
'PID file: %s does not exist. Daemon not running?'
'') % (self.pidfile)
log.log2warn(1063, log_message)
# Not an error in a restart
return
# Try killing the daemon process
try:
while 1:
if self.lockfile is None:
os.kill(pid, signal.SIGTERM)
else:
time.sleep(0.3)
if os.path.exists(self.lockfile) is True:
continue
else:
os.kill(pid, signal.SIGTERM)
time.sleep(0.3)
except OSError as err:
error = str(err.args)
if error.find("No such process") > 0:
self.delpid()
self.dellock()
else:
log_message = (str(err.args))
log_message = (
'%s - PID file: %s') % (log_message, self.pidfile)
log.log2die(1068, log_message)
except:
log_message = (
'Unknown daemon "stop" error for PID file: %s'
'') % (self.pidfile)
log.log2die(1066, log_message)
# Log success
self.delpid()
self.dellock()
log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile)
log.log2quiet(1071, log_message)
def api_make(config, host, verbose=False):
"""Process 'pagemaker' CLI option.
Args:
config: Configuration object
verbose: Verbose output if True
Returns:
None
"""
# Initialize key variables
threads_in_pool = 1
device_file_found = False
# Create directory if needed
perm_dir = config.web_directory()
temp_dir = tempfile.mkdtemp()
# Delete all files in temporary directory
jm_general.delete_files(temp_dir)
# Skip if device file not found
if os.path.isfile(config.snmp_device_file(host)) is False:
log_message = (
'No YAML device file for host %s found in %s. '
'Run toolbox.py with the "poll" option first.'
'') % (host, config.snmp_directory())
log.log2quiet(1018, log_message)
else:
device_file_found = True
####################################################################
#
# Define variables that will be required for the database update
# We have to initialize the dict during every loop to prevent
# data corruption
#
####################################################################
data_dict = {}
data_dict['host'] = host
data_dict['config'] = config
data_dict['verbose'] = verbose
data_dict['temp_dir'] = temp_dir
table = HTMLTable(config, host)
# Create HTML output
html = ('%s%s\n%s\n\n%s\n') % (
_html_header(host), host, table.device(),
table.ethernet())
# Do the rest if device_file_found
if device_file_found is True:
# Wait on the queue until everything has been processed
return html
def post(self, save=True, data=None):
"""Post data to central server.
Args:
save: When True, save data to cache directory if postinf fails
data: Data to post. If None, then uses self.data
Returns:
success: "True: if successful
"""
# Initialize key variables
success = False
response = False
timestamp = self.data['timestamp']
uid = self.data['uid']
# Create data to post
if data is None:
data = self.data
# Post data save to cache if this fails
try:
result = requests.post(self.url, json=data)
response = True
except:
if save is True:
# Create a unique very long filename to reduce risk of
hosthash = jm_general.hashstring(self.data['hostname'], sha=1)
filename = ('%s/%s_%s_%s.json') % (
self.cache_dir, timestamp, uid, hosthash)
# Save data
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
# Define success
if response is True:
if result.status_code == 200:
success = True
# Log message
if success is True:
log_message = (
'Agent "%s" successfully contacted server %s'
'') % (self.name(), self.url)
log.log2quiet(1027, log_message)
else:
log_message = (
'Agent "%s" failed to contact server %s'
'') % (self.name(), self.url)
log.log2warn(1028, log_message)
# Return
return success
def purge(self):
"""Purge data from cache by posting to central server.
Args:
None
Returns:
success: "True: if successful
"""
# Initialize key variables
uid = self.data["uid"]
# Add files in cache directory to list
filenames = [
filename
for filename in os.listdir(self.cache_dir)
if os.path.isfile(os.path.join(self.cache_dir, filename))
]
# Read cache file
for filename in filenames:
# Only post files for our own UID value
if uid not in filename:
continue
# Get the full filepath for the cache file and post
filepath = os.path.join(self.cache_dir, filename)
with open(filepath, "r") as f_handle:
try:
data = json.load(f_handle)
except:
# Log removal
log_message = (
"Error reading previously cached agent data file %s " "for agent %s. May be corrupted." ""
) % (filepath, self.name())
log.log2die(1064, log_message)
# Post file
success = self.post(save=False, data=data)
# Delete file if successful
if success is True:
os.remove(filepath)
# Log removal
log_message = ("Purging cache file %s after successfully " "contacting server %s" "") % (
filepath,
self.url,
)
log.log2quiet(1029, log_message)
def create(config, host):
"""Create topology page for host.
Args:
config: Configuration object
host: Hostname to create pages for
Returns:
None
"""
# Initialize key variables
device_file_found = False
# Create directory if needed
temp_dir = tempfile.mkdtemp()
# Delete all files in temporary directory
jm_general.delete_files(temp_dir)
# Skip if device file not found
if os.path.isfile(config.topology_device_file(host)) is False:
log_message = (
'No YAML device file for host %s found in %s. '
'topoloy agent has not discovered it yet.'
'') % (host, config.topology_directory())
log.log2quiet(1018, log_message)
else:
device_file_found = True
####################################################################
#
# Define variables that will be required for the database update
# We have to initialize the dict during every loop to prevent
# data corruption
#
####################################################################
table = HTMLTable(config, host)
# Create HTML output
html = ('%s%s\n%s\n\n%s\n') % (
_html_header(host), host, table.device(),
table.ethernet())
# Do the rest if device_file_found
if device_file_found is True:
# Wait on the queue until everything has been processed
return html
def run(self):
"""Update the database using threads."""
while True:
# Get the data_dict
data_dict = self.queue.get()
host = data_dict['host']
config = data_dict['config']
verbose = data_dict['verbose']
temp_dir = data_dict['temp_dir']
# Show host information
validate = snmp_manager.Validate(host, config.snmp_auth())
snmp_params = validate.credentials()
snmp_object = snmp_manager.Interact(snmp_params)
# Verbose output
if verbose is True:
output = ('Processing on: host %s') % (host)
print(output)
# Skip invalid, and uncontactable hosts
if bool(snmp_params) is False:
if verbose is True:
log_message = (
'Uncontactable host %s or no valid SNMP '
'credentials found for it.') % (host)
log.log2quiet(1019, log_message)
continue
# Process if valid
if bool(snmp_params) is True:
# Get data
status = snmp_info.Query(snmp_object)
data = status.everything()
yaml_string = jm_general.dict2yaml(data)
# Dump data
temp_file = ('%s/%s.yaml') % (temp_dir, host)
with open(temp_file, 'w') as file_handle:
file_handle.write(yaml_string)
# Verbose output
if verbose is True:
output = ('Completed run: host %s') % (host)
print(output)
# Signals to queue job is done
self.queue.task_done()
def _stop(agent_filepath, agent_name):
"""Stop agent.
Args:
agent_filepath: Filepath of agent to be restarted.
agent_name: Agent name
Returns:
None
"""
# Restart
log_message = (
'Stopping agent "%s".'
'') % (agent_name)
log.log2quiet(1033, log_message)
command2run = ('%s --stop --force') % (agent_filepath)
_execute(command2run)
def _start(agent_filepath, agent_name):
"""Start agent.
Args:
agent_filepath: Filepath of agent to be started.
agent_name: Agent name
Returns:
None
"""
# Start
log_message = (
'Starting agent "%s".'
'') % (agent_name)
log.log2quiet(1077, log_message)
command2run = ('%s --start') % (agent_filepath)
_execute(command2run)
def query(self):
"""Query all remote hosts for data.
Args:
None
Returns:
None
"""
# Check SNMP supported
if bool(self.snmp_params) is True:
# Get datapoints
self._create_yaml()
else:
log_message = (
'Uncontactable host %s or no valid SNMP '
'credentials found for it.') % (self.hostname)
log.log2quiet(1021, log_message)
def _create_yaml(self):
"""Create the master dictionary for the host.
Args:
None
Returns:
value: Index value
"""
# Initialize key variables
temp_dir = tempfile.mkdtemp()
temp_file = ('%s/%s.yaml') % (temp_dir, self.hostname)
perm_file = self.server_config.topology_device_file(self.hostname)
# Get data
log_message = (
'Querying topology data from host %s.'
'') % (self.hostname)
log.log2quiet(1125, log_message)
# Create YAML file by polling device
status = snmp_info.Query(self.snmp_object)
data = status.everything()
yaml_string = jm_general.dict2yaml(data)
# Dump data
with open(temp_file, 'w') as file_handle:
file_handle.write(yaml_string)
# Get data
log_message = (
'Completed topology query from host %s.'
'') % (self.hostname)
log.log2quiet(1019, log_message)
# Clean up files
os.rename(temp_file, perm_file)
os.rmdir(temp_dir)
def query(self):
"""Query all remote hosts for data.
Args:
None
Returns:
None
"""
# Initialize key variables
if self.config.agent_port():
data = self.config.agent_port()
port = int(data)
else:
port = 5001
# Do stuff
log_message = (
'Starting agent %s on localhost port %s.'
'') % (self.agent_name, port)
log.log2quiet(1088, log_message)
APP.run(host='0.0.0.0', port=port)
def _check_when_enabled(agent_name):
"""Stop agent.
Args:
agent_filepath: Filepath of agent to be restarted.
agent_name: Agent name
Returns:
None
"""
# Initialize key variables
agentconfig = jm_configuration.ConfigAgent(agent_name)
agent_filename = agentconfig.agent_filename()
# Get agent status variables
root_dir = jm_general.root_directory()
agent_filepath = ('%s/%s') % (root_dir, agent_filename)
pid = hidden.File()
pidfile = pid.pid(agent_name)
lockfile = pid.lock(agent_name)
# Ignore agents that cannot be found
if os.path.isfile(agent_filepath) is False:
log_message = (
'Agent executable file %s listed in the '
'configuration file '
'of agent "%s" does not exist. Please fix.'
'') % (agent_filepath, agent_name)
log.log2quiet(1075, log_message)
return
# Check for pid file
if os.path.isfile(pidfile) is True:
with open(pidfile, 'r') as f_handle:
pidvalue = int(f_handle.readline().strip())
# Check if service died catastrophically. No PID file
if psutil.pid_exists(pidvalue) is False:
log_message = (
'Agent "%s" is dead. Attempting to restart.'
'') % (agent_name)
log.log2quiet(1041, log_message)
# Remove PID file and restart
os.remove(pidfile)
_restart(agent_filepath, agent_name)
else:
# Check if agent hung without updating the PID
if agentconfig.monitor_agent_pid() is True:
try:
mtime = os.path.getmtime(pidfile)
except OSError:
mtime = 0
if mtime < int(time.time()) - (60 * 10):
log_message = (
'Agent "%s" is hung. Attempting to restart.'
'') % (agent_name)
log.log2quiet(1076, log_message)
_restart(agent_filepath, agent_name)
else:
if os.path.isfile(lockfile) is True:
_restart(agent_filepath, agent_name)
else:
_start(agent_filepath, agent_name)
def _update_unchartable(mapping, ingest, config):
"""Update unchartable data into the database "iset_datapoint" table.
Args:
mapping: Map of DIDs to database row index values
ingest: Drain object
config: Config object
Returns:
None
"""
# Initialize key variables
data = ingest.other()
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, tuple_value, timestamp) = item
idx_datapoint = int(mapping[did][0])
last_timestamp = int(mapping[did][2])
value = ('%s') % (tuple_value)
# Only update with data collected after
# the most recent update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
(idx_datapoint, value)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
for item in data_list:
(idx_datapoint, value) = item
fixed_value = str(value)[0:128]
# Prepare SQL query to read a record from the database.
sql_modify = (
'UPDATE iset_datapoint set uncharted_value="%s" WHERE '
'idx=%s') % (fixed_value, idx_datapoint)
# Do query and get results
database = db.Database(config)
database.modify(sql_modify, 1037)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
# Prepare SQL query to read a record from the database.
sql_modify = (
'UPDATE iset_datapoint SET last_timestamp=%s '
'WHERE iset_datapoint.idx=%s'
'') % (last_timestamp, idx_datapoint)
database.modify(sql_modify, 1044)
# Report success
log_message = (
'Successful cache drain (Uncharted Data) '
'for UID %s at timestamp %s') % (
ingest.uid(), ingest.timestamp())
log.log2quiet(1045, log_message)
def _update_unchartable(self, mapping):
"""Update unchartable data into the database "iset_datapoint" table.
Args:
mapping: Map of DIDs to database row index values
Returns:
None
"""
# Initialize key variables
data = self.agent_data['unchartable']
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, value, timestamp) = item
idx_datapoint = int(mapping[did][0])
last_timestamp = int(mapping[did][2])
# Only update with data collected after
# the most recent update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
(idx_datapoint, value)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
# Create a database session
# NOTE: We only do a single commit on the session.
# This is much faster (20x based on testing) than
# instantiating the database, updating records, and committing
# after every iteration of the "for loop"
database = db.Database()
session = database.session()
# Update uncharted data
for item in data_list:
(idx_datapoint, value) = item
data_dict = {'uncharted_value': jm_general.encode(value)}
session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).update(data_dict)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
data_dict = {'last_timestamp': last_timestamp}
session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).update(data_dict)
# Commit data
database.commit(session, 1128)
# Report success
log_message = (
'Successful cache drain (Uncharted Data) '
'for UID %s') % (self.agent_data['uid'])
log.log2quiet(1045, log_message)
def make(config, verbose=False):
"""Process 'pagemaker' CLI option.
Args:
config: Configuration object
verbose: Verbose output if True
Returns:
None
"""
# Initialize key variables
threads_in_pool = 10
device_file_found = False
# Create directory if needed
perm_dir = config.web_directory()
temp_dir = tempfile.mkdtemp()
# Delete all files in temporary directory
jm_general.delete_files(temp_dir)
# Spawn a pool of threads, and pass them queue instance
for _ in range(threads_in_pool):
update_thread = PageMaker(THREAD_QUEUE)
update_thread.daemon = True
update_thread.start()
# Get host data and write to file
for host in config.hosts():
# Skip if device file not found
if os.path.isfile(config.snmp_device_file(host)) is False:
log_message = (
'No YAML device file for host %s found in %s. '
'Run toolbox.py with the "poll" option first.'
'') % (host, config.snmp_directory())
log.log2quiet(1018, log_message)
continue
else:
device_file_found = True
####################################################################
#
# Define variables that will be required for the database update
# We have to initialize the dict during every loop to prevent
# data corruption
#
####################################################################
data_dict = {}
data_dict['host'] = host
data_dict['config'] = config
data_dict['verbose'] = verbose
data_dict['temp_dir'] = temp_dir
THREAD_QUEUE.put(data_dict)
# Do the rest if device_file_found
if device_file_found is True:
# Wait on the queue until everything has been processed
THREAD_QUEUE.join()
# PYTHON BUG. Join can occur while threads are still shutting down.
# This can create spurious "Exception in thread (most likely raised
# during interpreter shutdown)" errors.
# The "time.sleep(1)" adds a delay to make sure things really terminate
# properly. This seems to be an issue on virtual machines in Dev only
time.sleep(1)
# Create index file
write_file = ('%s/index.html') % (temp_dir)
index_html = _index_html(config)
with open(write_file, 'w') as file_handle:
file_handle.write(index_html)
# Cleanup, move temporary files to clean permanent directory.
# Delete temporary directory
if os.path.isdir(perm_dir):
jm_general.delete_files(perm_dir)
else:
os.makedirs(perm_dir, 0o755)
jm_general.move_files(temp_dir, perm_dir)
# Clean up
os.rmdir(temp_dir)
def process(self):
"""Update the database using threads."""
# Initialize key variables
updated = False
hostnames = []
uids = []
ingests = []
agent_names = []
agent_data = {
'hostname': None,
'uid': None,
'sources': [],
'chartable': [],
'unchartable': []
}
# Get the data_dict
metadata = self.metadata
config = self.config
# Initialize other values
max_timestamp = 0
# Get start time for activity
start_ts = time.time()
# Sort metadata by timestamp
metadata.sort()
# Process file for each timestamp, starting from the oldes file
for (timestamp, filepath) in metadata:
# Read in data
ingest = drain.Drain(filepath)
# Make sure file is OK
# Move it to a directory for further analysis
# by administrators
if ingest.valid() is False:
log_message = (
'Cache ingest file %s is invalid. Moving.'
'') % (filepath)
log.log2warn(1054, log_message)
shutil.copy(
filepath, config.ingest_failures_directory())
os.remove(filepath)
continue
# Append data
agent_data['chartable'].extend(ingest.chartable())
agent_data['unchartable'].extend(ingest.other())
agent_data['sources'].extend(ingest.sources())
hostnames.append(ingest.hostname())
uids.append(ingest.uid())
ingests.append(ingest)
agent_names.append(ingest.agent())
# Purge source file
# ingest.purge()
# Get the max timestamp
max_timestamp = max(timestamp, max_timestamp)
# Update update flag
updated = True
# Verify that we have only processed data for the same hostname
# UID and agent name
if (jm_general.all_same(hostnames) is False) or (
jm_general.all_same(uids) is False) or (
jm_general.all_same(agent_names) is False):
log_message = (
'Cache ingest files error for hostname %s,'
'agent name %s, UID %s.'
'') % (hostnames[0], agent_names[0], uids[0])
log.log2quiet(1083, log_message)
# Process the rest
if updated is True:
# Update remaining agent data
agent_data['hostname'] = hostnames[0]
agent_data['uid'] = uids[0]
agent_data['agent_name'] = agent_names[0]
# Update database
dbase = UpdateDB(agent_data)
dbase.update()
# Update the last time the agent was contacted
_update_agent_last_update(agent_data['uid'], max_timestamp)
# Update the host / agent table timestamp if
# hostname was processed
_host_agent_last_update(
agent_data['hostname'], agent_data['uid'], max_timestamp)
# Purge source files. Only done after complete
# success of database updates. If not we could lose data in the
# event of an ingester crash. Ingester would re-read the files
# and process the non-duplicates, while deleting the duplicates.
for ingest in ingests:
ingest.purge()
# Log duration of activity
duration = time.time() - start_ts
log_message = (
'UID %s was processed in %s seconds.'
'') % (agent_data['uid'], duration)
log.log2quiet(1127, log_message)
def valid(self):
"""Master method that defines whether data is OK.
Args:
None
Returns:
all_ok:
"""
# Initialize key variables
validity = []
ts_start = time.time()
# Check primary keys
validity.append(
_check_primary_keys_exist(self.information))
# Check timestamp key
if False not in validity:
validity.append(
_check_timestamp_key(self.information))
# Check validity of primary keys in file
if False not in validity:
validity.append(
self._check_primary_keys_in_file())
# Check chartable and unchartable data in the data
if False not in validity:
validity.append(_check_reported_data(self.information))
if False not in validity:
validity.append(_check_chartable_data(self.information))
# Check if data to be validated is already in the database
if False not in validity:
validity.append(_check_duplicates(self.information))
# Do final check
if False in validity:
# Log failure
if self.filepath is None:
mid_string = ''
else:
mid_string = ('in %s') % (self.filepath)
log_message = ('Cache data %s is invalid') % (mid_string)
log.log2warn(1059, log_message)
all_ok = False
else:
# Log success
ts_stop = time.time()
duration = ts_stop - ts_start
if self.filepath is None:
mid_string = ''
else:
mid_string = ('of %s') % (self.filepath)
log_message = (
'Data validation %s took %s seconds.'
'') % (mid_string, round(duration, 4))
log.log2quiet(1126, log_message)
all_ok = True
# Return
return all_ok
def _update_chartable(mapping, ingest, config):
"""Insert data into the database "iset_data" table.
Args:
mapping: Map of DIDs to database row index values
ingest: Drain object
config: Config object
Returns:
None
"""
# Initialize key variables
data = ingest.chartable()
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, tuple_value, timestamp) = item
idx_datapoint = int(mapping[did][0])
idx_agent = int(mapping[did][1])
last_timestamp = int(mapping[did][2])
value = float(tuple_value)
# Only update with data collected after
# the most recent update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
(idx_datapoint, idx_agent, value, timestamp)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
# Prepare SQL query to read a record from the database.
sql_insert = (
'REPLACE INTO iset_data '
'(idx_datapoint, idx_agent, value, timestamp) VALUES '
'(%s, %s, %s, %s)')
# Do query and get results
database = db.Database(config)
database.modify(sql_insert, 1056, data_list=data_list)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
# Prepare SQL query to read a record from the database.
sql_modify = (
'UPDATE iset_datapoint SET last_timestamp=%s '
'WHERE iset_datapoint.idx=%s'
'') % (last_timestamp, idx_datapoint)
database.modify(sql_modify, 1057)
# Report success
log_message = (
'Successful cache drain for UID %s at timestamp %s') % (
ingest.uid(), ingest.timestamp())
log.log2quiet(1058, log_message)
def _update_unchartable(self, mapping):
"""Update unchartable data into the database "iset_datapoint" table.
Args:
mapping: Map of DIDs to database row index values
Returns:
None
"""
# Initialize key variables
data = self.ingest.other()
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, value, timestamp) = item
idx_datapoint = int(mapping[did][0])
last_timestamp = int(mapping[did][2])
# Only update with data collected after
# the most recent update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
(idx_datapoint, value)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
for item in data_list:
(idx_datapoint, value) = item
# Update database
database = db.Database()
session = database.session()
record = session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).one()
record.uncharted_value = jm_general.encode(value)
database.commit(session, 1037)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
# Update database
database = db.Database()
session = database.session()
record = session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).one()
record.last_timestamp = last_timestamp
database.commit(session, 1083)
# Report success
log_message = (
'Successful cache drain (Uncharted Data) '
'for UID %s at timestamp %s') % (
self.ingest.uid(), self.ingest.timestamp())
log.log2quiet(1045, log_message)
def _update_chartable(self, mapping):
"""Insert data into the database "iset_data" table.
Args:
mapping: Map of DIDs to database row index values
Returns:
None
"""
# Initialize key variables
data = self.ingest.chartable()
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, string_value, timestamp) = item
idx_datapoint = int(mapping[did][0])
last_timestamp = int(mapping[did][2])
value = float(string_value)
# Only update with data collected after
# the most recent DID update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
Data(
idx_datapoint=idx_datapoint,
value=value,
timestamp=timestamp
)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
# Do performance data update
database = db.Database()
database.add_all(data_list, 1056)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
# Update the database
database = db.Database()
session = database.session()
record = session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).one()
record.last_timestamp = last_timestamp
database.commit(session, 1057)
# Report success
log_message = (
'Successful cache drain for UID %s at timestamp %s') % (
self.ingest.uid(), self.ingest.timestamp())
log.log2quiet(1058, log_message)
