def insert_agent_host():
"""Insert first agent and host in the database.
Args:
None
Returns:
None
"""
# Initialize key variables
idx_agent = 1
idx_host = 1
agent_name = '_infoset'
config = jm_configuration.ConfigAgent(agent_name)
# Add agent
if db_agent.idx_exists(idx_agent) is False:
# Generate a UID and add a record in the database
uid = agent.get_uid(config)
# Add host
if db_host.idx_exists(idx_host) is False:
record = Host(
description=jm_general.encode('Infoset Server'),
hostname=jm_general.encode(socket.getfqdn()))
database = db.Database()
database.add(record, 1106)
# Add to Agent / Host table
if db_hostagent.host_agent_exists(idx_host, idx_agent) is False:
record = HostAgent(idx_host=idx_host, idx_agent=idx_agent)
database = db.Database()
database.add(record, 1107)
def _insert_datapoint(metadata, idx_agent, idx_host):
"""Insert new datapoint into database.
Args:
metadata: Tuple of datapoint metadata.
(uid, did, label, source, description)
uid: Agent UID
did: Datapoint ID
label: Datapoint label created by agent
source: Source of the data (subsystem being tracked)
description: Description provided by agent config file (unused)
base_type = SNMP base type (Counter32, Counter64, Gauge etc.)
idx_agent: Index of agent in the Agent db table
idx_host: Index of host in the Host db table
Returns:
None
"""
# Initialize key variables
(_, did, agent_label, agent_source, _, base_type) = metadata
# Insert record
record = Datapoint(
id=jm_general.encode(did),
idx_agent=idx_agent,
idx_host=idx_host,
agent_label=jm_general.encode(agent_label),
agent_source=jm_general.encode(agent_source),
base_type=base_type)
database = db.Database()
database.add(record, 1082)
def insert_config():
"""Insert first config in the database.
Args:
None
Returns:
None
"""
# Initialize key variables
key_values = [('version', '0.0.0.0')]
# Cycle through all the key value pairs
for item in key_values:
key = item[0]
value = item[1]
# Check if value exists and insert if not
if db_configuration.config_key_exists(key) is False:
record = Configuration(
config_key=jm_general.encode(key),
config_value=jm_general.encode(value))
database = db.Database()
database.add(record, 1108)
def get_uid(config):
"""Create a permanent UID for the agent.
Args:
config: ConfigAgent configuration object
Returns:
uid: UID for agent
"""
# Initialize key variables
agent_name = config.agent_name()
# The way the UID is stored depends on the existence of
# a database
if config.store_uid_in_database() is False:
# Initialize key variables
filez = hidden.File()
dirz = hidden.Directory()
uid_dir = dirz.uid()
filename = filez.uid(agent_name)
# Create UID directory if not yet created
if os.path.exists(uid_dir) is False:
os.makedirs(uid_dir)
# Read environment file with UID if it exists
if os.path.isfile(filename):
with open(filename) as f_handle:
uid = f_handle.readline()
else:
# Create a UID and save
uid = _generate_uid()
with open(filename, 'w+') as env:
env.write(str(uid))
else:
if db_agent.unique_agent_exists(agent_name) is True:
agent = db_agent.GetName(agent_name)
uid = agent.uid()
else:
# Add record
uid = _generate_uid()
record = AgentDB(
id=jm_general.encode(uid),
name=jm_general.encode(agent_name))
database = db.Database()
database.add(record, 1109)
# Return
return uid
def _insert_host(self):
"""Insert new agent into database.
Args:
None
Returns:
None
"""
# Initialize key variables
uid = self.ingest.uid()
hostname = self.ingest.hostname()
# Update Host table
if dhost.hostname_exists(hostname) is False:
# Add to Host table
record = Host(hostname=jm_general.encode(hostname))
database = db.Database()
database.add(record, 1080)
# Get idx of host
host_info = dhost.GetHost(hostname)
idx_host = host_info.idx()
# Get idx of agent
uid_info = agent.GetUID(uid)
idx_agent = uid_info.idx()
# Update HostAgent table
if hagent.host_agent_exists(idx_host, idx_agent) is False:
# Add to HostAgent table
record = HostAgent(idx_host=idx_host, idx_agent=idx_agent)
database = db.Database()
database.add(record, 1038)
def insert_oids(directory=None):
"""Update the database with certain key data.
Args:
directory: Directory to add to list
Returns:
None
"""
# Initialize key variables
root_dir = jm_general.root_directory()
oids_directories = [('%s/infoset/metadata/oids') % (root_dir)]
# Create a list of existing agent labels, that are unique by definition
agent_labels = []
all_oids = db_oid.all_oids()
for item in all_oids:
agent_labels.append(item['agent_label'])
# Add directory to the search path if required
if directory is not None:
if os.path.isdir(directory) is True:
oids_directories.extend(directory)
# Read in the oid data
oids_yaml = jm_general.read_yaml_files(oids_directories)
# Get a list of all labels
for item in oids_yaml:
oid_values = item['oid_values']
oid_labels = item['oid_labels']
agent_label = item['agent_label']
base_type = item['base_type']
multiplier = item['multiplier']
if db_oid.oid_values_exists(oid_values) is False:
if agent_label not in agent_labels:
# Prepare SQL query to read a record from the database.
record = OID(
oid_values=jm_general.encode(oid_values),
oid_labels=jm_general.encode(oid_labels),
agent_label=jm_general.encode(agent_label),
base_type=base_type,
multiplier=multiplier)
database = db.Database()
database.add(record, 1091)
def insert_billtype():
"""Insert first billtype in the database.
Args:
None
Returns:
None
"""
# Initialize key variables
if db_billtype.idx_exists(1) is False:
record = BillType(
code=jm_general.encode('_SYSTEM_RESERVED_'),
name=jm_general.encode('_SYSTEM_RESERVED_'))
database = db.Database()
database.add(record, 1104)
def insert_department():
"""Insert first department in the database.
Args:
None
Returns:
None
"""
# Initialize key variables
if db_department.idx_exists(1) is False:
record = Department(
code=jm_general.encode('_SYSTEM_RESERVED_'),
name=jm_general.encode('_SYSTEM_RESERVED_'))
database = db.Database()
database.add(record, 1102)
def insert_agent_host():
"""Insert first agent and host in the database.
Args:
None
Returns:
None
"""
# Initialize key variables
idx_agent = 1
idx_host = 1
agent_name = '_infoset'
# Add agent
if db_agent.idx_exists(idx_agent) is False:
record = Agent(
id=jm_general.encode('_SYSTEM_RESERVED_'),
name=jm_general.encode(agent_name))
database = db.Database()
database.add(record, 1109)
# Generate a UID
uid = agent.get_uid(agent_name)
database = db.Database()
session = database.session()
record = session.query(Agent).filter(Agent.idx == idx_agent).one()
record.id = jm_general.encode(uid)
database.commit(session, 1073)
# Add host
if db_host.idx_exists(idx_host) is False:
record = Host(
description=jm_general.encode('Infoset Server'),
hostname=jm_general.encode(socket.getfqdn()))
database = db.Database()
database.add(record, 1106)
# Add to Agent / Host table
if db_hostagent.host_agent_exists(idx_host, idx_agent) is False:
record = HostAgent(idx_host=idx_host, idx_agent=idx_agent)
database = db.Database()
database.add(record, 1107)
def _insert_agent(self):
"""Insert new agent into database.
Args:
None
Returns:
None
"""
# Initialize key variables
uid = self.ingest.uid()
agent_name = self.ingest.agent()
# Return if agent already exists in the table
if agent.uid_exists(uid) is True:
return
# Prepare SQL query to read a record from the database.
record = Agent(
id=jm_general.encode(uid),
name=jm_general.encode(agent_name))
database = db.Database()
database.add(record, 1081)
def _update_agent_last_update(uid, last_timestamp):
"""Insert new datapoint into database.
Args:
uid: UID of agent
last_timestamp: The last time a DID for the agent was updated
in the database
Returns:
None
"""
# Initialize key variables
value = jm_general.encode(uid)
# Update the database
database = db.Database()
session = database.session()
record = session.query(Agent).filter(Agent.id == value).one()
record.last_timestamp = last_timestamp
database.commit(session, 1055)
def _update_unchartable(self, mapping):
"""Update unchartable data into the database "iset_datapoint" table.
Args:
mapping: Map of DIDs to database row index values
Returns:
None
"""
# Initialize key variables
data = self.ingest.other()
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, value, timestamp) = item
idx_datapoint = int(mapping[did][0])
last_timestamp = int(mapping[did][2])
# Only update with data collected after
# the most recent update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
(idx_datapoint, value)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
for item in data_list:
(idx_datapoint, value) = item
# Update database
database = db.Database()
session = database.session()
record = session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).one()
record.uncharted_value = jm_general.encode(value)
database.commit(session, 1037)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
# Update database
database = db.Database()
session = database.session()
record = session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).one()
record.last_timestamp = last_timestamp
database.commit(session, 1083)
# Report success
log_message = (
'Successful cache drain (Uncharted Data) '
'for UID %s at timestamp %s') % (
self.ingest.uid(), self.ingest.timestamp())
log.log2quiet(1045, log_message)
def main():
"""Process agent data.
Args:
None
Returns:
None
"""
# Process Cache
agent_name = 'snmp'
# Process CLI
cli_args = cli()
# Get configuration
config = jm_configuration.ConfigAgent(agent_name)
# Get hosts
if cli_args.hostname.lower() == 'all':
hostnames = config.agent_hostnames()
else:
# Check to make sure hostname exists in the configuration
all_hosts = config.agent_hostnames()
cli_hostname = cli_args.hostname
if cli_hostname not in all_hosts:
log_message = (
'Agent "%s": Hostname %s is not present '
'in the the configuration file.'
'') % (agent_name, cli_hostname)
log.log2die(1103, log_message)
else:
hostnames = [cli_hostname]
# Get OIDs
oids = db_oid.all_oids()
# Process each hostname
for hostname in hostnames:
# Get SNMP information
snmp_config = jm_configuration.ConfigSNMP()
validate = snmp_manager.Validate(hostname, snmp_config.snmp_auth())
snmp_params = validate.credentials()
# Check SNMP supported
if bool(snmp_params) is True:
snmp_object = snmp_manager.Interact(snmp_params)
# Check support for each OID
for item in oids:
# Get oid
oid = item['oid_values']
# Actions must be taken if a valid OID is found
if snmp_object.oid_exists(oid) is True:
# Insert into iset_hostoid table if necessary
if db_host.hostname_exists(hostname) is False:
record = Host(
hostname=jm_general.encode(hostname),
snmp_enabled=1)
database = db.Database()
database.add(record, 1089)
# Get idx for host and oid
idx_host = db_host.GetHost(hostname).idx()
idx_oid = item['idx']
# Insert an entry in the HostOID table if required
if db_hostoid.host_oid_exists(idx_host, idx_oid) is False:
# Prepare SQL query to read a record from the database.
record = HostOID(idx_host=idx_host, idx_oid=idx_oid)
database = db.Database()
database.add(record, 1090)
def _update_unchartable(self, mapping):
"""Update unchartable data into the database "iset_datapoint" table.
Args:
mapping: Map of DIDs to database row index values
Returns:
None
"""
# Initialize key variables
data = self.agent_data['unchartable']
data_list = []
timestamp_tracker = {}
# Update data
for item in data:
# Process each datapoint item found
(_, did, value, timestamp) = item
idx_datapoint = int(mapping[did][0])
last_timestamp = int(mapping[did][2])
# Only update with data collected after
# the most recent update. Don't do anything more
if timestamp > last_timestamp:
data_list.append(
(idx_datapoint, value)
)
# Update DID's last updated timestamp
if idx_datapoint in timestamp_tracker:
timestamp_tracker[idx_datapoint] = max(
timestamp, timestamp_tracker[idx_datapoint])
else:
timestamp_tracker[idx_datapoint] = timestamp
# Update if there is data
if bool(data_list) is True:
# Create a database session
# NOTE: We only do a single commit on the session.
# This is much faster (20x based on testing) than
# instantiating the database, updating records, and committing
# after every iteration of the "for loop"
database = db.Database()
session = database.session()
# Update uncharted data
for item in data_list:
(idx_datapoint, value) = item
data_dict = {'uncharted_value': jm_general.encode(value)}
session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).update(data_dict)
# Change the last updated timestamp
for idx_datapoint, last_timestamp in timestamp_tracker.items():
data_dict = {'last_timestamp': last_timestamp}
session.query(Datapoint).filter(
Datapoint.idx == idx_datapoint).update(data_dict)
# Commit data
database.commit(session, 1128)
# Report success
log_message = (
'Successful cache drain (Uncharted Data) '
'for UID %s') % (self.agent_data['uid'])
log.log2quiet(1045, log_message)
