def insert_agent_host(): """Insert first agent and host in the database. Args: None Returns: None """ # Initialize key variables idx_agent = 1 idx_host = 1 agent_name = '_infoset' config = jm_configuration.ConfigAgent(agent_name) # Add agent if db_agent.idx_exists(idx_agent) is False: # Generate a UID and add a record in the database uid = agent.get_uid(config) # Add host if db_host.idx_exists(idx_host) is False: record = Host( description=jm_general.encode('Infoset Server'), hostname=jm_general.encode(socket.getfqdn())) database = db.Database() database.add(record, 1106) # Add to Agent / Host table if db_hostagent.host_agent_exists(idx_host, idx_agent) is False: record = HostAgent(idx_host=idx_host, idx_agent=idx_agent) database = db.Database() database.add(record, 1107)
def _insert_datapoint(metadata, idx_agent, idx_host): """Insert new datapoint into database. Args: metadata: Tuple of datapoint metadata. (uid, did, label, source, description) uid: Agent UID did: Datapoint ID label: Datapoint label created by agent source: Source of the data (subsystem being tracked) description: Description provided by agent config file (unused) base_type = SNMP base type (Counter32, Counter64, Gauge etc.) idx_agent: Index of agent in the Agent db table idx_host: Index of host in the Host db table Returns: None """ # Initialize key variables (_, did, agent_label, agent_source, _, base_type) = metadata # Insert record record = Datapoint( id=jm_general.encode(did), idx_agent=idx_agent, idx_host=idx_host, agent_label=jm_general.encode(agent_label), agent_source=jm_general.encode(agent_source), base_type=base_type) database = db.Database() database.add(record, 1082)
def insert_config(): """Insert first config in the database. Args: None Returns: None """ # Initialize key variables key_values = [('version', '0.0.0.0')] # Cycle through all the key value pairs for item in key_values: key = item[0] value = item[1] # Check if value exists and insert if not if db_configuration.config_key_exists(key) is False: record = Configuration( config_key=jm_general.encode(key), config_value=jm_general.encode(value)) database = db.Database() database.add(record, 1108)
def get_uid(config): """Create a permanent UID for the agent. Args: config: ConfigAgent configuration object Returns: uid: UID for agent """ # Initialize key variables agent_name = config.agent_name() # The way the UID is stored depends on the existence of # a database if config.store_uid_in_database() is False: # Initialize key variables filez = hidden.File() dirz = hidden.Directory() uid_dir = dirz.uid() filename = filez.uid(agent_name) # Create UID directory if not yet created if os.path.exists(uid_dir) is False: os.makedirs(uid_dir) # Read environment file with UID if it exists if os.path.isfile(filename): with open(filename) as f_handle: uid = f_handle.readline() else: # Create a UID and save uid = _generate_uid() with open(filename, 'w+') as env: env.write(str(uid)) else: if db_agent.unique_agent_exists(agent_name) is True: agent = db_agent.GetName(agent_name) uid = agent.uid() else: # Add record uid = _generate_uid() record = AgentDB( id=jm_general.encode(uid), name=jm_general.encode(agent_name)) database = db.Database() database.add(record, 1109) # Return return uid
def _insert_host(self): """Insert new agent into database. Args: None Returns: None """ # Initialize key variables uid = self.ingest.uid() hostname = self.ingest.hostname() # Update Host table if dhost.hostname_exists(hostname) is False: # Add to Host table record = Host(hostname=jm_general.encode(hostname)) database = db.Database() database.add(record, 1080) # Get idx of host host_info = dhost.GetHost(hostname) idx_host = host_info.idx() # Get idx of agent uid_info = agent.GetUID(uid) idx_agent = uid_info.idx() # Update HostAgent table if hagent.host_agent_exists(idx_host, idx_agent) is False: # Add to HostAgent table record = HostAgent(idx_host=idx_host, idx_agent=idx_agent) database = db.Database() database.add(record, 1038)
def insert_oids(directory=None): """Update the database with certain key data. Args: directory: Directory to add to list Returns: None """ # Initialize key variables root_dir = jm_general.root_directory() oids_directories = [('%s/infoset/metadata/oids') % (root_dir)] # Create a list of existing agent labels, that are unique by definition agent_labels = [] all_oids = db_oid.all_oids() for item in all_oids: agent_labels.append(item['agent_label']) # Add directory to the search path if required if directory is not None: if os.path.isdir(directory) is True: oids_directories.extend(directory) # Read in the oid data oids_yaml = jm_general.read_yaml_files(oids_directories) # Get a list of all labels for item in oids_yaml: oid_values = item['oid_values'] oid_labels = item['oid_labels'] agent_label = item['agent_label'] base_type = item['base_type'] multiplier = item['multiplier'] if db_oid.oid_values_exists(oid_values) is False: if agent_label not in agent_labels: # Prepare SQL query to read a record from the database. record = OID( oid_values=jm_general.encode(oid_values), oid_labels=jm_general.encode(oid_labels), agent_label=jm_general.encode(agent_label), base_type=base_type, multiplier=multiplier) database = db.Database() database.add(record, 1091)
def insert_billtype(): """Insert first billtype in the database. Args: None Returns: None """ # Initialize key variables if db_billtype.idx_exists(1) is False: record = BillType( code=jm_general.encode('_SYSTEM_RESERVED_'), name=jm_general.encode('_SYSTEM_RESERVED_')) database = db.Database() database.add(record, 1104)
def insert_department(): """Insert first department in the database. Args: None Returns: None """ # Initialize key variables if db_department.idx_exists(1) is False: record = Department( code=jm_general.encode('_SYSTEM_RESERVED_'), name=jm_general.encode('_SYSTEM_RESERVED_')) database = db.Database() database.add(record, 1102)
def insert_agent_host(): """Insert first agent and host in the database. Args: None Returns: None """ # Initialize key variables idx_agent = 1 idx_host = 1 agent_name = '_infoset' # Add agent if db_agent.idx_exists(idx_agent) is False: record = Agent( id=jm_general.encode('_SYSTEM_RESERVED_'), name=jm_general.encode(agent_name)) database = db.Database() database.add(record, 1109) # Generate a UID uid = agent.get_uid(agent_name) database = db.Database() session = database.session() record = session.query(Agent).filter(Agent.idx == idx_agent).one() record.id = jm_general.encode(uid) database.commit(session, 1073) # Add host if db_host.idx_exists(idx_host) is False: record = Host( description=jm_general.encode('Infoset Server'), hostname=jm_general.encode(socket.getfqdn())) database = db.Database() database.add(record, 1106) # Add to Agent / Host table if db_hostagent.host_agent_exists(idx_host, idx_agent) is False: record = HostAgent(idx_host=idx_host, idx_agent=idx_agent) database = db.Database() database.add(record, 1107)
def _insert_agent(self): """Insert new agent into database. Args: None Returns: None """ # Initialize key variables uid = self.ingest.uid() agent_name = self.ingest.agent() # Return if agent already exists in the table if agent.uid_exists(uid) is True: return # Prepare SQL query to read a record from the database. record = Agent( id=jm_general.encode(uid), name=jm_general.encode(agent_name)) database = db.Database() database.add(record, 1081)
def _update_agent_last_update(uid, last_timestamp): """Insert new datapoint into database. Args: uid: UID of agent last_timestamp: The last time a DID for the agent was updated in the database Returns: None """ # Initialize key variables value = jm_general.encode(uid) # Update the database database = db.Database() session = database.session() record = session.query(Agent).filter(Agent.id == value).one() record.last_timestamp = last_timestamp database.commit(session, 1055)
def _update_unchartable(self, mapping): """Update unchartable data into the database "iset_datapoint" table. Args: mapping: Map of DIDs to database row index values Returns: None """ # Initialize key variables data = self.ingest.other() data_list = [] timestamp_tracker = {} # Update data for item in data: # Process each datapoint item found (_, did, value, timestamp) = item idx_datapoint = int(mapping[did][0]) last_timestamp = int(mapping[did][2]) # Only update with data collected after # the most recent update. Don't do anything more if timestamp > last_timestamp: data_list.append( (idx_datapoint, value) ) # Update DID's last updated timestamp if idx_datapoint in timestamp_tracker: timestamp_tracker[idx_datapoint] = max( timestamp, timestamp_tracker[idx_datapoint]) else: timestamp_tracker[idx_datapoint] = timestamp # Update if there is data if bool(data_list) is True: for item in data_list: (idx_datapoint, value) = item # Update database database = db.Database() session = database.session() record = session.query(Datapoint).filter( Datapoint.idx == idx_datapoint).one() record.uncharted_value = jm_general.encode(value) database.commit(session, 1037) # Change the last updated timestamp for idx_datapoint, last_timestamp in timestamp_tracker.items(): # Update database database = db.Database() session = database.session() record = session.query(Datapoint).filter( Datapoint.idx == idx_datapoint).one() record.last_timestamp = last_timestamp database.commit(session, 1083) # Report success log_message = ( 'Successful cache drain (Uncharted Data) ' 'for UID %s at timestamp %s') % ( self.ingest.uid(), self.ingest.timestamp()) log.log2quiet(1045, log_message)
def main(): """Process agent data. Args: None Returns: None """ # Process Cache agent_name = 'snmp' # Process CLI cli_args = cli() # Get configuration config = jm_configuration.ConfigAgent(agent_name) # Get hosts if cli_args.hostname.lower() == 'all': hostnames = config.agent_hostnames() else: # Check to make sure hostname exists in the configuration all_hosts = config.agent_hostnames() cli_hostname = cli_args.hostname if cli_hostname not in all_hosts: log_message = ( 'Agent "%s": Hostname %s is not present ' 'in the the configuration file.' '') % (agent_name, cli_hostname) log.log2die(1103, log_message) else: hostnames = [cli_hostname] # Get OIDs oids = db_oid.all_oids() # Process each hostname for hostname in hostnames: # Get SNMP information snmp_config = jm_configuration.ConfigSNMP() validate = snmp_manager.Validate(hostname, snmp_config.snmp_auth()) snmp_params = validate.credentials() # Check SNMP supported if bool(snmp_params) is True: snmp_object = snmp_manager.Interact(snmp_params) # Check support for each OID for item in oids: # Get oid oid = item['oid_values'] # Actions must be taken if a valid OID is found if snmp_object.oid_exists(oid) is True: # Insert into iset_hostoid table if necessary if db_host.hostname_exists(hostname) is False: record = Host( hostname=jm_general.encode(hostname), snmp_enabled=1) database = db.Database() database.add(record, 1089) # Get idx for host and oid idx_host = db_host.GetHost(hostname).idx() idx_oid = item['idx'] # Insert an entry in the HostOID table if required if db_hostoid.host_oid_exists(idx_host, idx_oid) is False: # Prepare SQL query to read a record from the database. record = HostOID(idx_host=idx_host, idx_oid=idx_oid) database = db.Database() database.add(record, 1090)
def _update_unchartable(self, mapping): """Update unchartable data into the database "iset_datapoint" table. Args: mapping: Map of DIDs to database row index values Returns: None """ # Initialize key variables data = self.agent_data['unchartable'] data_list = [] timestamp_tracker = {} # Update data for item in data: # Process each datapoint item found (_, did, value, timestamp) = item idx_datapoint = int(mapping[did][0]) last_timestamp = int(mapping[did][2]) # Only update with data collected after # the most recent update. Don't do anything more if timestamp > last_timestamp: data_list.append( (idx_datapoint, value) ) # Update DID's last updated timestamp if idx_datapoint in timestamp_tracker: timestamp_tracker[idx_datapoint] = max( timestamp, timestamp_tracker[idx_datapoint]) else: timestamp_tracker[idx_datapoint] = timestamp # Update if there is data if bool(data_list) is True: # Create a database session # NOTE: We only do a single commit on the session. # This is much faster (20x based on testing) than # instantiating the database, updating records, and committing # after every iteration of the "for loop" database = db.Database() session = database.session() # Update uncharted data for item in data_list: (idx_datapoint, value) = item data_dict = {'uncharted_value': jm_general.encode(value)} session.query(Datapoint).filter( Datapoint.idx == idx_datapoint).update(data_dict) # Change the last updated timestamp for idx_datapoint, last_timestamp in timestamp_tracker.items(): data_dict = {'last_timestamp': last_timestamp} session.query(Datapoint).filter( Datapoint.idx == idx_datapoint).update(data_dict) # Commit data database.commit(session, 1128) # Report success log_message = ( 'Successful cache drain (Uncharted Data) ' 'for UID %s') % (self.agent_data['uid']) log.log2quiet(1045, log_message)