def __init__(self, filepath): """Method initializing the class. Args: filepath: Cache filename Returns: None """ # Initialize key variables self.filepath = filepath self.data = None self._valid = False name_ok = _valid_filename(filepath) # Read data from file if name_ok is True: self.data = _read_data_from_file(filepath) else: # Log status log_message = ('File %s does has incorrect filename format.' '') % (filepath) log.log2warning(1026, log_message) # Check main keys in data. contents = _CheckMainKeys(self.data) if contents.valid() is True: if name_ok is True: self._valid = True
def __init__(self, data): do_update = False success = None agent_data = { 'devicename': None, 'id_agent': None, 'sources': [], 'timeseries': [], 'timefixed': [], 'max_timestamp': 0 } ingest = drain.Drain(filename=None, data=data) if ingest.valid() is False: log_message = ( 'Cache data at timestamp %s is invalid. Moving.' '') % (ingest.timestamp()) log.log2warning(1054, log_message) else: agent_data['timeseries'] = ingest.timeseries() agent_data['timefixed'] = ingest.timefixed() agent_data['sources'] = ingest.sources() agent_data['devicename'] = ingest.devicename() agent_data['id_agent'] = ingest.id_agent() agent_data['agent_name'] = ingest.agent() agent_data['max_timestamp'] = ingest.timestamp() # Upadate and note success (success, datapoints_processed) = self._do_update( agent_data)
def _read_data_from_file(filepath): """Provide validated information when valid. Args: filepath: Path to file Returns: data: Data """ # Initialize key variables data = {} # Ingest data try: with open(filepath, 'r') as f_handle: data = json.load(f_handle) except: # Log status log_message = ('File %s does not contain JSON data, does not exist, ' 'or is unreadable.') % (filepath) log.log2warning(1006, log_message) # Return return data
def _data_keys_ok(self): """Check if data keys are OK. Args: None Returns: valid: True if valid """ # Initialize key variables valid = False count = 0 data_types = ['timeseries', 'timefixed'] # Process data if self._valid is True: for data_type in data_types: # Skip if data type isn't in the data if data_type in self.data: count += 1 # Verify validity if count > 0: valid = True # Log error if valid is False: log_message = ('Ingest data does not contain all data keys.') log.log2warning(1003, log_message) # Return return valid
def valid(self): """Validate main keys. Args: None Returns: valid: Valid if True """ # Initialize key variables valid = False valid_list = [self._valid] valid_list.append(self._timestamp()) valid_list.append(self._agent()) valid_list.append(self._id_agent()) valid_list.append(self._devicename()) # Define validity if len(valid_list) == valid_list.count(True): valid = True else: log_message = ('Ingest data does not have all main keys') log.log2warning(1000, log_message) # Return return valid
def connectivity(): """Check connectivity to the database. Args: None Returns: valid: True if connectivity is OK """ # Initialize key variables valid = False # Do test database = Database() session = database.session() try: result = session.query(Agent.id_agent).filter( and_(Agent.id_agent == '-1'.encode(), Agent.idx_agent == -1)) for _ in result: break valid = True except Exception as e: log_message = str(e) log.log2warning(1053, log_message) database.close() # Return return valid
def valid(self): """Validate main keys in filename. Args: None Returns: valid: Valid if True """ # Initialize key variables valid = False valid_list = [self._valid] # Check keys if len(valid_list) == valid_list.count(True): valid_list.append(self._keys_in_filename()) # Return if len(valid_list) == valid_list.count(True): valid = True else: log_message = ('File %s failed validity testing.') % ( self.filepath) log.log2warning(1001, log_message) return valid
def stop(self): """Stop the daemon. Args: None Returns: """ # Get the pid from the pidfile try: with open(self.pidfile, 'r') as pf_handle: pid = int(pf_handle.read().strip()) except IOError: pid = None if not pid: log_message = ( 'PID file: %s does not exist. Daemon not running?' '') % (self.pidfile) log.log2warning(1063, log_message) # Not an error in a restart return # Try killing the daemon process try: while 1: # Sleep a while time.sleep(0.3) # Process lockfile state when trying to stop if self.lockfile is None: os.kill(pid, signal.SIGTERM) else: if os.path.exists(self.lockfile) is True: continue else: os.kill(pid, signal.SIGTERM) except OSError as err: error = str(err.args) if error.find("No such process") > 0: self.delpid() self.dellock() else: log_message = (str(err.args)) log_message = ( '%s - PID file: %s') % (log_message, self.pidfile) log.log2die(1068, log_message) except: log_message = ( 'Unknown daemon "stop" error for PID file: %s' '') % (self.pidfile) log.log2die(1066, log_message) # Log success self.delpid() self.dellock() log_message = ('Daemon Stopped - PID file: %s') % (self.pidfile) log.log2info(1071, log_message)
def post(self, save=True, data=None): """Post data to central server. Args: save: When True, save data to cache directory if postinf fails data: Data to post. If None, then uses self.data Returns: success: "True: if successful """ # Initialize key variables success = False response = False timestamp = self.data['timestamp'] id_agent = self.data['id_agent'] # Create data to post if data is None: data = self.data # Post data save to cache if this fails try: result = requests.post(self.url, json=data) response = True except: if save is True: # Create a unique very long filename to reduce risk of devicehash = general.hashstring(self.data['devicename'], sha=1) filename = ('%s/%s_%s_%s.json') % (self.cache_dir, timestamp, id_agent, devicehash) # Save data with open(filename, 'w') as f_handle: json.dump(data, f_handle) # Define success if response is True: if result.status_code == 200: success = True # Log message if success is True: log_message = ('Agent "%s" successfully contacted server %s' '') % (self.name(), self.url) log.log2info(1027, log_message) else: log_message = ('Agent "%s" failed to contact server %s' '') % (self.name(), self.url) log.log2warning(1028, log_message) # Return return success
def add_all(self, data_list, error_code, die=True): """Do a database modification. Args: data_list: List of sqlalchemy table objects error_code: Error number to use if one occurs die: Don't die if False, just return success Returns: success: True is successful """ # Initialize key variables success = False # Open database connection. Prepare cursor session = self.session() try: # Update the database cache session.add_all(data_list) # Commit change session.commit() # disconnect from server self.close() # Update success success = True except Exception as exception_error: success = False session.rollback() log_message = ( 'Unable to modify database connection. ' 'Error: \"%s\"') % (exception_error) if die is True: log.log2die(error_code, log_message) else: log.log2warning(error_code, log_message) except: success = False session.rollback() log_message = ('Unexpected database exception') if die is True: log.log2die(error_code, log_message) else: log.log2warning(error_code, log_message) # Return return success
def valid(self): """Check if data keys are OK. Args: None Returns: valid: True if valid """ # Initialize key variables valid = True # Return if instantiation tests have failed if self._valid is False: valid = False return valid # Assign other values timestamp = int(self.data['timestamp']) id_agent = self.data['id_agent'] devicename = self.data['devicename'] # Check if there is a duplicate entry for this id_agent if db_agent.id_agent_exists(id_agent) is not False: idx_agent = db_agent.GetIDAgent(id_agent).idx_agent() # Check if device exists if db_device.devicename_exists(devicename) is True: idx_device = db_device.GetDevice(devicename).idx_device() # Check for device / agent entry existence if db_deviceagent.device_agent_exists( idx_device, idx_agent) is True: # Check if this device / agent has been updated before last_timesamp = db_deviceagent.GetDeviceAgent( idx_device, idx_agent).last_timestamp() # Validate if timestamp <= last_timesamp: log_message = ( 'Data for id_agent %s, devicename %s ' 'at timestamp %s ' 'is already found in database.' '') % (id_agent, devicename, timestamp) log.log2warning(1113, log_message) valid = False # Return return valid
def post(self, save=True, data=None): """Post data to central server. Args: save: When True, save data to cache directory if postinf fails data: Data to post. If None, then uses self.data Returns: success: "True: if successful """ # Initialize key variables success = False timestamp = self.data['timestamp'] id_agent = self.data['id_agent'] # Create data to post if data is None: data = self.data # Post data save to cache if this fails uri = ('/receive/%s') % (id_agent) success = self._api.post(uri, data) # Log message if success is True: log_message = ('Agent "%s" successfully contacted server' '') % (self.name()) log.log2info(1012, log_message) else: # Save data if requested if save is True: # Create a unique very long filename to reduce risk of filename = ('%s/%s_%s.json') % (self.cache_dir, timestamp, self.cache_suffix) # Save data with open(filename, 'w') as f_handle: json.dump(data, f_handle) # Log message log_message = ('Agent "%s" failed to contact server' '') % (self.name()) log.log2warning(1013, log_message) # Return return success
def __init__(self, data): """Method initializing the class. Args: data: Ingested data to validate Returns: None """ # Initialize key variables self.data = data self._valid = True # Check if data is a dict if isinstance(data, dict) is False: log_message = ('Ingest data is not a dictionary') log.log2warning(1093, log_message) self._valid = False
def valid(self): """Master method that defines whether data is OK. Args: None Returns: all_ok: """ # Initialize key variables valid_list = [self._valid] ts_start = time.time() # Check timeseries and timefixed data in the data if len(valid_list) == valid_list.count(True): check = _CheckData(self.information) valid_list.append(check.valid()) # Check if data to be validated is already in the database if len(valid_list) == valid_list.count(True): check = _CheckDuplicates(self.information) valid_list.append(check.valid()) # Do final check if len(valid_list) == valid_list.count(True): # Log success ts_stop = time.time() duration = ts_stop - ts_start log_message = ( 'Data validation of %s took %s seconds.' '') % (self.filepath, round(duration, 4)) log.log2debug(1126, log_message) all_ok = True else: # Log failure log_message = ('Cache data in %s is invalid') % (self.filepath) log.log2warning(1059, log_message) all_ok = False # Return return all_ok
def _agent_label_keys_ok(self): """Check if agent label keys are OK. Args: None Returns: valid: True if valid """ # Initialize key variables valid = True data_types = ['timeseries', 'timefixed'] # Check major keys expected under each ageng label if self._data_keys_ok() is False: valid = False return valid # Check each data_type for data_type in data_types: # Skip if key is not present if data_type not in self.data: continue # Process next major key # The "_" in this case is the agent label in the dict for _, agent_items in sorted( self.data[data_type].items()): # Process keys in data reported by agents for key in ['base_type', 'description', 'data']: if key not in agent_items: log_message = ( '"%s" data type does not contain a "%s" key.' '') % (data_type, key) log.log2warning(1115, log_message) valid = False return valid # Process data if 'data' in agent_items: for datapoint in agent_items['data']: if len(datapoint) != 3: log_message = ( '"%s" data type does not contain valid ' 'datapoints in it\'s "data" key.' '') % (data_type) log.log2warning(1114, log_message) valid = False else: # If there is no data, then it must be invalid log_message = ( 'Ingest data has no "data" label ' 'values for database.') log.log2warning(1004, log_message) valid = False return valid # Return return valid
def valid(self): """Validate Data. Args: None Returns: valid: Valid if True """ # Initialize key variables valid = False valid_list = [self._valid] # All other tests need to pass for _timeseries_data_ok to pass valid_list.append(self._timeseries_data_ok()) # Return if len(valid_list) == valid_list.count(True): valid = True else: log_message = ('Failed validity testing. %s') % (valid_list) log.log2warning(1002, log_message) return valid
def _keys_in_filename(self): """Validate main keys contained in the file are in the filename. Args: None Returns: valid: Valid if True """ # Initialize key variables valid = True # Get timestamp and id_agent from filename filename = os.path.basename(self.filepath) (name, _) = filename.split('.') (tstamp, id_agent, _) = name.split('_') timestamp = int(tstamp) # Double check that the id_agent and timestamp in the # filename matches that in the file. # Ignore invalid files as a safety measure. # Don't try to delete. They could be owned by some # one else and the daemon could crash if id_agent != self.data['id_agent']: log_message = ( 'id_agent %s in file %s does not match ' 'id_agent %s in filename.' '') % ( self.data['id_agent'], id_agent, self.filepath) log.log2warning(1123, log_message) valid = False # Check timestamp if timestamp != self.data['timestamp']: log_message = ( 'Timestamp %s in file %s does not match timestamp ' '%s in filename.' '') % ( self.data['timestamp'], timestamp, self.filepath) log.log2warning(1111, log_message) valid = False # Check timestamp validity if general.validate_timestamp(timestamp) is False: log_message = ( 'Timestamp %s in file %s is not normalized' '') % (self.data['timestamp'], self.filepath) log.log2warning(1112, log_message) valid = False # Return return valid
def process(ingester_agent_name): """Process cache data by adding it to the database using subprocesses. Args: ingester_agent_name: Ingester agent name Returns: None """ # Initialize key variables argument_list = [] id_agent_metadata = defaultdict(lambda: defaultdict(dict)) # Configuration setup config = configuration.Config() configured_pool_size = config.ingest_pool_size() # Make sure we have database connectivity if db.connectivity() is False: log_message = ('No connectivity to database. Check if running. ' 'Check database authentication parameters.' '') log.log2warning(1053, log_message) return # Get meta data on files id_agent_metadata = validate_cache_files() # Spawn processes only if we have files to process if bool(id_agent_metadata.keys()) is True: # Process lock file lockfile = daemon.lock_file(ingester_agent_name) if os.path.exists(lockfile) is True: # Return if lock file is present log_message = ( 'Ingest lock file %s exists. Multiple ingest daemons running ' 'or lots of cache files to ingest. Ingester may have died ' 'catastrophically in the past, in which case the lockfile ' 'should be deleted. Exiting ingest process. ' 'Will try again later.' '') % (lockfile) log.log2warning(1069, log_message) return else: # Create lockfile open(lockfile, 'a').close() # Read each cache file for devicehash in id_agent_metadata.keys(): for id_agent in id_agent_metadata[devicehash].keys(): # Create a list of arguments to process argument_list.append( (config, id_agent_metadata[devicehash][id_agent], ingester_agent_name)) # Create a pool of sub process resources pool_size = int(min(configured_pool_size, len(id_agent_metadata))) with Pool(processes=pool_size) as pool: # Create sub processes from the pool pool.map(_wrapper_process, argument_list) # Wait for all the processes to end # pool.join() # Return if lock file is present if os.path.exists(lockfile) is True: os.remove(lockfile)
def process(self): """Update the database using threads.""" # Initialize key variables do_update = False success = None ingests = [] agent_data = { 'devicename': None, 'id_agent': None, 'sources': [], 'timeseries': [], 'timefixed': [] } # Get the directory to which failed files will be moved failure_directory = self.config.ingest_failures_directory() # Initialize other values max_timestamp = 0 # Get start time for activity start_ts = time.time() # Process file for each timestamp, starting from the oldes file for data_dict in self.metadata: # Initialize key variables timestamp = data_dict['timestamp'] filepath = data_dict['filepath'] # Read in data ingest = drain.Drain(filepath) # Make sure file is OK # Move it to a directory for further analysis # by administrators if ingest.valid() is False: log_message = ('Cache ingest file %s is invalid. Moving.' '') % (filepath) log.log2warning(1054, log_message) shutil.copy(filepath, failure_directory) os.remove(filepath) continue # Append data agent_data['timeseries'].extend(ingest.timeseries()) agent_data['timefixed'].extend(ingest.timefixed()) agent_data['sources'].extend(ingest.sources()) # Append ingest object to a list for later processing ingests.append(ingest) # Get the max timestamp max_timestamp = max(timestamp, max_timestamp) # Update information that doesn't change if do_update is False: agent_data['devicename'] = ingest.devicename() agent_data['id_agent'] = ingest.id_agent() agent_data['agent_name'] = ingest.agent() # Get the PID file for the agent pid_file = daemon.pid_file(self.ingester_agent_name) # Update the PID file for the agent to ensure agentd.py # doesn't kill the ingest while processing a long stream # of files. If we are running this using __main__ = process() # then the pid file wouldn't have been created, hence the logic. if os.path.isfile(pid_file) is True: daemon.update_pid(self.ingester_agent_name) # Update update flag do_update = True # Process the rest if do_update is True: # Update remaining agent data agent_data['max_timestamp'] = max_timestamp # Add datapoints to the database db_prepare = _PrepareDatabase(agent_data) db_prepare.add_datapoints() # Get the latest datapoints datapoints = db_prepare.get_datapoints() # Get the assigned index values for the device and agent idx_device = db_prepare.idx_device() idx_agent = db_prepare.idx_agent() # Update database with data db_update = _UpdateDB(agent_data, datapoints) success = db_update.update() # Update database table timestamps update_timestamps = _UpdateLastTimestamp(idx_device, idx_agent, max_timestamp) update_timestamps.agent() update_timestamps.deviceagent() update_timestamps.datapoint() # Purge source files. Only done after complete # success of database updates. If not we could lose data in the # event of an ingester crash. Ingester would re-read the files # and process the non-duplicates, while deleting the duplicates. for ingest in ingests: ingest.purge() # Log duration of activity duration = time.time() - start_ts if success is True: log_message = ( 'Agent %s was processed from %s cache files in %s ' 'seconds (%s seconds/file, %s seconds/datapoint)' '') % (agent_data['id_agent'], len(ingests), round(duration, 4), round( duration / len(ingests), 4), round(duration / len(datapoints), 6)) log.log2info(1007, log_message) else: log_message = ( 'Failed to process all cache files for agent %s. ' 'Investigate.') % (agent_data['id_agent']) log.log2info(1008, log_message)
def process(self): """Update the database using threads.""" # Initialize key variables do_update = False success = None ingests = [] agent_data = { 'devicename': None, 'id_agent': None, 'sources': [], 'timeseries': [], 'timefixed': [], 'max_timestamp': 0 } # Get the directory to which failed files will be moved failure_directory = self.config.ingest_failures_directory() # Get start time for activity start_ts = time.time() # Process file for each timestamp, starting from the oldes file for data_dict in self.metadata: # Initialize key variables timestamp = data_dict['timestamp'] filepath = data_dict['filepath'] # Read in data ingest = drain.Drain(filepath) # Make sure file is OK # Move it to a directory for further analysis # by administrators if ingest.valid() is False: log_message = ('Cache ingest file %s is invalid. Moving.' '') % (filepath) log.log2warning(1054, log_message) shutil.copy(filepath, failure_directory) os.remove(filepath) continue # Append data agent_data['timeseries'].extend(ingest.timeseries()) agent_data['timefixed'].extend(ingest.timefixed()) agent_data['sources'].extend(ingest.sources()) # Append ingest object to a list for later processing ingests.append(ingest) # Update information that doesn't change if do_update is False: agent_data['devicename'] = ingest.devicename() agent_data['id_agent'] = ingest.id_agent() agent_data['agent_name'] = ingest.agent() # Get the PID file for the agent pid_file = daemon.pid_file(self.ingester_agent_name) else: # Get the max timestamp agent_data['max_timestamp'] = max(timestamp, agent_data['max_timestamp']) # Update the PID file for the agent to ensure agentd.py # doesn't kill the ingest while processing a long stream # of files. If we are running this using __main__ = process() # then the pid file wouldn't have been created, hence the logic. if os.path.isfile(pid_file) is True: daemon.update_pid(self.ingester_agent_name) # Update update flag do_update = True # Process the rest if do_update is True: # Upadate and note success (success, datapoints_processed) = self._do_update(agent_data, ingests) # Log duration of activity duration = time.time() - start_ts if success is True: log_message = ( 'Agent %s was processed from %s cache files in %s ' 'seconds (%s seconds/file, %s seconds/datapoint)' '') % (agent_data['id_agent'], len(ingests), round(duration, 4), round( duration / len(ingests), 4), round(duration / datapoints_processed, 6)) log.log2info(1007, log_message) else: log_message = ( 'Failed to process all cache files for agent %s. ' 'Investigate.') % (agent_data['id_agent']) log.log2info(1008, log_message)
def __init__(self, filename): """Method initializing the class. Args: filename: Cache filename Returns: None """ # Initialize key variables self.filename = filename self._information = defaultdict(lambda: defaultdict(dict)) self._sources = [] self.validated = False self.agent_meta = {} data_types = ['timeseries', 'timefixed'] # Ingest data validator = validate.ValidateCache(filename) information = validator.getinfo() # Log if data is bad if information is False: log_message = ('Cache ingest file %s is invalid.') % (filename) log.log2warning(1051, log_message) return else: self.validated = True # Process validated data if self.validated is True: # Get main keys self.agent_meta = _main_keys(information) timestamp = self.agent_meta['timestamp'] id_agent = self.agent_meta['id_agent'] # Process timeseries data for data_type in data_types: # Skip if data type isn't in the data if data_type not in information: continue # Process the data type for agent_label, label_dict in sorted( information[data_type].items()): # Get universal parameters for label_dict base_type = _base_type(label_dict['base_type']) description = label_dict['description'] # Create a key in the data based on the base_type if base_type not in self._information[data_type]: self._information[data_type][base_type] = [] # Process the data associated with the agent_label for datapoint in label_dict['data']: # Create a unique, unchangeable id_datapoint for data index = datapoint[0] value = datapoint[1] source = datapoint[2] id_datapoint = _id_datapoint( id_agent, agent_label, index, self.agent_meta['agent'], self.agent_meta['devicename']) # Convert values to float if this is # data that could be charted if base_type is not None: value = float(value) # Update the data self._information[data_type][base_type].append({ 'id_agent': id_agent, 'id_datapoint': id_datapoint, 'value': value, 'timestamp': timestamp }) # Update sources after fixing encoding self._sources.append({ 'id_agent': id_agent, 'id_datapoint': id_datapoint, 'agent_label': agent_label, 'agent_source': source, 'description': description, 'base_type': base_type })
def _timeseries_data_ok(self): """Check if timeseries data is OK. Args: None Returns: valid: True if valid """ # Initialize key variables valid = True data_type = 'timeseries' # Check major keys expected under each ageng label if self._agent_label_keys_ok() is False: valid = False return valid # Check for timeseries data if data_type in self.data: # Process the data type for _, reported_data in sorted(self.data[data_type].items()): # Make sure the base types are numeric if 'base_type' in reported_data: try: float(reported_data['base_type']) except: log_message = ( 'TimeSeries "base_type" key is non numeric.') log.log2warning(1120, log_message) valid = False return valid else: log_message = ('TimeSeries data has no "base_type" key.') log.log2warning(1117, log_message) valid = False return valid # Process data if 'data' in reported_data: for datapoint in reported_data['data']: # Check to make sure value is numeric value = datapoint[1] try: float(value) except: log_message = ( 'TimeSeries data has non numeric data values.') log.log2warning(1119, log_message) valid = False return valid else: log_message = ('TimeSeries data has no "data" key.') log.log2warning(1118, log_message) valid = False return valid # Return return valid