def last_contacts(ts_start): """Get the last time each timeseries datapoint was updated. Args: config: Configuration object ts_start: Timestamp to start from Returns: data: List of dicts of last contact information """ # Initialize key variables data = [] last_contact = defaultdict(lambda: defaultdict(dict)) # Get start and stop times ts_stop = general.normalized_timestamp() if ts_start > ts_stop: ts_start = ts_stop else: ts_start = general.normalized_timestamp(ts_start) # Establish a database session database = db.Database() session = database.session() result = session.query(Data.value, Data.idx_datapoint, Data.timestamp).filter( and_(Data.timestamp >= ts_start, Data.timestamp <= ts_stop)) # Add to the list of device idx values for instance in result: idx_datapoint = instance.idx_datapoint timestamp = instance.timestamp value = instance.value # Update dictionary if idx_datapoint in last_contact: if timestamp > last_contact[idx_datapoint]['timestamp']: last_contact[idx_datapoint]['timestamp'] = timestamp last_contact[idx_datapoint]['value'] = value else: continue else: last_contact[idx_datapoint]['timestamp'] = timestamp last_contact[idx_datapoint]['value'] = value # Return the session to the pool after processing database.close() # Convert dict to list of dicts for idx_datapoint in last_contact: data_dict = {} data_dict['idx_datapoint'] = idx_datapoint data_dict['timestamp'] = last_contact[idx_datapoint]['timestamp'] data_dict['value'] = last_contact[idx_datapoint]['value'] data.append(data_dict) # Return return data
def test_normalized_timestamp(self): """Testing method / function normalized_timestamp.""" # Initialize key variables result = general.normalized_timestamp(300) self.assertEqual(result, 300) result = general.normalized_timestamp(350) self.assertEqual(result, 300) result = general.normalized_timestamp(599) self.assertEqual(result, 300)
def __init__(self, config, idx_datapoint, start=None, stop=None): """Function for intializing the class. Args: config: Config object idx_datapoint: idx_datapoint of datapoint start: Starting timestamp stop: Ending timestamp Returns: None """ # Initialize important variables self.data = defaultdict(dict) self.config = config # Get the datapoint's base_type datapointer = db_datapoint.GetIDXDatapoint(idx_datapoint) self.base_type = datapointer.base_type() self.agent_label = datapointer.agent_label() # Redefine start times if start is None: self.ts_start = general.normalized_timestamp() - (3600 * 24) else: self.ts_start = general.normalized_timestamp(start) # Redefine stop times if stop is None: self.ts_stop = general.normalized_timestamp() else: self.ts_stop = general.normalized_timestamp(stop) # Fix edge cases if self.ts_start > self.ts_stop: self.ts_start = self.ts_stop # Establish a database session database = db.Database() session = database.session() result = session.query(Data.timestamp, Data.value).filter( and_(Data.timestamp >= self.ts_start, Data.timestamp <= self.ts_stop, Data.idx_datapoint == idx_datapoint)) # Massage data for instance in result: self.data[instance.timestamp] = instance.value # Return the session to the database pool after processing database.close()
def test_normalized_timestamp(self): """Testing method / function normalized_timestamp.""" # Initialize key variables result = general.normalized_timestamp(300) self.assertEqual(result, 300) result = general.normalized_timestamp(350) self.assertEqual(result, 300) result = general.normalized_timestamp(599) self.assertEqual(result, 300) # Test that we get a UTC timestamp when no timestamp value is provided. result = general.normalized_timestamp() expected = general.normalized_timestamp( int(datetime.utcnow().timestamp())) self.assertEqual(result, expected)
def _start_timestamp(secondsago, relative=True): """Determine the default starting timestamp when not provided. Args: None Returns: ts_start: Timestamp """ # Provide a UTC timestamp 10x the configured interval ts_limit = int(datetime.utcnow().timestamp()) - (3600 * 12) # Correct secondsago if it is None if bool(secondsago) is None: secondsago = 0 # Calculate timing if bool(relative) is True: timestamp = int(datetime.utcnow().timestamp()) - abs(secondsago) else: timestamp = abs(secondsago) # Place a maximum value on timestamp to prevent DB overload timestamp = max(timestamp, ts_limit) # Return ts_start = general.normalized_timestamp(timestamp) return ts_start
def __init__(self, config, devicename): """Method initializing the class. Args: config: ConfigAgent configuration object agent_name: Name of agent devicename: Devicename that the agent applies to Returns: None """ # Initialize key variables self.data = defaultdict(lambda: defaultdict(dict)) agent_name = config.agent_name() id_agent = get_id_agent(config) # Add timestamp self.data['timestamp'] = general.normalized_timestamp() self.data['id_agent'] = id_agent self.data['agent'] = agent_name self.data['devicename'] = devicename # Construct URL for server if config.api_server_https() is True: prefix = 'https://' else: prefix = 'http://' self.url = ('%s%s:%s/%s/receive/%s') % (prefix, config.api_server_name( ), config.api_server_port(), config.api_server_uri(), id_agent) # Create the cache directory self.cache_dir = config.agent_cache_directory() if os.path.exists(self.cache_dir) is False: os.mkdir(self.cache_dir)
def test_contents(self): """Testing function contents.""" # Initialize key variables devicename = general.hashstring(general.randomstring()) id_agent = general.hashstring(general.randomstring()) agent_name = general.hashstring(general.randomstring()) hosthash = general.hashstring(devicename) timestamp = general.normalized_timestamp() # Create a directory and filename for testing directory = tempfile.mkdtemp() filename = ('%s/%s_%s_%s.json') % (directory, timestamp, id_agent, hosthash) # Test with good data data = { 'id_agent': id_agent, 'agent': agent_name, 'devicename': devicename, 'timestamp': timestamp } with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) contents = result.contents() for key in contents.keys(): self.assertEqual(contents[key], data[key]) self.assertEqual(len(contents), len(data))
def __init__(self, config, devicename, test=False): """Method initializing the class. Args: config: ConfigAgent configuration object agent_name: Name of agent devicename: Devicename that the agent applies to test: True if testing functionality Returns: None """ # Initialize key variables self.data = defaultdict(lambda: defaultdict(dict)) agent_name = config.agent_name() id_agent = get_id_agent(agent_name, test=test) # Add timestamp self.data['timestamp'] = general.normalized_timestamp() self.data['id_agent'] = id_agent self.data['agent'] = agent_name self.data['devicename'] = devicename # Create an object for API interaction self._api = ReferenceSampleAPI(config) # Create the cache directory self.cache_dir = config.agent_cache_directory() if os.path.exists(self.cache_dir) is False: os.mkdir(self.cache_dir) # All cache files created by this agent will end with this suffix. devicehash = general.hashstring(self.data['devicename'], sha=1) self.cache_suffix = ('%s_%s.json') % (id_agent, devicehash)
def getdata(value): """Get Agent data from the DB by idx value. Args: value: idx_datapoint value Returns: data: JSON data for the selected agent """ # Initialize key variables idx_datapoint = int(value) secondsago = general.integerize(request.args.get('secondsago')) ts_stop = general.integerize(request.args.get('ts_start')) ts_start = general.integerize(request.args.get('ts_start')) # Process start and stop times if bool(secondsago) is True: ts_stop = int(datetime.utcnow().timestamp()) ts_start = ts_stop - abs(secondsago) else: if bool(ts_start) is True and bool(ts_stop) is True: ts_start = abs(general.normalized_timestamp( general.integerize(request.args.get('ts_start')) )) ts_stop = abs(general.normalized_timestamp( general.integerize(request.args.get('ts_stop')) )) else: abort(404) # Fix start and stop times if ts_start > ts_stop: ts_start = ts_stop # Fail if more than a year of data is being requested if ts_stop - ts_start >= 31536000: abort(404) # Get data query = db_data.GetIDXData(CONFIG, idx_datapoint, ts_start, ts_stop) data = query.everything() # Return return jsonify(data)
def test_valid(self): """Testing function valid.""" # Initialize key variables devicename = general.hashstring(general.randomstring()) id_agent = general.hashstring(general.randomstring()) agent_name = general.hashstring(general.randomstring()) hosthash = general.hashstring(devicename) timestamp = general.normalized_timestamp() # Create a directory and filename for testing directory = tempfile.mkdtemp() filename = ('%s/%s_%s_%s.json') % ( directory, timestamp, id_agent, hosthash) # Test with good data data = { 'id_agent': id_agent, 'agent': agent_name, 'devicename': devicename, 'timestamp': timestamp } with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result.valid(), True) # Test with bad data (missing id_agent value) data = { 'id_agent': '', 'agent': agent_name, 'devicename': devicename, 'timestamp': timestamp } with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result.valid(), False) # Cleanup os.remove(filename)
def lastcontacts(): """Get last contact data from the DB. Args: None Returns: data: JSON data for the selected agent """ # Get starting timestamp secondsago = general.integerize(request.args.get('secondsago')) timestamp = general.integerize(request.args.get('ts_start')) ts_start = general.normalized_timestamp() # Get data data = db_data.last_contacts(ts_start) # Return return jsonify(data)
def _timestamps(): """Create a list of timestamps staring starting 30 minutes ago. Args: None Returns: timestamps: List of timestamps """ # Initialize key variables timestamps = [] config = configuration.Config() interval = config.interval() # Get the UTC timestamp utc_timestamp = int(datetime.utcnow().timestamp()) starting_timestamp = general.normalized_timestamp(utc_timestamp) - 1800 # Create a list of timestamps based on UTC timesamp timestamps = list( range(starting_timestamp, starting_timestamp - 1800, -interval)) return timestamps
def test__keys_in_filename(self): """Testing function _keys_in_filename.""" # Initialize key variables id_agent = general.hashstring(general.randomstring()) hosthash = general.hashstring(general.randomstring()) timestamp = general.normalized_timestamp() # Create a directory and filename for testing directory = tempfile.mkdtemp() filename = ('%s/%s_%s_%s.json') % (directory, timestamp, id_agent, hosthash) # Test with good data data = {'id_agent': id_agent, 'timestamp': timestamp} with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result._keys_in_filename(), True) # Test with bad data (missing id_agent value) data = {'id_agent': '', 'timestamp': timestamp} with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result._keys_in_filename(), False) # Test with bad data (mismatched timestamp value) data = {'id_agent': id_agent, 'timestamp': 0} with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result._keys_in_filename(), False) # Test with bad data (string timestamp value) data = {'id_agent': id_agent, 'timestamp': str(timestamp)} with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result._keys_in_filename(), False) # Test with bad data (string timestamp value) data = {'id_agent': id_agent, 'timestamp': str(timestamp)} with open(filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(filename) self.assertEqual(result._keys_in_filename(), False) # Test with bad data # (non normalized timestamp value in filename and data) bad_time = timestamp + 1 bad_filename = ('%s/%s_%s_%s.json') % (directory, bad_time, id_agent, hosthash) data = {'id_agent': id_agent, 'timestamp': str(timestamp)} with open(bad_filename, 'w') as f_handle: json.dump(data, f_handle) result = validate._CheckFile(bad_filename) self.assertEqual(result._keys_in_filename(), False) # Cleanup # Get all files in directory filenames = [ filename for filename in os.listdir(directory) if os.path.isfile(os.path.join(directory, filename)) ] # Get the full filepath for the cache file and post for filename in filenames: filepath = os.path.join(directory, filename) os.remove(filepath)
def __init__(self): """Method initializing the class.""" # Initialize key variables self.data = {} self.data['idx_datapoint'] = 1 self.data['idx_agentname'] = 1 self.data['idx_billcode'] = 1 self.data['idx_device'] = 1 self.data['idx_deviceagent'] = 1 self.data['idx_department'] = 1 self.data['timestamp'] = general.normalized_timestamp() self.data['id_datapoint'] = general.hashstring(general.randomstring()) self.data['devicename'] = general.hashstring(general.randomstring()) self.data['id_agent'] = general.hashstring(general.randomstring()) self.data['id_datapoint'] = general.hashstring(general.randomstring()) self.data['devicename'] = general.hashstring(general.randomstring()) self.data['device_description'] = general.hashstring( general.randomstring()) self.data['agent_name'] = general.hashstring(general.randomstring()) self.data['agent_source'] = general.hashstring(general.randomstring()) self.data['agent_label'] = general.hashstring(general.randomstring()) self.data['department_code'] = general.hashstring( general.randomstring()) self.data['department_name'] = general.hashstring( general.randomstring()) self.data['billcode_code'] = general.hashstring(general.randomstring()) self.data['billcode_name'] = general.hashstring(general.randomstring()) # Drop the database and create tables initialize_db() # Initialize agent variables agent_data = {} agent_data['devicename'] = self.data['devicename'] agent_data['device_description'] = self.data['device_description'] agent_data['id_agent'] = self.data['id_agent'] agent_data['agent'] = self.data['agent_name'] agent_data['timestamp'] = self.data['timestamp'] (self.data['idx_device'], self.data['idx_agent']) = _setup_db_deviceagent(agent_data) # Get DeviceAgent index value deviceagent = hagent.GetDeviceAgent(self.data['idx_device'], self.data['idx_agent']) self.data['idx_deviceagent'] = deviceagent.idx_deviceagent() # Insert Department data into database dept_data = Department(name=self.data['department_name'].encode(), code=self.data['department_code'].encode()) database = db.Database() database.add_all([dept_data], 1035) # Insert Billcode data into database bill_data = Billcode(name=self.data['billcode_name'].encode(), code=self.data['billcode_code'].encode()) database = db.Database() database.add_all([bill_data], 1039) # Insert Datapoint data into database new_data = Datapoint(agent_source=self.data['agent_source'].encode(), agent_label=self.data['agent_label'].encode(), last_timestamp=self.data['timestamp'], idx_deviceagent=self.data['idx_deviceagent'], id_datapoint=self.data['id_datapoint'].encode()) database = db.Database() database.add_all([new_data], 1072)
def __init__(self): """Method initializing the class.""" # Initialize key variables self.data = {} self.data['idx_datapoint'] = 1 self.data['idx_agentname'] = 1 self.data['idx_billcode'] = 1 self.data['idx_device'] = 1 self.data['idx_deviceagent'] = 1 self.data['idx_department'] = 1 self.data['timestamp'] = general.normalized_timestamp() self.data['last_timestamp'] = general.normalized_timestamp() self.data['devicename'] = general.hashstring(general.randomstring()) self.data['id_agent'] = general.hashstring(general.randomstring()) self.data['id_datapoint'] = general.hashstring(general.randomstring()) self.data['devicename'] = general.hashstring(general.randomstring()) self.data['device_description'] = general.hashstring( general.randomstring()) self.data['agent'] = general.hashstring(general.randomstring()) self.data['agent_source'] = general.hashstring(general.randomstring()) self.data['agent_label'] = general.hashstring(general.randomstring()) self.data['department_code'] = general.hashstring( general.randomstring()) self.data['department_name'] = general.hashstring( general.randomstring()) self.data['billcode_code'] = general.hashstring( general.randomstring()) self.data['billcode_name'] = general.hashstring( general.randomstring()) # Define data to Insert self.data['values'] = [] for timestamp in _timestamps(): value_dict = { 'idx_datapoint': self.data['idx_datapoint'], 'value': timestamp * (1 + random.uniform(0, 1)), 'timestamp': timestamp} self.data['values'].append(value_dict) # Drop the database and create tables initialize_db() # Initialize agent variables agent_data = {} agent_data['devicename'] = self.data['devicename'] agent_data['device_description'] = self.data['device_description'] agent_data['id_agent'] = self.data['id_agent'] agent_data['agent'] = self.data['agent'] agent_data['timestamp'] = self.data['timestamp'] ( self.data['idx_device'], self.data['idx_agent']) = _setup_db_deviceagent(agent_data) # Get DeviceAgent index value deviceagent = hagent.GetDeviceAgent( self.data['idx_device'], self.data['idx_agent']) self.data['idx_deviceagent'] = deviceagent.idx_deviceagent() # Insert Department data into database dept_data = Department( name=self.data['department_name'].encode(), code=self.data['department_code'].encode() ) database = db.Database() database.add_all([dept_data], 1035) # Insert Billcode data into database bill_data = Billcode( name=self.data['billcode_name'].encode(), code=self.data['billcode_code'].encode() ) database = db.Database() database.add_all([bill_data], 1039) # Insert Datapoint data into database new_data = Datapoint( agent_source=self.data['agent_source'].encode(), agent_label=self.data['agent_label'].encode(), last_timestamp=self.data['last_timestamp'], idx_deviceagent=self.data['idx_deviceagent'], id_datapoint=self.data['id_datapoint'].encode()) database = db.Database() database.add_all([new_data], 1144) # Insert timeseries data into database new_data_list = [] for item in self.data['values']: new_data_list.append( Data( idx_datapoint=item['idx_datapoint'], timestamp=item['timestamp'], value=item['value'])) database = db.Database() database.add_all(new_data_list, 1072)