def last_contacts(ts_start):
"""Get the last time each timeseries datapoint was updated.
Args:
config: Configuration object
ts_start: Timestamp to start from
Returns:
data: List of dicts of last contact information
"""
# Initialize key variables
data = []
last_contact = defaultdict(lambda: defaultdict(dict))
# Get start and stop times
ts_stop = general.normalized_timestamp()
if ts_start > ts_stop:
ts_start = ts_stop
else:
ts_start = general.normalized_timestamp(ts_start)
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(Data.value, Data.idx_datapoint,
Data.timestamp).filter(
and_(Data.timestamp >= ts_start,
Data.timestamp <= ts_stop))
# Add to the list of device idx values
for instance in result:
idx_datapoint = instance.idx_datapoint
timestamp = instance.timestamp
value = instance.value
# Update dictionary
if idx_datapoint in last_contact:
if timestamp > last_contact[idx_datapoint]['timestamp']:
last_contact[idx_datapoint]['timestamp'] = timestamp
last_contact[idx_datapoint]['value'] = value
else:
continue
else:
last_contact[idx_datapoint]['timestamp'] = timestamp
last_contact[idx_datapoint]['value'] = value
# Return the session to the pool after processing
database.close()
# Convert dict to list of dicts
for idx_datapoint in last_contact:
data_dict = {}
data_dict['idx_datapoint'] = idx_datapoint
data_dict['timestamp'] = last_contact[idx_datapoint]['timestamp']
data_dict['value'] = last_contact[idx_datapoint]['value']
data.append(data_dict)
# Return
return data
def test_normalized_timestamp(self):
"""Testing method / function normalized_timestamp."""
# Initialize key variables
result = general.normalized_timestamp(300)
self.assertEqual(result, 300)
result = general.normalized_timestamp(350)
self.assertEqual(result, 300)
result = general.normalized_timestamp(599)
self.assertEqual(result, 300)
def __init__(self, config, idx_datapoint, start=None, stop=None):
"""Function for intializing the class.
Args:
config: Config object
idx_datapoint: idx_datapoint of datapoint
start: Starting timestamp
stop: Ending timestamp
Returns:
None
"""
# Initialize important variables
self.data = defaultdict(dict)
self.config = config
# Get the datapoint's base_type
datapointer = db_datapoint.GetIDXDatapoint(idx_datapoint)
self.base_type = datapointer.base_type()
self.agent_label = datapointer.agent_label()
# Redefine start times
if start is None:
self.ts_start = general.normalized_timestamp() - (3600 * 24)
else:
self.ts_start = general.normalized_timestamp(start)
# Redefine stop times
if stop is None:
self.ts_stop = general.normalized_timestamp()
else:
self.ts_stop = general.normalized_timestamp(stop)
# Fix edge cases
if self.ts_start > self.ts_stop:
self.ts_start = self.ts_stop
# Establish a database session
database = db.Database()
session = database.session()
result = session.query(Data.timestamp, Data.value).filter(
and_(Data.timestamp >= self.ts_start,
Data.timestamp <= self.ts_stop,
Data.idx_datapoint == idx_datapoint))
# Massage data
for instance in result:
self.data[instance.timestamp] = instance.value
# Return the session to the database pool after processing
database.close()
def test_normalized_timestamp(self):
"""Testing method / function normalized_timestamp."""
# Initialize key variables
result = general.normalized_timestamp(300)
self.assertEqual(result, 300)
result = general.normalized_timestamp(350)
self.assertEqual(result, 300)
result = general.normalized_timestamp(599)
self.assertEqual(result, 300)
# Test that we get a UTC timestamp when no timestamp value is provided.
result = general.normalized_timestamp()
expected = general.normalized_timestamp(
int(datetime.utcnow().timestamp()))
self.assertEqual(result, expected)
def _start_timestamp(secondsago, relative=True):
"""Determine the default starting timestamp when not provided.
Args:
None
Returns:
ts_start: Timestamp
"""
# Provide a UTC timestamp 10x the configured interval
ts_limit = int(datetime.utcnow().timestamp()) - (3600 * 12)
# Correct secondsago if it is None
if bool(secondsago) is None:
secondsago = 0
# Calculate timing
if bool(relative) is True:
timestamp = int(datetime.utcnow().timestamp()) - abs(secondsago)
else:
timestamp = abs(secondsago)
# Place a maximum value on timestamp to prevent DB overload
timestamp = max(timestamp, ts_limit)
# Return
ts_start = general.normalized_timestamp(timestamp)
return ts_start
def __init__(self, config, devicename):
"""Method initializing the class.
Args:
config: ConfigAgent configuration object
agent_name: Name of agent
devicename: Devicename that the agent applies to
Returns:
None
"""
# Initialize key variables
self.data = defaultdict(lambda: defaultdict(dict))
agent_name = config.agent_name()
id_agent = get_id_agent(config)
# Add timestamp
self.data['timestamp'] = general.normalized_timestamp()
self.data['id_agent'] = id_agent
self.data['agent'] = agent_name
self.data['devicename'] = devicename
# Construct URL for server
if config.api_server_https() is True:
prefix = 'https://'
else:
prefix = 'http://'
self.url = ('%s%s:%s/%s/receive/%s') % (prefix, config.api_server_name(
), config.api_server_port(), config.api_server_uri(), id_agent)
# Create the cache directory
self.cache_dir = config.agent_cache_directory()
if os.path.exists(self.cache_dir) is False:
os.mkdir(self.cache_dir)
def test_contents(self):
"""Testing function contents."""
# Initialize key variables
devicename = general.hashstring(general.randomstring())
id_agent = general.hashstring(general.randomstring())
agent_name = general.hashstring(general.randomstring())
hosthash = general.hashstring(devicename)
timestamp = general.normalized_timestamp()
# Create a directory and filename for testing
directory = tempfile.mkdtemp()
filename = ('%s/%s_%s_%s.json') % (directory, timestamp, id_agent,
hosthash)
# Test with good data
data = {
'id_agent': id_agent,
'agent': agent_name,
'devicename': devicename,
'timestamp': timestamp
}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
contents = result.contents()
for key in contents.keys():
self.assertEqual(contents[key], data[key])
self.assertEqual(len(contents), len(data))
def __init__(self, config, devicename, test=False):
"""Method initializing the class.
Args:
config: ConfigAgent configuration object
agent_name: Name of agent
devicename: Devicename that the agent applies to
test: True if testing functionality
Returns:
None
"""
# Initialize key variables
self.data = defaultdict(lambda: defaultdict(dict))
agent_name = config.agent_name()
id_agent = get_id_agent(agent_name, test=test)
# Add timestamp
self.data['timestamp'] = general.normalized_timestamp()
self.data['id_agent'] = id_agent
self.data['agent'] = agent_name
self.data['devicename'] = devicename
# Create an object for API interaction
self._api = ReferenceSampleAPI(config)
# Create the cache directory
self.cache_dir = config.agent_cache_directory()
if os.path.exists(self.cache_dir) is False:
os.mkdir(self.cache_dir)
# All cache files created by this agent will end with this suffix.
devicehash = general.hashstring(self.data['devicename'], sha=1)
self.cache_suffix = ('%s_%s.json') % (id_agent, devicehash)
def getdata(value):
"""Get Agent data from the DB by idx value.
Args:
value: idx_datapoint value
Returns:
data: JSON data for the selected agent
"""
# Initialize key variables
idx_datapoint = int(value)
secondsago = general.integerize(request.args.get('secondsago'))
ts_stop = general.integerize(request.args.get('ts_start'))
ts_start = general.integerize(request.args.get('ts_start'))
# Process start and stop times
if bool(secondsago) is True:
ts_stop = int(datetime.utcnow().timestamp())
ts_start = ts_stop - abs(secondsago)
else:
if bool(ts_start) is True and bool(ts_stop) is True:
ts_start = abs(general.normalized_timestamp(
general.integerize(request.args.get('ts_start'))
))
ts_stop = abs(general.normalized_timestamp(
general.integerize(request.args.get('ts_stop'))
))
else:
abort(404)
# Fix start and stop times
if ts_start > ts_stop:
ts_start = ts_stop
# Fail if more than a year of data is being requested
if ts_stop - ts_start >= 31536000:
abort(404)
# Get data
query = db_data.GetIDXData(CONFIG, idx_datapoint, ts_start, ts_stop)
data = query.everything()
# Return
return jsonify(data)
def test_valid(self):
"""Testing function valid."""
# Initialize key variables
devicename = general.hashstring(general.randomstring())
id_agent = general.hashstring(general.randomstring())
agent_name = general.hashstring(general.randomstring())
hosthash = general.hashstring(devicename)
timestamp = general.normalized_timestamp()
# Create a directory and filename for testing
directory = tempfile.mkdtemp()
filename = ('%s/%s_%s_%s.json') % (
directory,
timestamp,
id_agent,
hosthash)
# Test with good data
data = {
'id_agent': id_agent,
'agent': agent_name,
'devicename': devicename,
'timestamp': timestamp
}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result.valid(), True)
# Test with bad data (missing id_agent value)
data = {
'id_agent': '',
'agent': agent_name,
'devicename': devicename,
'timestamp': timestamp
}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result.valid(), False)
# Cleanup
os.remove(filename)
def lastcontacts():
"""Get last contact data from the DB.
Args:
None
Returns:
data: JSON data for the selected agent
"""
# Get starting timestamp
secondsago = general.integerize(request.args.get('secondsago'))
timestamp = general.integerize(request.args.get('ts_start'))
ts_start = general.normalized_timestamp()
# Get data
data = db_data.last_contacts(ts_start)
# Return
return jsonify(data)
def _timestamps():
"""Create a list of timestamps staring starting 30 minutes ago.
Args:
None
Returns:
timestamps: List of timestamps
"""
# Initialize key variables
timestamps = []
config = configuration.Config()
interval = config.interval()
# Get the UTC timestamp
utc_timestamp = int(datetime.utcnow().timestamp())
starting_timestamp = general.normalized_timestamp(utc_timestamp) - 1800
# Create a list of timestamps based on UTC timesamp
timestamps = list(
range(starting_timestamp, starting_timestamp - 1800, -interval))
return timestamps
def test__keys_in_filename(self):
"""Testing function _keys_in_filename."""
# Initialize key variables
id_agent = general.hashstring(general.randomstring())
hosthash = general.hashstring(general.randomstring())
timestamp = general.normalized_timestamp()
# Create a directory and filename for testing
directory = tempfile.mkdtemp()
filename = ('%s/%s_%s_%s.json') % (directory, timestamp, id_agent,
hosthash)
# Test with good data
data = {'id_agent': id_agent, 'timestamp': timestamp}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result._keys_in_filename(), True)
# Test with bad data (missing id_agent value)
data = {'id_agent': '', 'timestamp': timestamp}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result._keys_in_filename(), False)
# Test with bad data (mismatched timestamp value)
data = {'id_agent': id_agent, 'timestamp': 0}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result._keys_in_filename(), False)
# Test with bad data (string timestamp value)
data = {'id_agent': id_agent, 'timestamp': str(timestamp)}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result._keys_in_filename(), False)
# Test with bad data (string timestamp value)
data = {'id_agent': id_agent, 'timestamp': str(timestamp)}
with open(filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(filename)
self.assertEqual(result._keys_in_filename(), False)
# Test with bad data
# (non normalized timestamp value in filename and data)
bad_time = timestamp + 1
bad_filename = ('%s/%s_%s_%s.json') % (directory, bad_time, id_agent,
hosthash)
data = {'id_agent': id_agent, 'timestamp': str(timestamp)}
with open(bad_filename, 'w') as f_handle:
json.dump(data, f_handle)
result = validate._CheckFile(bad_filename)
self.assertEqual(result._keys_in_filename(), False)
# Cleanup
# Get all files in directory
filenames = [
filename for filename in os.listdir(directory)
if os.path.isfile(os.path.join(directory, filename))
]
# Get the full filepath for the cache file and post
for filename in filenames:
filepath = os.path.join(directory, filename)
os.remove(filepath)
def __init__(self):
"""Method initializing the class."""
# Initialize key variables
self.data = {}
self.data['idx_datapoint'] = 1
self.data['idx_agentname'] = 1
self.data['idx_billcode'] = 1
self.data['idx_device'] = 1
self.data['idx_deviceagent'] = 1
self.data['idx_department'] = 1
self.data['timestamp'] = general.normalized_timestamp()
self.data['id_datapoint'] = general.hashstring(general.randomstring())
self.data['devicename'] = general.hashstring(general.randomstring())
self.data['id_agent'] = general.hashstring(general.randomstring())
self.data['id_datapoint'] = general.hashstring(general.randomstring())
self.data['devicename'] = general.hashstring(general.randomstring())
self.data['device_description'] = general.hashstring(
general.randomstring())
self.data['agent_name'] = general.hashstring(general.randomstring())
self.data['agent_source'] = general.hashstring(general.randomstring())
self.data['agent_label'] = general.hashstring(general.randomstring())
self.data['department_code'] = general.hashstring(
general.randomstring())
self.data['department_name'] = general.hashstring(
general.randomstring())
self.data['billcode_code'] = general.hashstring(general.randomstring())
self.data['billcode_name'] = general.hashstring(general.randomstring())
# Drop the database and create tables
initialize_db()
# Initialize agent variables
agent_data = {}
agent_data['devicename'] = self.data['devicename']
agent_data['device_description'] = self.data['device_description']
agent_data['id_agent'] = self.data['id_agent']
agent_data['agent'] = self.data['agent_name']
agent_data['timestamp'] = self.data['timestamp']
(self.data['idx_device'],
self.data['idx_agent']) = _setup_db_deviceagent(agent_data)
# Get DeviceAgent index value
deviceagent = hagent.GetDeviceAgent(self.data['idx_device'],
self.data['idx_agent'])
self.data['idx_deviceagent'] = deviceagent.idx_deviceagent()
# Insert Department data into database
dept_data = Department(name=self.data['department_name'].encode(),
code=self.data['department_code'].encode())
database = db.Database()
database.add_all([dept_data], 1035)
# Insert Billcode data into database
bill_data = Billcode(name=self.data['billcode_name'].encode(),
code=self.data['billcode_code'].encode())
database = db.Database()
database.add_all([bill_data], 1039)
# Insert Datapoint data into database
new_data = Datapoint(agent_source=self.data['agent_source'].encode(),
agent_label=self.data['agent_label'].encode(),
last_timestamp=self.data['timestamp'],
idx_deviceagent=self.data['idx_deviceagent'],
id_datapoint=self.data['id_datapoint'].encode())
database = db.Database()
database.add_all([new_data], 1072)
def __init__(self):
"""Method initializing the class."""
# Initialize key variables
self.data = {}
self.data['idx_datapoint'] = 1
self.data['idx_agentname'] = 1
self.data['idx_billcode'] = 1
self.data['idx_device'] = 1
self.data['idx_deviceagent'] = 1
self.data['idx_department'] = 1
self.data['timestamp'] = general.normalized_timestamp()
self.data['last_timestamp'] = general.normalized_timestamp()
self.data['devicename'] = general.hashstring(general.randomstring())
self.data['id_agent'] = general.hashstring(general.randomstring())
self.data['id_datapoint'] = general.hashstring(general.randomstring())
self.data['devicename'] = general.hashstring(general.randomstring())
self.data['device_description'] = general.hashstring(
general.randomstring())
self.data['agent'] = general.hashstring(general.randomstring())
self.data['agent_source'] = general.hashstring(general.randomstring())
self.data['agent_label'] = general.hashstring(general.randomstring())
self.data['department_code'] = general.hashstring(
general.randomstring())
self.data['department_name'] = general.hashstring(
general.randomstring())
self.data['billcode_code'] = general.hashstring(
general.randomstring())
self.data['billcode_name'] = general.hashstring(
general.randomstring())
# Define data to Insert
self.data['values'] = []
for timestamp in _timestamps():
value_dict = {
'idx_datapoint': self.data['idx_datapoint'],
'value': timestamp * (1 + random.uniform(0, 1)),
'timestamp': timestamp}
self.data['values'].append(value_dict)
# Drop the database and create tables
initialize_db()
# Initialize agent variables
agent_data = {}
agent_data['devicename'] = self.data['devicename']
agent_data['device_description'] = self.data['device_description']
agent_data['id_agent'] = self.data['id_agent']
agent_data['agent'] = self.data['agent']
agent_data['timestamp'] = self.data['timestamp']
(
self.data['idx_device'],
self.data['idx_agent']) = _setup_db_deviceagent(agent_data)
# Get DeviceAgent index value
deviceagent = hagent.GetDeviceAgent(
self.data['idx_device'], self.data['idx_agent'])
self.data['idx_deviceagent'] = deviceagent.idx_deviceagent()
# Insert Department data into database
dept_data = Department(
name=self.data['department_name'].encode(),
code=self.data['department_code'].encode()
)
database = db.Database()
database.add_all([dept_data], 1035)
# Insert Billcode data into database
bill_data = Billcode(
name=self.data['billcode_name'].encode(),
code=self.data['billcode_code'].encode()
)
database = db.Database()
database.add_all([bill_data], 1039)
# Insert Datapoint data into database
new_data = Datapoint(
agent_source=self.data['agent_source'].encode(),
agent_label=self.data['agent_label'].encode(),
last_timestamp=self.data['last_timestamp'],
idx_deviceagent=self.data['idx_deviceagent'],
id_datapoint=self.data['id_datapoint'].encode())
database = db.Database()
database.add_all([new_data], 1144)
# Insert timeseries data into database
new_data_list = []
for item in self.data['values']:
new_data_list.append(
Data(
idx_datapoint=item['idx_datapoint'],
timestamp=item['timestamp'],
value=item['value']))
database = db.Database()
database.add_all(new_data_list, 1072)
