def parse_line(self):
"""
Parse each of the lines of the Apache HTTP server access log.
:return:
"""
parsed = parse_apache_line(self.parser, self.line)
measurements = []
user_agent = parsed['%{User-Agent}\i"'].replace('/', '')
bytes = int(parsed['%b'])
status_code = parsed['%>s']
properties = {"app_id": self.app_id}
# Split the line by spaces and get the request in the first value
request = parsed['%r'].split(' ')[0]
logging.info("user_agent: {0}, bytes: {1}, request: {2}, status_code: {3}".format(
user_agent, bytes, request, status_code))
measurements.append(Measurement(
metric='HTTP_REQUESTS',
value=1,
source=request,
properties=properties))
measurements.append(Measurement(
metric='HTTP_BYTES',
value=bytes,
source=user_agent,
properties=properties))
self.send_measurements(measurements)
def run(self):
"""
Main loop
"""
properties = {"app_id": self.app_id}
while True:
# Loop over the tickers and lookup the stock price and volume
for ticker in self.tickers:
measurements = []
price = ystockquote.get_price(ticker)
volume = ystockquote.get_volume(ticker)
timestamp = int(time.time())
if volume == 'N/A' or price == 'N/A':
sys.stderr.write(
'Could not find ticker \"{0}\", skipping'.format(
ticker))
else:
print("ticker: {0}, price: {1}, volume: {2}".format(
ticker, price, volume))
measurements.append(
Measurement(metric='STOCK_PRICE',
value=price,
source=ticker,
properties=properties))
measurements.append(
Measurement(metric='STOCK_VOLUME',
value=volume,
source=ticker,
properties=properties))
self.send_measurements(measurements)
time.sleep(self.interval)
def test_measurement_create_batch_with_properties(self):
measurements = []
properties = {
"app_id": "red",
"source_type": "blue",
"origin": "green"
}
timestamp = int(datetime.now().strftime('%s'))
measurements.append(
Measurement(metric='CPU',
value=0.5,
source='red',
timestamp=timestamp,
properties=properties))
measurements.append(
Measurement(metric='CPU',
value=0.6,
source='green',
timestamp=timestamp,
properties=properties))
measurements.append(
Measurement(metric='CPU',
value=0.7,
source='blue',
timestamp=timestamp,
properties=properties))
self.api.measurement_create_batch(measurements)
def test_parse_date_bad_date_format(self):
try:
s = 'foobar'
timestamp = Measurement.parse_timestamp(s)
self.assertTrue(False)
except ValueError:
pass
def test_measurement_defaults(self):
measurement = Measurement()
self.assertIsNone(measurement.metric)
self.assertIsNone(measurement.value)
self.assertIsNone(measurement.source)
self.assertIsNone(measurement.timestamp)
self.assertIsNone(measurement.properties)
def test_measurement_create_batch(self):
measurements = []
timestamp = int(datetime.now().strftime('%s'))
measurements.append(
Measurement(metric='CPU',
value=0.5,
source='red',
timestamp=timestamp))
measurements.append(
Measurement(metric='CPU',
value=0.6,
source='green',
timestamp=timestamp))
measurements.append(
Measurement(metric='CPU',
value=0.7,
source='blue',
timestamp=timestamp))
self.api.measurement_create_batch(measurements)
def tweet_term_to_measurements(self):
"""
Convert the tweet term counts into measurements
:return:
"""
for key in self.tweet_term_count:
source = key.replace(' ', '_')
value = self.tweet_term_count[key]['count']
self.measurements.append(Measurement(metric='TWEET_COUNT',
source=source,
value=value,
properties=self.properties))
def process_records(self):
"""
Handles querying and extraction
:return:
"""
rows = petl.values(self.table, 'dt', 'total', 'duration')
row_count = 0
measurements = []
properties = {'app_id': self.app_id}
source = "littledog.com"
for row in rows:
timestamp = int(row[0].strftime('%s'))
total = int(row[1])
duration = int(row[2])
logging.debug("Add Measurements => dt: {0}, total: {1}, duration: {2} ".format(timestamp, total, duration))
row_count += 1
measurements.append(Measurement(metric='ONLINE_TRANSACTION_COUNT',
source=source,
value=total,
timestamp=timestamp,
properties=properties))
measurements.append(Measurement(metric='ONLINE_TRANSACTION_TIME',
source=source,
value=duration,
timestamp=timestamp,
properties=properties))
# Send when we have batch of 10 measurements
if row_count == 10:
# send measurements
self.send_measurements(measurements)
measurements = []
row_count = 0
# If there are any remaining measurements send them on
if len(measurements) > 0:
self.api.measurement_create_batch(measurements)
def test_measurement_constructor(self):
metric = 'CPU'
value = 0.5
source = 'foobar'
timestamp = int(datetime.now().strftime('%s'))
properties = {
"app_id": "red",
"source_type": "blue",
"origin": "green"
}
measurement = Measurement(metric=metric,
value=value,
source=source,
timestamp=timestamp,
properties=properties)
self.assertEqual(metric, measurement.metric)
self.assertEqual(value, measurement.value)
self.assertEqual(source, measurement.source)
self.assertEqual(timestamp, measurement.timestamp)
self.assertEqual(properties, measurement.properties)
def run(self):
"""
Main loop
"""
while True:
# Loop over the tickers and lookup the stock price and volume
for city in self.cities:
observation = self.owm.weather_at_place(city)
weather = observation.get_weather()
measurements = []
temperature = float(
weather.get_temperature('fahrenheit')['temp'])
source = city.replace(',', '_').replace(' ', '_')
properties = {"app_id": "LittleDog"}
print('city: {0}, temperature: {1}'.format(
city.replace(',', '_').replace(' ', '_'), temperature))
measurements.append(
Measurement(metric='TEMPERATURE',
value=temperature,
source=source,
properties=properties))
self.send_measurements(measurements)
time.sleep(self.interval)
def test_parse_date_epoch(self):
expected = int(datetime.now().strftime('%s'))
timestamp = Measurement.parse_timestamp(expected)
self.assertEqual(expected, timestamp)
def test_parse_timestamp_date_string_epoch_time(self):
s = '1466704787'
d = Measurement.parse_timestamp(s)
self.assertEqual(type(d), int)
self.assertEqual(d, 1466704787)
def test_parse_timestamp_date_string_yymmddHHMM(self):
d = datetime.utcnow()
s = d.strftime('%Y-%m-%d %I:%M:%S%p')
ts = Measurement.parse_timestamp(s)
self.assertEqual(type(ts), int)
self.assertEqual(ts, int(d.strftime('%s')))
def test_parse_date_ymd_hms(self):
s = '2014-06-30 02:27:16PM'
timestamp = Measurement.parse_timestamp(s)
expected = int(datetime(2014, 6, 30, 14, 27, 16).strftime('%s'))
self.assertEqual(expected, timestamp)
def test_parse_date_ymd(self):
s = '2015-06-30'
timestamp = Measurement.parse_timestamp(s)
expected = int(datetime(2015, 6, 30).strftime('%s'))
self.assertEqual(expected, timestamp)
def test_parse_timestamp_date_string_epoch_time(self):
s = '1466704787'
d = Measurement.parse_timestamp(s)
self.assertEqual(type(d), int)
self.assertEqual(d, 1466704787)
def test_parse_timestamp_date_string_yymmddHHMM(self):
d = datetime.utcnow()
s = d.strftime('%Y-%m-%d %I:%M:%S%p')
ts = Measurement.parse_timestamp(s)
self.assertEqual(type(ts), int)
self.assertEqual(ts, int(d.strftime('%s')))