class DatadogMiddleware(object):
DD_TIMING_ATTRIBUTE = '_dd_start_time'
def __init__(self):
app_name = settings.DATADOG_APP_NAME
self.stats = ThreadStats()
self.stats.start()
self.error_metric = '{0}.errors'.format(app_name)
self.timing_metric = '{0}.request_time'.format(app_name)
self.event_tags = [app_name, 'exception']
def process_request(self, request):
setattr(request, self.DD_TIMING_ATTRIBUTE, time.time())
def process_response(self, request, response):
""" Submit timing metrics from the current request """
if not hasattr(request, self.DD_TIMING_ATTRIBUTE):
return response
# Calculate request time and submit to Datadog
request_time = time.time() - getattr(request, self.DD_TIMING_ATTRIBUTE)
tags = self._get_metric_tags(request)
self.stats.histogram(self.timing_metric, request_time, tags=tags)
return response
def process_exception(self, request, exception):
""" Captures Django view exceptions as Datadog events """
if isinstance(exception, Http404):
# Don't report 404 not found
return
# Get a formatted version of the traceback.
exc = traceback.format_exc()
# Make request.META json-serializable.
szble = {}
for k, v in request.META.items():
if isinstance(v, (list, basestring, bool, int, float, long)):
szble[k] = v
else:
szble[k] = str(v)
title = 'Exception from {0}'.format(request.path)
text = "Traceback:\n@@@\n{0}\n@@@\nMetadata:\n@@@\n{1}\n@@@" \
.format(exc, json.dumps(szble, indent=2))
# Submit the exception to Datadog
self.stats.event(title,
text,
alert_type='error',
aggregation_key=request.path,
tags=self.event_tags)
# Increment our errors metric
tags = self._get_metric_tags(request)
self.stats.increment(self.error_metric, tags=tags)
def _get_metric_tags(self, request):
return ['path:{0}'.format(request.path)]
class DatadogMiddleware(object):
DD_TIMING_ATTRIBUTE = '_dd_start_time'
def __init__(self):
app_name = settings.DATADOG_APP_NAME
self.stats = ThreadStats()
self.stats.start()
self.error_metric = '{0}.errors'.format(app_name)
self.timing_metric = '{0}.request_time'.format(app_name)
self.event_tags = [app_name, 'exception']
def process_request(self, request):
setattr(request, self.DD_TIMING_ATTRIBUTE, time.time())
def process_response(self, request, response):
""" Submit timing metrics from the current request """
if not hasattr(request, self.DD_TIMING_ATTRIBUTE):
return response
# Calculate request time and submit to Datadog
request_time = time.time() - getattr(request, self.DD_TIMING_ATTRIBUTE)
tags = self._get_metric_tags(request)
self.stats.histogram(self.timing_metric, request_time, tags=tags)
return response
def process_exception(self, request, exception):
""" Captures Django view exceptions as Datadog events """
if isinstance(exception, Http404):
# Don't report 404 not found
return
# Get a formatted version of the traceback.
exc = traceback.format_exc()
# Make request.META json-serializable.
szble = {}
for k, v in request.META.items():
if isinstance(v, (list, basestring, bool, int, float, long)):
szble[k] = v
else:
szble[k] = str(v)
title = 'Exception from {0}'.format(request.path)
text = "Traceback:\n@@@\n{0}\n@@@\nMetadata:\n@@@\n{1}\n@@@" \
.format(exc, json.dumps(szble, indent=2))
# Submit the exception to Datadog
self.stats.event(title, text, alert_type='error', aggregation_key=request.path, tags=self.event_tags)
# Increment our errors metric
tags = self._get_metric_tags(request)
self.stats.increment(self.error_metric, tags=tags)
def _get_metric_tags(self, request):
return ['path:{0}'.format(request.path)]
def test_tags_from_environment_and_constant(self):
test_tags = ['country:china', 'age:45', 'blue']
constant_tags = ['country:canada', 'red']
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = ','.join(test_tags)
dog = ThreadStats(constant_tags=constant_tags)
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event2_title = "Event 1 title"
event1_text = "Event 1 text"
event2_text = "Event 2 text"
dog.event(event1_title, event1_text)
dog.event(event2_title, event2_text)
# Flush and test
dog.flush()
event1, event2 = reporter.events
nt.assert_equal(event1['title'], event1_title)
nt.assert_equal(event1['text'], event1_text)
nt.assert_equal(event1['tags'], constant_tags + test_tags)
nt.assert_equal(event2['title'], event2_title)
nt.assert_equal(event2['text'], event2_text)
nt.assert_equal(event2['text'], event2_text)
nt.assert_equal(event2['tags'], constant_tags + test_tags)
# Test more parameters
reporter.events = []
event1_priority = "low"
event1_date_happened = 1375296969
event1_tag = "Event 2 tag"
dog.event(event1_title,
event1_text,
priority=event1_priority,
date_happened=event1_date_happened,
tags=[event1_tag])
# Flush and test
dog.flush()
event, = reporter.events
nt.assert_equal(event['title'], event1_title)
nt.assert_equal(event['text'], event1_text)
nt.assert_equal(event['priority'], event1_priority)
nt.assert_equal(event['date_happened'], event1_date_happened)
nt.assert_equal(event['tags'],
[event1_tag] + constant_tags + test_tags)
dog.start(flush_interval=1, roll_up_interval=1)
def test_tags_from_environment_and_constant(self):
test_tags = ['country:china', 'age:45', 'blue']
constant_tags = ['country:canada', 'red']
with preserve_environment_variable('DATADOG_TAGS'):
os.environ['DATADOG_TAGS'] = ','.join(test_tags)
dog = ThreadStats(constant_tags=constant_tags)
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event2_title = "Event 1 title"
event1_text = "Event 1 text"
event2_text = "Event 2 text"
dog.event(event1_title, event1_text)
dog.event(event2_title, event2_text)
# Flush and test
dog.flush()
event1, event2 = reporter.events
nt.assert_equal(event1['title'], event1_title)
nt.assert_equal(event1['text'], event1_text)
nt.assert_equal(event1['tags'], constant_tags + test_tags)
nt.assert_equal(event2['title'], event2_title)
nt.assert_equal(event2['text'], event2_text)
nt.assert_equal(event2['text'], event2_text)
nt.assert_equal(event2['tags'], constant_tags + test_tags)
# Test more parameters
reporter.events = []
event1_priority = "low"
event1_date_happened = 1375296969
event1_tag = "Event 2 tag"
dog.event(event1_title, event1_text, priority=event1_priority,
date_happened=event1_date_happened, tags=[event1_tag])
# Flush and test
dog.flush()
event, = reporter.events
nt.assert_equal(event['title'], event1_title)
nt.assert_equal(event['text'], event1_text)
nt.assert_equal(event['priority'], event1_priority)
nt.assert_equal(event['date_happened'], event1_date_happened)
nt.assert_equal(event['tags'], [event1_tag] + constant_tags + test_tags)
dog.start(flush_interval=1, roll_up_interval=1)
def test_event_constant_tags(self):
constant_tag = 'type:constant'
dog = ThreadStats(constant_tags=[constant_tag])
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event2_title = "Event 1 title"
event1_text = "Event 1 text"
event2_text = "Event 2 text"
dog.event(event1_title, event1_text)
dog.event(event2_title, event2_text)
# Flush and test
dog.flush()
event1, event2 = reporter.events
nt.assert_equal(event1['title'], event1_title)
nt.assert_equal(event1['text'], event1_text)
nt.assert_equal(event1['tags'], [constant_tag])
nt.assert_equal(event2['title'], event2_title)
nt.assert_equal(event2['text'], event2_text)
nt.assert_equal(event2['text'], event2_text)
nt.assert_equal(event2['tags'], [constant_tag])
# Test more parameters
reporter.events = []
event1_priority = "low"
event1_date_happened = 1375296969
event1_tag = "Event 2 tag"
dog.event(event1_title,
event1_text,
priority=event1_priority,
date_happened=event1_date_happened,
tags=[event1_tag])
# Flush and test
dog.flush()
event, = reporter.events
nt.assert_equal(event['title'], event1_title)
nt.assert_equal(event['text'], event1_text)
nt.assert_equal(event['priority'], event1_priority)
nt.assert_equal(event['date_happened'], event1_date_happened)
nt.assert_equal(event['tags'], [event1_tag, constant_tag])
def test_event_constant_tags(self):
constant_tag = "type:constant"
dog = ThreadStats(constant_tags=[constant_tag])
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event2_title = "Event 1 title"
event1_text = "Event 1 text"
event2_text = "Event 2 text"
dog.event(event1_title, event1_text)
dog.event(event2_title, event2_text)
# Flush and test
dog.flush()
event1, event2 = reporter.events
nt.assert_equal(event1["title"], event1_title)
nt.assert_equal(event1["text"], event1_text)
nt.assert_equal(event1["tags"], [constant_tag])
nt.assert_equal(event2["title"], event2_title)
nt.assert_equal(event2["text"], event2_text)
nt.assert_equal(event2["text"], event2_text)
nt.assert_equal(event2["tags"], [constant_tag])
# Test more parameters
reporter.events = []
event1_priority = "low"
event1_date_happened = 1375296969
event1_tag = "Event 2 tag"
dog.event(
event1_title, event1_text, priority=event1_priority, date_happened=event1_date_happened, tags=[event1_tag]
)
# Flush and test
dog.flush()
event, = reporter.events
nt.assert_equal(event["title"], event1_title)
nt.assert_equal(event["text"], event1_text)
nt.assert_equal(event["priority"], event1_priority)
nt.assert_equal(event["date_happened"], event1_date_happened)
nt.assert_equal(event["tags"], [event1_tag, constant_tag])
def test_event(self):
dog = ThreadStats()
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event2_title = "Event 1 title"
event1_text = "Event 1 text"
event2_text = "Event 2 text"
dog.event(event1_title, event1_text)
dog.event(event2_title, event2_text)
# Flush and test
dog.flush()
event1, event2 = reporter.events
assert event1['title'] == event1_title
assert event1['text'] == event1_text
assert event2['title'] == event2_title
assert event2['text'] == event2_text
# Test more parameters
reporter.events = []
event1_priority = "low"
event1_date_happened = 1375296969
event1_tag = "Event 2 tag"
dog.event(event1_title,
event1_text,
priority=event1_priority,
date_happened=event1_date_happened,
tags=[event1_tag])
# Flush and test
dog.flush()
event, = reporter.events
assert event['title'] == event1_title
assert event['text'] == event1_text
assert event['priority'] == event1_priority
assert event['date_happened'] == event1_date_happened
assert event['tags'] == [event1_tag]
def test_tags_from_environment_env_service_version(self):
test_tags = set(['env:staging', 'service:food', 'version:1.2.3'])
with EnvVars(env_vars={
"DD_ENV": "staging",
"DD_VERSION": "1.2.3",
"DD_SERVICE": "food",
}):
dog = ThreadStats()
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event1_text = "Event 1 text"
dog.event(event1_title, event1_text)
# Flush and test
dog.flush()
[event1] = reporter.events
assert event1['title'] == event1_title
assert event1['text'] == event1_text
assert set(event1['tags']) == test_tags
def test_event(self):
dog = ThreadStats()
dog.start(roll_up_interval=10, flush_in_thread=False)
reporter = dog.reporter = MemoryReporter()
# Add two events
event1_title = "Event 1 title"
event2_title = "Event 1 title"
event1_text = "Event 1 text"
event2_text = "Event 2 text"
dog.event(event1_title, event1_text)
dog.event(event2_title, event2_text)
# Flush and test
dog.flush()
event1, event2 = reporter.events
nt.assert_equal(event1['title'], event1_title)
nt.assert_equal(event1['text'], event1_text)
nt.assert_equal(event2['title'], event2_title)
nt.assert_equal(event2['text'], event2_text)
# Test more parameters
reporter.events = []
event1_priority = "low"
event1_date_happened = 1375296969
event1_tag = "Event 2 tag"
dog.event(event1_title, event1_text, priority=event1_priority,
date_happened=event1_date_happened, tags=[event1_tag])
# Flush and test
dog.flush()
event, = reporter.events
nt.assert_equal(event['title'], event1_title)
nt.assert_equal(event['text'], event1_text)
nt.assert_equal(event['priority'], event1_priority)
nt.assert_equal(event['date_happened'], event1_date_happened)
nt.assert_equal(event['tags'], [event1_tag])
class OpenvpnMonitor():
def __init__(self,
monitor_host,
monitor_port,
interval,
datadog=True,
elstic=False):
self.host = monitor_host
self.port = monitor_port
self.interval = interval
self.s = None
self.datadog = datadog
self.init_datadog()
self.stats = ThreadStats()
self.stats.start(flush_interval=interval, flush_in_thread=False)
self.tags = ['server:{}'.format(os.uname()[1]), 'type:openvpn']
def connect(self):
try:
self.s = socket.create_connection((self.host, self.port), 2)
except:
print('Unable to connect')
sys.exit()
def init_datadog(self):
options = {
'api_key': os.getenv('DD_API_KEY'),
'app_key': os.getenv('DD_APP_KEY')
}
initialize(**options)
logging.basicConfig(level=logging.DEBUG)
def flush_datadog(self):
self.stats.flush()
def disconnect(self):
self.s.send('quit\n'.encode('ascii'))
self.s.shutdown(socket.SHUT_RDWR)
self.s.close()
def get_loadstats(self):
self.s.send('load-stats\n'.encode('ascii'))
return self.get_data()
def get_status(self):
self.s.send('status 2\n'.encode('ascii'))
return self.get_data()
def get_version(self):
self.s.send('version\n'.encode('ascii'))
return self.get_data()
def get_data(self):
socket_list = [self.s]
read_sockets, write_sockets, error_sockets = select.select(
socket_list, [], [])
for sock in read_sockets:
data = sock.recv(65565)
return data.decode('utf8')
def parse_version(self, version, datadog=True, elastic=False):
"""OpenVPN Version: OpenVPN 2.4.3 x86_64-redhat-linux-gnu [Fedora EPEL patched] [SSL (OpenSSL)] [LZO] [LZ4] [EPOLL] [PKCS11] [MH/PKTINFO] [AEAD] built on Jun 21 2017
OpenVPN Version: OpenVPN 2.3.14 x86_64-alpine-linux-musl [SSL (OpenSSL)] [LZO] [EPOLL] [MH] [IPv6] built on Dec 18 2016"""
ver = version.split(" ")
tags = ["version:{}_{}".format(ver[2], ver[3])]
self.tags += tags
def parse_loadstats(self, loadstats, datadog=True, elastic=False):
pattern = re.compile(
r"SUCCESS:.*nclients=(?P<nclients>\d*),bytesin=(?P<bytesin>\d*),bytesout=(?P<bytesout>\d*).*"
)
for line in loadstats.splitlines():
o_stats = pattern.match(line)
if o_stats:
if self.datadog:
self.stats.gauge('openvpn.nclients',
int(o_stats.group('nclients')),
tags=self.tags)
self.stats.gauge('openvpn.bytesin',
int(o_stats.group('bytesin')),
tags=self.tags)
self.stats.gauge('openvpn.bytesout',
int(o_stats.group('bytesout')),
tags=self.tags)
def parse_status(self, status):
"""HEADER,CLIENT_LIST,Common Name,Real Address,Virtual Address,Bytes Received,Bytes Sent,Connected Since,Connected Since (time_t),Username
HEADER,CLIENT_LIST,Common Name,Real Address,Virtual Address,Virtual IPv6 Address,Bytes Received,Bytes Sent,Connected Since,Connected Since (time_t),Username,Client ID,Peer ID
CLIENT_LIST,globbi,192.168.1.112:56513,10.8.0.18,,2735402,5955826,Sun Oct 1 20:15:18 2017,1506888918,jakobant,36,1"""
COMMONNAME = 1
REAL_ADDR = 2
VIRT_ADDR = 3
BYTESIN = 5 # 4
BYTESOUT = 6 # 5
USERNAME = 9 # 8
CONN_SINCET = 8 # 7
for line in status.splitlines():
if line.startswith('CLIENT_LIST'):
o_stats = line.split(',')
if len(o_stats) < 10:
BYTESIN = 4 # 4
BYTESOUT = 5 # 5
USERNAME = 8 # 8
CONN_SINCET = 7 # 7
if self.datadog:
tags = [
'commonname:{}'.format(
o_stats[COMMONNAME]), 'real_addr:{}'.format(
o_stats[REAL_ADDR].split(":")[0]),
'virt_addr:{}'.format(o_stats[VIRT_ADDR]),
'username:{}'.format(o_stats[USERNAME])
] + self.tags
connected_time = int(time.time()) - int(
o_stats[CONN_SINCET])
self.stats.gauge('openvpn.client.bytesin',
int(o_stats[BYTESIN]),
tags=tags)
self.stats.gauge('openvpn.client.bytesout',
int(o_stats[BYTESOUT]),
tags=tags)
self.stats.gauge('openvpn.client.conntime',
connected_time,
tags=tags)
def tail_log(self, logfile):
"""Fri Sep 29 21:29:59 2017 192.168.1.112:62493 TLS: Username/Password authentication succeeded for username 'jakobant'
Fri Sep 29 21:31:57 2017 192.168.1.112:62787 VERIFY OK: depth=1, C=IS, ST=Rkv, L=Reykjavik, O=Heima, OU=Ops, CN=Heima CA, name=EasyRSA, [email protected]
Fri Sep 29 21:31:57 2017 192.168.1.112:62787 VERIFY OK: depth=0, C=IS, ST=Rkv, L=Reykjavik, O=Heima, OU=Ops, CN=globbi, name=EasyRSA, [email protected]
AUTH-PAM: BACKGROUND: user 'jakobant' failed to authenticate: Authentication failure"""
login = re.compile(r".*authentication succeeded.*")
faild_login = re.compile(
r".*(failed to authenticate|Incorrect password|was not found).*")
for line in Pygtail(logfile):
match = login.match(line)
if match:
print(line)
self.stats.event('Login success',
line,
alert_type='success',
tags=self.tags)
match = faild_login.match(line)
if match:
print(line)
self.stats.event('Authentication failure',
line,
alert_type='error',
tags=self.tags)
