-
Notifications
You must be signed in to change notification settings - Fork 4
/
ldapChangeMonitor.py
executable file
·486 lines (420 loc) · 17.4 KB
/
ldapChangeMonitor.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2014 Mozilla Corporation
# Contributor: gdestuynder@mozilla.com
# Contributor: jbryner@mozilla.com
import copy
import os
import sys
import re
import json
import glob
import string
import ConfigParser
import tempfile
import logging
import socket
from logging.handlers import SysLogHandler
from requests import Session
from optparse import OptionParser
from datetime import datetime
from dateutil.tz import tzlocal
from os import stat
from os.path import exists, getsize
from ldif import LDIFRecordList, MOD_OP_STR
"""
Script to monitor changes in an openldap .ldif audit log
Uses the pygtail class to tail lines and follow rotated files
Reports changes to syslog as a standard syslog message
Uses the standard python ldif library wih an overridden parse class to catch all changes.
(The original ldif library doesn't report object deletions and is difficult navigate to report changes.)
"""
class MozDefError(Exception):
def __init__(self, msg):
self.msg = msg
def __str__(self):
return repr(self.msg)
class MozDefEvent():
# create requests session to allow for keep alives
httpsession = Session()
# Turns off needless and repetitive .netrc check for creds
httpsession.trust_env = False
debug = False
verify_certificate = True
# Never fail (ie no unexcepted exceptions sent to user, such as server/network not responding)
fire_and_forget_mode = True
log = {}
log['timestamp'] = datetime.isoformat(datetime.now(tzlocal()))
log['hostname'] = socket.getfqdn()
log['processid'] = os.getpid()
log['processname'] = sys.argv[0]
log['severity'] = 'INFO'
log['summary'] = None
log['category'] = 'event'
log['tags'] = list()
log['details'] = dict()
def __init__(self, url='http://localhost/events', summary=None, category='event', severity='INFO', tags=[], details={}):
self.summary = summary
self.category = category
self.severity = severity
self.tags = tags
self.details = details
self.url = url
def send(self, summary=None, category=None, severity=None, tags=None, details=None):
log_msg = copy.copy(self.log)
if summary is None:
log_msg['summary'] = self.summary
else:
log_msg['summary'] = summary
if category is None:
log_msg['category'] = self.category
else:
log_msg['category'] = category
if severity is None:
log_msg['severity'] = self.severity
else:
log_msg['severity'] = severity
if tags is None:
log_msg['tags'] = self.tags
else:
log_msg['tags'] = tags
if details is None:
log_msg['details'] = self.details
else:
log_msg['details'] = details
if type(log_msg['details']) != dict:
raise MozDefError('details must be a dict')
elif type(log_msg['tags']) != list:
raise MozDefError('tags must be a list')
elif summary is None:
raise MozDefError('Summary is a required field')
if self.debug:
print(json.dumps(log_msg, sort_keys=True, indent=4))
try:
r = self.httpsession.post(self.url, json.dumps(log_msg), verify=self.verify_certificate)
except Exception as e:
if not self.fire_and_forget_mode:
raise e
class mzLDIFRecordList(LDIFRecordList):
def parse(self):
"""
Continously read and parse LDIF records
assumes records start with
# <changetype>
and end with
# end <changetype>
builds a python structure consisting of:
a list:
list[0] is the dn of the item being changed
list[1] is a dictionary consisting of several keys:
list[1]['dn'] is a repeat of the dn for ease of access
list[1]['actions'] is a summary list of the actions taken against the dn
list[1]['changes'] is a detailed list of all changes (attribute/value pairs)
retrieve these with:
changes=list[1]['changes']
changepairs=zip(changes,changes[1:])[::2]
actions=list[1]['actions']
actionpairs=zip(actions,actions[1:])[::2]
"""
beginActionRe = re.compile(r"""# (add|change|delete|modify) ([0-9]{1,100}) (.*)""", re.IGNORECASE)
self._line = self._input_file.readline()
dn = None
changetype = None
entry = {}
action = ''
actor = ''
while self._line and (not self._max_entries
or self.records_read < self._max_entries):
# for deletes the modifier is on the beginning comment line
# read it before parseAttrTypeAndValue which ignores comments and folds lines
if beginActionRe.match(self._line):
actor = beginActionRe.search(self._line).groups()[2]
entry['actor'] = actor
attr_type, attr_value = self._parseAttrTypeandValue()
if attr_type == 'dn':
dn = attr_value
entry['dn'] = dn
elif attr_type in MOD_OP_STR.values():
# this is an action (add/delete/replace)
action = attr_type
if 'actions' in entry.keys():
entry['actions'] += [attr_type, attr_value]
else:
entry['actions'] = [attr_type, attr_value]
entry['actions']
elif attr_type == 'changetype':
# generally used to denote the type of change to the dn ('add/delete/modify')
changetype = attr_value
entry['changetype'] = changetype
elif attr_value != None and not self._ignored_attr_types.has_key(attr_type.lower()):
# this is an attribute/value pair of a change:
# telephonenumber: +1 408 555 1234
# since there can be many of these, this is stored as a list
# since there can be adds/deletes/replaces of many of these the list includes the
# action:attributeName as the [0] item in the value/pair
if action == '':
# adding a new dn can have no attribute action
action = changetype
if attr_type == 'modifiersName':
entry['actor'] = attr_value
if 'changes' in entry.keys():
entry['changes'] += (action + ':' + attr_type, attr_value)
else:
entry['changes'] = (action + ':' + attr_type, attr_value)
# append entry to result list
if dn != None and len(entry) > 0 and "# end" in self._line:
if 'actor' not in entry.keys():
# we didn't find an actor..set the default
entry['actor'] = 'unknown'
self.handle(dn, entry)
self.records_read = self.records_read + 1
# reset record
dn = None
changetype = None
entry = {}
action
actor = ''
return
def createLogRecord(dictIn):
# make an event message that can be used
# for syslog, text, json, etc.
log = {}
log['category'] = 'ldapChange'
log['summary'] = '{0} {1} {2} '.format(dictIn['actor'], dictIn['changetype'], dictIn['dn'])
log['details'] = dict()
log['details']['actor'] = dictIn['actor']
log['details']['changetype'] = dictIn['changetype']
log['details']['dn'] = dictIn['dn']
# gather the actions and change lists into pairs of action,value and action:attribute,value
if 'actions' in dictIn.keys():
actionpairs = zip(dictIn['actions'], dictIn['actions'][1:])[::2]
changepairs = zip(dictIn['changes'], dictIn['changes'][1:])[::2]
log['details']['actionpairs'] = actionpairs
log['details']['changepairs'] = changepairs
# what to show in the summary field?
if ('member' in dictIn['actions']) or 'memberUid' in dictIn['actions']:
# likely a group membership change (add or delete)
for a, v in actionpairs:
if v in ('member', 'memberUid'):
for ca, cv in changepairs:
if ca == a + ':' + v:
if ' {0}: {1} '.format(ca, cv) not in log['summary']:
log['summary'] += ' {0}: {1} '.format(ca, cv)
else:
# default message logs action pairs
for action, value in actionpairs:
log['summary'] += '{0} {1} '.format(action, value)
log['timestamp'] = datetime.isoformat(datetime.now(tzlocal()))
return(log)
class Pygtail(object):
"""
Creates an iterable object that returns only unread lines.
"""
def __init__(self, filename, offset_file=None, paranoid=False, pretend=False):
self.filename = filename
self.paranoid = paranoid
self._offset_file = offset_file or "%s.offset" % self.filename
self._offset_file_inode = 0
self._offset = 0
self._fh = None
self._rotated_logfile = None
self.pretend = pretend
# if offset file exists and non-empty, open and parse it
if exists(self._offset_file) and getsize(self._offset_file):
offset_fh = open(self._offset_file, "r")
(self._offset_file_inode, self._offset) = \
[int(line.strip()) for line in offset_fh]
offset_fh.close()
if self._offset_file_inode != stat(self.filename).st_ino:
# The inode has changed, so the file might have been rotated.
# Look for the rotated file and process that if we find it.
self._rotated_logfile = self._determine_rotated_logfile()
def __del__(self):
if self._filehandle():
self._filehandle().close()
def __iter__(self):
return self
def next(self):
"""
Return the next line in the file, updating the offset.
"""
try:
line = next(self._filehandle())
except StopIteration:
# we've reached the end of the file; if we're processing the
# rotated log file, we can continue with the actual file; otherwise
# update the offset file
if self._rotated_logfile:
self._rotated_logfile = None
self._fh.close()
self._offset = 0
self._update_offset_file()
# open up current logfile and continue
try:
line = next(self._filehandle())
except StopIteration: # oops, empty file
self._update_offset_file()
raise
else:
self._update_offset_file()
raise
if self.paranoid:
self._update_offset_file()
return line
def __next__(self):
"""`__next__` is the Python 3 version of `next`"""
return self.next()
def readlines(self):
"""
Read in all unread lines and return them as a list.
"""
return [line for line in self]
def read(self):
"""
Read in all unread lines and return them as a single string.
"""
lines = self.readlines()
if lines:
return ''.join(lines)
else:
return None
def _filehandle(self):
"""
Return a filehandle to the file being tailed, with the position set
to the current offset.
"""
if not self._fh or self._fh.closed:
filename = self._rotated_logfile or self.filename
self._fh = open(filename, "r")
self._fh.seek(self._offset)
return self._fh
def _update_offset_file(self):
"""
Update the offset file with the current inode and offset.
"""
if not self.pretend:
offset = self._filehandle().tell()
inode = stat(self.filename).st_ino
fh = open(self._offset_file, "w")
fh.write("%s\n%s\n" % (inode, offset))
fh.close()
def _determine_rotated_logfile(self):
"""
We suspect the logfile has been rotated, so try to guess what the
rotated filename is, and return it.
"""
for rotated_filename in self._check_rotated_filename_candidates():
if exists(rotated_filename) and stat(rotated_filename).st_ino == self._offset_file_inode:
return rotated_filename
return None
def _check_rotated_filename_candidates(self):
"""
Check for various rotated logfile filename patterns and return the
matches we find.
"""
candidates = []
# savelog(8)
candidate = "%s.0" % self.filename
if (exists(candidate) and
exists("%s.1.gz" % self.filename) and
(stat(candidate).st_mtime >
stat("%s.1.gz" % self.filename).st_mtime)):
candidates.append(candidate)
# logrotate(8)
candidate = "%s.1" % self.filename
if exists(candidate):
candidates.append(candidate)
# dateext rotation scheme
for candidate in glob.glob("%s-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" % self.filename):
candidates.append(candidate)
# for TimedRotatingFileHandler
for candidate in glob.glob("%s.[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]" % self.filename):
candidates.append(candidate)
return candidates
def main():
if not exists(options.inputfile):
print('no file found')
return
if options.output == 'syslog':
logger = logging.getLogger()
logger.addHandler(SysLogHandler(address=(options.sysloghostname, options.syslogport), facility='local4'))
ptlines = 0
# take a look the file to see if it has a complete # begin # end multi-line
# structure..if not bail as we likely read it while it was being appended
bRecords = False
pt = Pygtail(options.inputfile, options.offsetfile, pretend=True)
temp = tempfile.NamedTemporaryFile(suffix='_ldif', delete=False)
for line in pt:
temp.write(line)
ptlines += 1
if "# end" in line:
bRecords = True
temp.close()
if ptlines == 0 or not bRecords:
os.unlink(temp.name)
elif ptlines > 0 and bRecords:
mdEvent = MozDefEvent(options.url)
mdEvent.debug = False
mdEvent.fire_and_forget_mode = False
# read and ignore certain noisy fields
l = mzLDIFRecordList(open(temp.name, 'rb'), ['jpegPhoto', 'lmPassword', 'ntPassword', 'userPassword', 'sshPublicKey', 'pwdHistory', 'other', 'description'])
l.parse()
temp.close()
os.unlink(temp.name)
pt.pretend = False
pt._update_offset_file()
for i in l.all_records:
log = createLogRecord(i[1])
# http post/syslog/stdout
if options.output == 'syslog':
logger.warn(log["summary"])
elif options.output == 'http':
# some changes have no details
logDetails = dict()
if 'details' in log:
logDetails = log['details']
mdEvent.send(summary=log['summary'],
category=log['category'],
tags=['ldap', 'ldif'],
details=logDetails)
elif options.output == 'stdout':
print(log['summary'])
def getConfig(optionname, thedefault, configfile):
"""read an option from a config file or set a default
send 'thedefault' as the data class you want to get a string back
i.e. 'True' will return a string
True will return a bool
1 will return an int
"""
retvalue = thedefault
opttype = type(thedefault)
if os.path.isfile(configfile):
config = ConfigParser.ConfigParser()
config.readfp(open(configfile))
if config.has_option('options', optionname):
if opttype == bool:
retvalue = config.getboolean('options', optionname)
elif opttype == int:
retvalue = config.getint('options', optionname)
elif opttype == float:
retvalue = config.getfloat('options', optionname)
else:
retvalue = config.get('options', optionname)
return retvalue
def initConfig(configfile):
# default options
options.format = getConfig('format', 'text', configfile)
options.inputfile = getConfig('inputfile', '', configfile)
options.output = getConfig('output', 'stdout', configfile)
options.sysloghostname = getConfig('sysloghostname', 'localhost', configfile)
options.syslogport = getConfig('syslogport', 514, configfile)
options.offsetfile = getConfig('offsetfile', 'ldapchangetail.offset', configfile)
options.url = getConfig('url', 'http://localhost:8080/events', configfile)
if __name__ == '__main__':
parser = OptionParser()
parser.add_option("-c", dest='configfile', default='', help="configuration file to use")
(options, args) = parser.parse_args()
initConfig(options.configfile)
main()