def __init__(self, *argz, **kwz):
super(NFLogDump, self).__init__(*argz, **kwz)
self.zmq_ctx = zmq.Context()
self.zmq_optz = self.conf.traffic_dump.nflog_pipe_interface
log.noise("Compiling regexes: {!r}".format(self.conf.patterns))
self.patterns = dict((pat, re.compile(pat)) for pat in self.conf.patterns)
self.last_dump = 0
def dispatch(self, msg, source=None):
if msg != self.conf.command:
log.noise('Ignoring unknown command: {!r} (source: {})'.format(msg, source))
return
if not self.conf.direct: source = None # reply to whatever destination channel
msg = '\n'.join(it.imap(self.handler.format, list(self.buffer)))
reactor.callLater( 0, self.interface.dispatch,
msg, source=self, user=source, direct=True )
def __init__(self, *argz, **kwz):
super(SnortLog, self).__init__(*argz, **kwz)
import zmq
self.zmq = zmq
self.zmq_ctx = zmq.Context()
self.zmq_optz = self.conf.traffic_dump.nflog_pipe_interface
log.noise('Compiling regex: {!r}'.format(self.conf.sig_match))
self.regex = re.compile(self.conf.sig_match)
def __init__(self, *argz, **kwz):
super(SnortRefs, self).__init__(*argz, **kwz)
log.noise('Compiling regex: {!r}'.format(self.conf.sig_match))
self.regex = re.compile(self.conf.sig_match)
self.gid_ignore = set(it.imap(str, ( [self.conf.gid_ignore]
if isinstance(self.conf.gid_ignore, (int, str)) else self.conf.gid_ignore )))\
if self.conf.gid_ignore else set()
self.sid_db_ts = 0
def handle_line(self, line, repo_lock=defer.DeferredLock()):
try:
line = line.decode('utf-8', 'ignore').strip()
match = re.search(r'(^|\s+)!pq\s+(?P<link>\S+)(\s+::\S+|$)', line)
if not match:
log.noise('Non-patchbot line, ignoring: {}'.format(line.encode('utf-8', 'ignore')))
defer.returnValue(None)
link = match.group('link').encode('ascii')
if not re.search('https?://', link, re.IGNORECASE):
log.warn('Incorrect non-http link, skipping: {}'.format(link))
defer.returnValue(None)
except UnicodeError as err:
log.warn('Failed to recode line ({!r}): {}'.format(line, err))
defer.returnValue(None)
## Grab the patch
dst_base = '{}.patch'.format(sha1(link).hexdigest())
dst_path = self.dst_path.child(dst_base)
if dst_path.exists():
log.debug( 'Patch already exists'
' (file: {}, link: {}), skipping'.format(dst_path, link) )
defer.returnValue(None)
# Not via tmpfile to prevent multiple downloads of the same paste
try: yield downloadPage(link, dst_path.open('wb'), timeout=120)
except:
if dst_path.exists(): dst_path.remove()
raise
## Commit into repo and push
yield repo_lock.acquire()
try:
for cmd, check in [
(['add', dst_base], True),
(['commit', '-m', 'New patch: {}'.format(link)], False),
(['push'], True) ]:
out, err, code = yield getProcessOutputAndValue(
'/usr/bin/git', cmd, path=self.dst_path.path )
if check and code:
log.error('\n'.join([
'Failed to commit/push new patch into repo',
'Command: {}'.format(cmd), 'Exit code: {}'.format(code),
'Stdout:\n {}'.format('\n '.join(out.splitlines())),
'Stderr:\n {}'.format('\n '.join(err.splitlines())) ]))
break
else: log.debug('Successfully pushed paste: {}'.format(link))
finally: repo_lock.release()
def __init__(self, *argz, **kwz):
super(FilterPipe, self).__init__(*argz, **kwz)
self.rules = OrderedDict()
for name, rule in self.conf.rules.viewitems():
if 'regex' in rule:
log.noise('Compiling filter (name: {}): {!r}'.format(name, rule.regex))
check = re.compile(rule.regex)
else: check = None # boolean rule
try: action, optz = rule.action.split('-', 1)
except ValueError: action, optz = rule.action, list()
else:
if action == 'limit': optz = map(int, optz.split('/'))
else: optz = [optz]
self.rules[name] = check, action, optz, rule.get('match')
self.rule_hits, self.rule_notes, self.rule_drops = dict(), set(), defaultdict(int)
def dispatch(self, msg):
for name, (check, action, optz, attr) in self.rules.viewitems():
try: msg_match = msg if not attr else (('{'+attr+'}').format(data=msg.data) or '')
except Exception as err:
log.noise('Filtering attr-get error ({}) for message: {!r}'.format(err, msg))
msg_match = ''
if not ( check.search(msg_match)
if check is not None else bool(msg_match) ):
if 'nomatch' in optz:
if action == 'allow': return msg
elif action == 'drop': return
continue
if action == 'limit':
if name not in self.rule_hits: self.rule_hits[name] = deque()
win, ts, (c, t) = self.rule_hits[name], time(), optz
ts_thresh = ts - t
win.append(ts)
while win[0] < ts_thresh: win.popleft()
rate = len(win)
if rate > c:
log.noise(( 'Rule ({}) triggering rate'
' above threshold ({}/{}): {}' ).format(name, c, t, rate))
self.rule_drops[name] += 1
if name not in self.rule_notes:
self.rule_notes.add(name)
return ( ' ...limiting messages matching'
' filter-rule {} ({}/{}, dropped (for uptime): {})' )\
.format(name, c, t, self.rule_drops[name])
else: return
self.rule_notes.discard(name)
return msg
elif 'nomatch' not in optz:
if action == 'allow': return msg
elif action == 'drop': return
if self.conf.policy == 'allow': return msg
def handle_change(self, stuff, path, mask):
mask_str = inotify.humanReadableMask(mask)
log.noise('Event: {} ({})'.format(path, mask_str))
## Filtering
path_real = path.realpath()
if not path_real.isfile():
log.debug( 'Ignoring event for'
' non-regular file: {} (realpath: {})'.format(path, path_real) )
return
dir_key = path_real.parent().realpath()
if dir_key not in self.paths_watch:
log.warn( 'Ignoring event for file outside of watched'
' set of paths: {} (realpath: {})'.format(path, path_real) )
return
for pat in self.paths_watch[dir_key]:
if fnmatch(bytes(path.basename()), pat): break
else:
log.noise( 'Non-matched path in one of'
' the watched dirs: {} (realpath: {})'.format(path, path_real) )
return
## Get last position
if self.paths_pos.get(path_real) is not None:
pos, size, data = self.paths_pos[path_real]
if self.file_end_check(path_real, pos, size=size, data=data):
log.debug(( 'Event (mask: {}) for unchanged'
' path, ignoring: {}' ).format(mask_str, path))
return
if path_real.getsize() < pos:
log.debug( 'Detected truncation'
' of a path, rewinding: {}'.format(path) )
pos = None
else: pos = None
## Actual processing
line = self.paths_buff.setdefault(path_real, '')
with path_real.open('rb') as src:
if pos:
src.seek(pos)
pos = None
while True:
buff = src.readline()
if not buff: # eof
self.paths_pos[path_real] = self.file_end_mark(path_real, data=line)
line += buff
if line.endswith('\n'):
log.noise('New line (source: {}): {!r}'.format(path, line))
reactor.callLater(0, self.handle_line, line)
line = self.paths_buff[path_real] = ''
else:
line, self.paths_buff[path_real] = None, line
break
def update(self, relays, channels, routes):
def resolve(route, k, fork=False, lvl=0):
if k not in route: route[k] = list()
elif isinstance(route[k], types.StringTypes): route[k] = [route[k]]
modules = list()
for v in route[k]:
if v not in routes: modules.append(v)
else:
for subroute in routes[v]:
if fork is None:
resolve(subroute, k, lvl=lvl+1)
modules.extend(subroute[k])
else:
fork = route.clone()
fork.pipe = (list(fork.pipe) + subroute.pipe)\
if fork is True else (subroute.pipe + list(fork.pipe))
resolve(subroute, k, fork=True, lvl=lvl+1)
fork[k] = subroute[k]
routes[route.name].append(fork)
route[k] = modules
for name, route in routes.viewitems():
if not route.get('pipe'): route.pipe = list()
route.name = name
routes[name] = [route]
for k, fork in ('pipe', None), ('src', True), ('dst', False):
for name, route_set in routes.items():
for route in route_set:
if k == 'pipe':
for v in route.pipe or list():
if v in channels:
log.fatal( 'Channels are not allowed'
' in route.pipe sections (route: {}, channel: {})'.format(name, v) )
sys.exit(1)
resolve(route, k, fork=fork)
pipes, pipes_chk = dict(), set()
pipes_valid = set(relays).union(channels)
for route in it.chain.from_iterable(routes.viewvalues()):
if not route.src or not route.dst: continue
for src, dst in it.product(route.src, route.dst):
pipe = tuple([src] + route.pipe + [dst])
if pipe in pipes_chk: continue
for v in pipe:
if v not in pipes_valid:
log.fatal('Unknown route component (route: {}): {}'.format(route.name, v))
sys.exit(1)
pipes_chk.add(pipe) # to eliminate duplicates
pipes.setdefault(src, list()).append((dst, route.pipe))
log.noise('Pipelines (by src): {}'.format(pipes))
# Add reverse (obj -> name) mapping to relays
for name, relay_obj in relays.items(): relays[relay_obj] = name
self.relays, self.channels, self.routes = relays.copy(), channels.copy(), pipes
# Remove channels that aren't used in any of the routes
self.channel_map, channels = dict(), set()
for src, routes in self.routes.viewitems():
channels.add(src)
for dst, pipe in routes: channels.add(dst)
for channel in list(self.channels):
if channel not in channels:
log.debug('Ignoring channel, not used in any of the routes: {}'.format(channel))
del self.channels[channel]
else:
alias, channel = channel, self.channels[channel]
name = channel.get('name') or alias
self.channel_map[name] = alias
def __init__(self, *argz, **kwz):
super(Resolver, self).__init__(*argz, **kwz)
log.noise('Compiling regex: {!r}'.format(self.conf.addr))
self.regex = re.compile(self.conf.addr)
def handle_line(self, line, path):
log.noise('New line: {!r}'.format(line))
event = RelayedEvent(force_unicode(line))
event.data = AttrDict(path=path.path)
reactor.callLater(0, self.interface.dispatch, event, source=self)
def handle_change(self, stuff, path, mask):
mask_str = inotify.humanReadableMask(mask)
log.noise('Event: {} ({})'.format(path, mask_str))
## Filtering
path_real = path.realpath()
if not path_real.isfile():
log.debug( 'Ignoring event for'
' non-regular file: {} (realpath: {})'.format(path, path_real) )
return
dir_key = path_real.parent().realpath()
if dir_key not in self.paths_watch:
log.warn( 'Ignoring event for file outside of watched'
' set of paths: {} (realpath: {})'.format(path, path_real) )
return
for pat in self.paths_watch[dir_key]:
if fnmatch(bytes(path.basename()), pat): break
else:
log.noise( 'Non-matched path in one of'
' the watched dirs: {} (realpath: {})'.format(path, path_real) )
return
for pat in self.exclude:
if pat.search(path.path):
log.noise( 'Matched path by exclude-pattern'
' ({}): {} (realpath: {})'.format(pat, path, path_real) )
return
## Get last position
pos = self.paths_pos.get(path_real)
if not pos: # try restoring it from xattr
try: pos = pickle.loads(xattr(path_real.path)[self.conf.xattr_name])
except KeyError:
log.debug('Failed to restore last log position from xattr for path: {}'.format(path))
else:
log.noise(
'Restored pos from xattr ({}) for path {}: {!r}'\
.format(self.conf.xattr_name, path_real, pos) )
if pos:
pos, size, data_hash = pos
if self.file_end_check(path_real, pos, size=size, data_hash=data_hash):
log.noise(( 'Event (mask: {}) for unchanged'
' path, ignoring: {}' ).format(mask_str, path))
return
if path_real.getsize() < pos:
log.debug( 'Detected truncation'
' of a path, rewinding: {}'.format(path) )
pos = None
## Actual processing
buff_agg = self.paths_buff.setdefault(path_real, '')
with path_real.open() as src:
if pos:
src.seek(pos)
pos = None
while True:
pos = src.tell()
try: buff, pos = self.read(src), src.tell()
except StopIteration:
buff_agg = ''
src.seek(pos) # revert back to starting position
buff, pos = self.read(src), src.tell()
if not buff: # eof, try to mark the position
if not buff_agg: # clean eof at the end of the chunk - mark it
pos = self.file_end_mark(path_real, pos=pos, data=buff_agg)
self.paths_pos[path_real] = pos
xattr(path_real.path)[self.conf.xattr_name] = pickle.dumps(pos)
log.noise( 'Updated xattr ({}) for path {} to: {!r}'\
.format(self.conf.xattr_name, path_real, pos) )
break
buff_agg = self.paths_buff[path_real] = self.process(buff_agg + buff, path)
def schedule_fetch(self, url, fast=False):
interval = self.feeds[url].interval
jitter = interval.jitter * interval.base * random.random()
interval = jitter if fast else (interval.base + (jitter * random.choice([-1, 1])))
log.noise('Scheduling fetch for feed (url: {}) in {}s'.format(url, interval))
reactor.callLater(interval, self.fetch_feed, url)
def dispatch(self, msg):
if not msg.strip(): return
## Event lines are cached until EOE msg is encountered
match = self._re_base.search(msg)
if not match:
log.warn('Failed to match audit event spec: {!r}'.format(msg))
return
node, ev_id, ev_type, msg = (match.group(k) for k in ['node', 'ev_id', 'type', 'msg'])
ev_key = node, ev_id
if ev_key not in self._ev_cache:
self._ev_cache[ev_key] = defaultdict(list)
self._ev_cache[ev_key].update(ts=time.time(), node=node, ev_id=ev_id)
self._ev_cache_gc()
ev = self._ev_cache[ev_key]
if ev_type != 'EOE': # cache event data
ev[ev_type].append(msg)
return
del self._ev_cache[ev_key]
## Get "key" value for event, if present
ev_key = None
try: syscall, = ev['SYSCALL'] # currently handled events always have it
except ValueError: pass
else:
try: ev_key = self.get_msg_val(syscall, 'key', ur'"(?P<val>[^"]+)"')
except KeyError as err:
log.noise('Failed to get ev_key from syscall: {}'.format(err))
if not ev_key:
log.noise('Unhandled event: {!r}'.format(ev))
return
## Processing
if ev_key in self.conf.events.watches.ev_keys:
# Extract all necessary attributes
ev_vals = dict(node=ev['node'], ev_id=ev['ev_id'], key=ev_key)
for k in it.imap(''.join, it.product(['', 'e', 's', 'fs'], ['uid', 'gid'])):
ev_vals[k] = self.get_msg_val(syscall, k)
for k in 'comm', 'exe':
ev_vals[k] = self.get_msg_val(syscall, k, ur'"(?P<val>[^"]+)"')
ev_vals['tty'] = self.get_msg_val(syscall, 'tty', '(?P<val>\S+)')
paths = ev_vals['paths'] = list()
for msg in ev['PATH']:
path = self.get_msg_val(msg, 'name', ur'(?P<val>"[^"]+"|\(null\)|[0-9A-F]+)')
paths.append(dict( path=path,
inode=self.get_msg_val(msg, 'inode', fallback='nil'),
dev=self.get_msg_val(msg, 'dev', '(?P<val>[a-f\d]{2}:[a-f\d]{2})', fallback='nil') ))
# Formatting
err, tpl = None, force_unicode(self.conf.events.watches.template_path)
ev_vals['paths'] = list()
for val in paths:
try: ev_vals['paths'].append(tpl.format(**val))
except self._lookup_error as err: break
if not err:
ev_vals['paths'] = ', '.join(ev_vals['paths'])
tpl, val = force_unicode(self.conf.events.watches.template), ev_vals
try: event = tpl.format(**val)
except self._lookup_error as err: pass
event = RelayedEvent(event)
event.data = ev_vals
return event
raise ValueError( 'Failed to format template {!r} (data: {}): {}'.format(tpl, val, err))
def __init__(self, *argz, **kwz):
super(PkgMon, self).__init__(*argz, **kwz)
log.noise('Compiling regexes: {!r}'.format(self.conf.seek))
self.seek = map(re.compile, self.conf.seek)
def request(self, url, method='get', decode=None, encode=None, data=None):
method, url = force_bytes(method).upper(), force_bytes(url)
headers = {'User-Agent': self.user_agent}
if method == 'GET' and self.use_cache_headers:
# Avoid doing extra work
cache = self.fetch_cache.get(url, dict())
if 'cache-control' in cache and cache['cache-control'] >= time.time():
defer.returnValue(None) # no need to re-process same thing
if 'last-modified' in cache:
headers['If-Modified-Since'] = rfc822date(cache['last-modified'])
if 'etag' in cache: headers['If-None-Match'] = '"{}"'.format(cache['etag'])
log.noise( 'HTTP request: {} {} (h: {}, enc: {}, dec: {}, data: {!r})'\
.format(method, url[:100], headers, encode, decode, type(data)) )
if data is not None:
if encode is None:
if isinstance(data, types.StringTypes): data = io.BytesIO(data)
elif encode == 'form':
headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
data = io.BytesIO(urlencode(data))
elif encode == 'json':
headers.setdefault('Content-Type', 'application/json')
data = io.BytesIO(json.dumps(data))
else: raise ValueError('Unknown request encoding: {}'.format(encode))
data_raw, data = data, FileBodyProducer(data)
else: data_raw = None
if decode not in ['json', None]:
raise ValueError('Unknown response decoding method: {}'.format(decode))
requests = None # indicates fallback to requests module (for e.g. ipv6-only site)
err = None
try:
res = yield self.request_agent.request( method, url,
Headers(dict((k,[v]) for k,v in (headers or dict()).viewitems())), data )
except error.DNSLookupError:
import requests, socket
try:
res = yield self.sync_wrap(
getattr(requests, method.lower()), url, headers=headers, data=data_raw )
except ( socket.error, SyncTimeout,
requests.exceptions.RequestException ) as err: pass
except ( RequestTransmissionFailed,
RequestNotSent, ResponseFailed ) as err: pass
if err:
if not self.hide_connection_errors:
raise HTTPClientError(None, 'Lookup/connection error: {}'.format(err))
else:
log.debug('Lookup/connection error (supressed): {}'.format(err))
defer.returnValue(None) # should also supress fast refetching
code, phrase, version = (res.code, res.phrase, res.version)\
if not requests else ( res.status_code,
http.RESPONSES[res.status_code], ('HTTP', 1, 1) )
log.noise( 'HTTP request done ({} {}): {} {} {}'\
.format(method, url[:100], code, phrase, version) )
if code in [http.NO_CONTENT, http.NOT_MODIFIED]: defer.returnValue(None)
if code not in [http.OK, http.CREATED]: raise HTTPClientError(code, phrase)
if not requests:
data = defer.Deferred()
res.deliverBody(DataReceiver(data))
data = yield data
headers = dict((k, v[-1]) for k,v in res.headers.getAllRawHeaders())
else:
try:
data = yield self.sync_wrap(getattr, res, 'text')
headers = yield self.sync_wrap(getattr, res, 'headers')
except (requests.exceptions.RequestException, SyncTimeout) as err:
raise HTTPClientError(None, 'Sync connection error: {}'.format(err))
if method == 'GET' and self.use_cache_headers:
cache = dict((k.lower(), v) for k,v in headers.items())
cache = dict( (k, cache[k]) for k in
['last-modified', 'cache-control', 'etag'] if k in cache )
# Update headers' cache
if 'last-modified' in cache:
ts = rfc822.parsedate_tz(cache['last-modified'])
cache['last-modified'] = time.mktime(ts[:9]) + (ts[9] or 0)
if 'cache-control' in cache:
match = re.search(r'\bmax-age=(\d+)\b', cache.pop('cache-control'))
if match: cache['cache-control'] = time.time() + int(match.group(1))
if cache: self.fetch_cache[url] = cache
defer.returnValue((json.loads(data) if decode is not None else data, headers))
def dispatch(self, msg, source, user=None, direct=False):
if not isinstance(msg, list): msg = [msg]
channels = dict()
if direct and user:
# Direct reply
log.noise('Dispatching msg from {!r} directly to user: {!r}'.format(source, user))
channels[user] = msg
else:
try: route = self.routes[self.relays.get(source) or source]
except KeyError:
log.noise('No routes to dispatch message to, dropping: {!r}'.format(msg))
return
# Pull msg through all the pipelines and build dst channels / msgs buffer
for dst, pipe in route:
msg_copy = list(msg)
for name in pipe:
relay = self.relays[name]
results = yield defer.DeferredList(list(
defer.maybeDeferred(relay.dispatch, part) for part in msg_copy ))
msg_copy = set()
for chk, result in results:
if not chk:
log.error(
'Detected pipeline failure (src: {}, dst: {}, pipe: {}, relay: {}, msg: {}): {}'\
.format(source, dst, pipe, name, msg_copy, result) )
elif isinstance(result, list): msg_copy.update(result)
else: msg_copy.add(result)
msg_copy = msg_copy.difference({None})
if not msg_copy: break
else:
if dst in self.relays:
extra_kwz = dict()
if isinstance(dst, types.StringTypes): dst = self.relays[dst]
if user and 'source' in inspect.getargspec(dst.dispatch).args:
extra_kwz['source'] = user
log.noise('Delivering msgs to dst relay: {}, extra_kwz: {}'.format(dst, extra_kwz))
yield defer.DeferredList(list(
defer.maybeDeferred(dst.dispatch, msg_copy, **extra_kwz)
for msg_copy in msg_copy ))
else:
channels.setdefault(self.channels[dst].name, set()).update(msg_copy)
# Check whether anything can be delivered to channels at all
if not self.proto:
log.warn( 'Failed to deliver message(s)'
' ({!r}) to the following channels: {}'.format(msg, channels) )
defer.returnValue(None)
# Encode and deliver
for channel, msg in channels.viewitems():
for msg in msg:
if not isinstance(msg, types.StringTypes):
log.warn('Dropping non-string message: {!r}'.format(msg))
continue
if isinstance(msg, unicode):
try: msg = msg.encode(self.irc_enc)
except UnicodeEncodeError as err:
log.warn('Failed to encode ({}) unicode msg ({!r}): {}'.format(self.irc_enc, msg, err))
msg = msg.encode(self.irc_enc, 'replace')
max_len = min( self.max_line_length,
self.proto._safeMaximumLineLength('PRIVMSG {} :'.format(channel)) - 2 )
first_line = True
for line in irc.split(msg, length=max_len):
if not first_line: line = ' {}'.format(line)
if not self.dry_run: self.proto.msg(channel, line)
else: log.info('IRC line (channel: {}): {}'.format(channel, line))
first_line = False
def proto_msg(self, irc, user, nick, channel, msg):
if channel not in self.channel_map:
log.noise( 'Ignoring msg for unmonitored source'
' (user: {!r}, nick: {!r}, channel: {!r})'.format(user, nick, channel) )
return
self.dispatch(msg, source=self.channel_map[channel], user=nick)
def __init__(self, *argz, **kwz):
super(Shortener, self).__init__(*argz, **kwz)
log.noise('Compiling regex: {!r}'.format(self.conf.regex))
self.regex = re.compile(self.conf.regex)
self.client = HTTPClient()
def privmsg(self, user, channel, message):
nick = user.split('!', 1)[0]
if self.conf.nickname_lstrip: nick = nick.lstrip(self.conf.nickname_lstrip)
log.noise('Got msg: {}'.format([user, nick, channel, message]))
self.interface.proto_msg(self, user, nick, channel, message)
def check(self, name):
check = self.conf.get('check_path')
if check and check.path:
path = check.path.format(name=name)
log.noise('Checking package path: {}'.format(path))
if glob(check.path.format(name=name)): return check.line
def dispatch(self, msg):
msg_sub = self.regex.sub(self.conf.dst, msg)
if msg == msg_sub: log.noise('RegexSub failed, msg: {!r}'.format(msg))
return msg_sub
