def grab_category_scores(store, categories=None, debug=False):
'''Program to create and return some python Dicts with security scores and totals by category
and totals by drone/category
Categories is None or a list of desired categories.
'''
cypher = '''START drone=node:Drone('*:*') RETURN drone'''
BestPractices(CMAdb.io.config, CMAdb.io, store, CMAdb.log, debug=debug)
dtype_totals = {} # scores organized by (category, discovery-type)
drone_totals = {} # scores organized by (category, discovery-type, drone)
rule_totals = {} # scores organized by (category, discovery-type, rule)
for drone in store.load_cypher_nodes(cypher, Drone):
designation = drone.designation
discoverytypes = drone.bp_discoverytypes_list()
for dtype in discoverytypes:
dattr = Drone.bp_discoverytype_result_attrname(dtype)
statuses = getattr(drone, dattr)
for rule_obj in BestPractices.eval_objects[dtype]:
rulesobj = rule_obj.fetch_rules(drone, None, dtype)
_, scores, rulescores = BestPractices.compute_scores(drone, rulesobj, statuses)
for category in scores:
if category not in categories and categories:
continue
# Accumulate scores by (category, discovery_type)
setup_dict2(dtype_totals, category, dtype)
dtype_totals[category][dtype] += scores[category]
# Accumulate scores by (category, discovery_type, drone)
setup_dict3(drone_totals, category, dtype, designation)
drone_totals[category][dtype][designation] += scores[category]
# Accumulate scores by (category, discovery_type, ruleid)
for ruleid in rulescores[category]:
setup_dict3(rule_totals, category, dtype, ruleid)
rule_totals[category][dtype][ruleid] += rulescores[category][ruleid]
return dtype_totals, drone_totals, rule_totals
def main():
'Main program for the CMA (Collective Management Authority)'
py2neo_major_version = int(PY2NEO_VERSION.partition('.')[0])
if py2neo_major_version not in SUPPORTED_PY2NEO_VERSIONS:
raise EnvironmentError('py2neo version %s not supported' %
PY2NEO_VERSION)
DefaultPort = 1984
# VERY Linux-specific - but useful and apparently correct ;-)
PrimaryIPcmd = \
"ip address show primary scope global | grep '^ *inet' | sed -e 's%^ *inet *%%' -e 's%/.*%%'"
ipfd = os.popen(PrimaryIPcmd, 'r')
OurAddrStr = ('%s:%d' % (ipfd.readline().rstrip(), DefaultPort))
ipfd.close()
parser = optparse.OptionParser(
prog='CMA',
version=AssimCtypes.VERSION_STRING,
description=
'Collective Management Authority for the Assimilation System',
usage='cma.py [--bind address:port]')
parser.add_option(
'-b',
'--bind',
action='store',
default=None,
dest='bind',
metavar='address:port-to-bind-to',
help='Address:port to listen to - for nanoprobes to connect to')
parser.add_option(
'-d',
'--debug',
action='store',
default=0,
dest='debug',
help=
'enable debug for CMA and libraries - value is debug level for C libraries.'
)
parser.add_option('-s',
'--status',
action='store_true',
default=False,
dest='status',
help='Return status of running CMA')
parser.add_option('-k',
'--kill',
action='store_true',
default=False,
dest='kill',
help='Shut down running CMA.')
parser.add_option('-e',
'--erasedb',
action='store_true',
default=False,
dest='erasedb',
help='Erase Neo4J before starting')
parser.add_option('-f',
'--foreground',
action='store_true',
default=False,
dest='foreground',
help='keep the CMA from going into the background')
parser.add_option('-p',
'--pidfile',
action='store',
default='/var/run/assimilation/cma',
dest='pidfile',
metavar='pidfile-pathname',
help='full pathname of where to locate our pid file')
parser.add_option('-T',
'--trace',
action='store_true',
default=False,
dest='doTrace',
help='Trace CMA execution')
parser.add_option('-u',
'--user',
action='store',
default=CMAUSERID,
dest='userid',
metavar='userid',
help='userid to run the CMA as')
opt = parser.parse_args()[0]
from AssimCtypes import daemonize_me, assimilation_openlog, are_we_already_running, \
kill_pid_service, pidrunningstat_to_status, remove_pid_file, rmpid_and_exit_on_signal
if opt.status:
rc = pidrunningstat_to_status(are_we_already_running(
opt.pidfile, None))
return rc
if opt.kill:
if kill_pid_service(opt.pidfile, 15) < 0:
print >> sys.stderr, "Unable to stop CMA."
return 1
return 0
opt.debug = int(opt.debug)
# This doesn't seem to work no matter where I invoke it...
# But if we don't fork in daemonize_me() ('C' code), it works great...
# def cleanup():
# remove_pid_file(opt.pidfile)
# atexit.register(cleanup)
# signal.signal(signal.SIGTERM, lambda sig, stack: sys.exit(0))
# signal.signal(signal.SIGINT, lambda sig, stack: sys.exit(0))
from cmadb import CMAdb
CMAdb.running_under_docker()
make_pid_dir(opt.pidfile, opt.userid)
make_key_dir(CRYPTKEYDIR, opt.userid)
cryptwarnings = pyCryptCurve25519.initkeys()
for warn in cryptwarnings:
print >> sys.stderr, ("WARNING: %s" % warn)
#print >> sys.stderr, 'All known key ids:'
keyids = pyCryptFrame.get_key_ids()
keyids.sort()
for keyid in keyids:
if not keyid.startswith(CMA_KEY_PREFIX):
try:
# @FIXME This is not an ideal way to associate identities with hosts
# in a multi-tenant environment
# @FIXME - don't think I need to do the associate_identity at all any more...
hostname, notused_post = keyid.split('@@', 1)
notused_post = notused_post
pyCryptFrame.associate_identity(hostname, keyid)
except ValueError:
pass
#print >> sys.stderr, '> %s/%s' % (keyid, pyCryptFrame.get_identity(keyid))
daemonize_me(opt.foreground, '/', opt.pidfile, 20)
rmpid_and_exit_on_signal(opt.pidfile, signal.SIGTERM)
# Next statement can't appear before daemonize_me() or bind() fails -- not quite sure why...
assimilation_openlog("cma")
from packetlistener import PacketListener
from messagedispatcher import MessageDispatcher
from dispatchtarget import DispatchTarget
from monitoring import MonitoringRule
from AssimCclasses import pyNetAddr, pySignFrame, pyReliableUDP, \
pyPacketDecoder
from AssimCtypes import CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR, CONFIGNAME_CMADISCOVER, \
CONFIGNAME_CMAFAIL, CONFIGNAME_CMAPORT, CONFIGNAME_OUTSIG, CONFIGNAME_COMPRESSTYPE, \
CONFIGNAME_COMPRESS, proj_class_incr_debug, LONG_LICENSE_STRING, MONRULEINSTALL_DIR
if opt.debug:
print >> sys.stderr, ('Setting debug to %s' % opt.debug)
for debug in range(opt.debug):
debug = debug
print >> sys.stderr, ('Incrementing C-level debug by one.')
proj_class_incr_debug(None)
# Input our monitoring rule templates
# They only exist in flat files and in memory - they aren't in the database
MonitoringRule.load_tree(MONRULEINSTALL_DIR)
print >> sys.stderr, ('Monitoring rules loaded from %s' %
MONRULEINSTALL_DIR)
execobserver_constraints = {
'nodetype': [
'Drone',
'IPaddrNode',
'MonitorAction',
'NICNode',
'ProcessNode',
'SystemNode',
]
}
ForkExecObserver(constraints=execobserver_constraints,
scriptdir=NOTIFICATION_SCRIPT_DIR)
print >> sys.stderr, ('Fork/Event observer dispatching from %s' %
NOTIFICATION_SCRIPT_DIR)
if opt.bind is not None:
OurAddrStr = opt.bind
OurAddr = pyNetAddr(OurAddrStr)
if OurAddr.port() == 0:
OurAddr.setport(DefaultPort)
try:
configinfo = ConfigFile(filename=CMAINITFILE)
except IOError:
configinfo = ConfigFile()
if opt.bind is not None:
bindaddr = pyNetAddr(opt.bind)
if bindaddr.port() == 0:
bindaddr.setport(ConfigFile[CONFIGNAME_CMAPORT])
configinfo[CONFIGNAME_CMAINIT] = bindaddr
configinfo[CONFIGNAME_CMADISCOVER] = OurAddr
configinfo[CONFIGNAME_CMAFAIL] = OurAddr
configinfo[CONFIGNAME_CMAADDR] = OurAddr
if (CONFIGNAME_COMPRESSTYPE in configinfo):
configinfo[CONFIGNAME_COMPRESS] \
= pyCompressFrame(compression_method=configinfo[CONFIGNAME_COMPRESSTYPE])
configinfo[CONFIGNAME_OUTSIG] = pySignFrame(1)
config = configinfo.complete_config()
addr = config[CONFIGNAME_CMAINIT]
# pylint is confused: addr is a pyNetAddr, not a pyConfigContext
# pylint: disable=E1101
if addr.port() == 0:
addr.setport(DefaultPort)
ourport = addr.port()
for elem in (CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR,
CONFIGNAME_CMADISCOVER, CONFIGNAME_CMAFAIL):
if elem in config:
config[elem] = pyNetAddr(str(config[elem]), port=ourport)
io = pyReliableUDP(config, pyPacketDecoder())
io.setrcvbufsize(
10 * 1024 *
1024) # No harm in asking - it will get us the best we can get...
io.setsendbufsize(
1024 * 1024) # Most of the traffic volume is inbound from discovery
drop_privileges_permanently(opt.userid)
try:
cmainit.CMAinit(io, cleanoutdb=opt.erasedb, debug=(opt.debug > 0))
except RuntimeError:
remove_pid_file(opt.pidfile)
raise
for warn in cryptwarnings:
CMAdb.log.warning(warn)
if CMAdb.cdb.db.neo4j_version[0] not in SUPPORTED_NEO4J_VERSIONS:
raise EnvironmentError('Neo4j version %s.%s.%s not supported' %
CMAdb.cdb.db.neo4j_version)
CMAdb.log.info('Listening on: %s' % str(config[CONFIGNAME_CMAINIT]))
CMAdb.log.info('Requesting return packets sent to: %s' % str(OurAddr))
CMAdb.log.info('Socket input buffer size: %d' % io.getrcvbufsize())
CMAdb.log.info('Socket output buffer size: %d' % io.getsendbufsize())
keyids = pyCryptFrame.get_key_ids()
keyids.sort()
for keyid in keyids:
CMAdb.log.info('KeyId %s Identity %s' %
(keyid, pyCryptFrame.get_identity(keyid)))
if CMAdb.debug:
CMAdb.log.debug('C-library Debug was set to %s' % opt.debug)
CMAdb.log.debug('TheOneRing created - id = %s' % CMAdb.TheOneRing)
CMAdb.log.debug('Config Object sent to nanoprobes: %s' % config)
jvmfd = os.popen('java -version 2>&1')
jvers = jvmfd.readline()
jvmfd.close()
disp = MessageDispatcher(DispatchTarget.dispatchtable)
neovers = CMAdb.cdb.db.neo4j_version
neoversstring = (('%s.%s.%s' if len(neovers) == 3 else '%s.%s.%s%s') %
neovers[0:3])
CMAdb.log.info('Starting CMA version %s - licensed under %s' %
(AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING))
CMAdb.log.info(
'Neo4j version %s // py2neo version %s // Python version %s // %s' %
(('%s.%s.%s' % CMAdb.cdb.db.neo4j_version), str(py2neo.__version__),
('%s.%s.%s' % sys.version_info[0:3]), jvers))
if opt.foreground:
print >> sys.stderr, (
'Starting CMA version %s - licensed under %s' %
(AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING))
print >> sys.stderr, (
'Neo4j version %s // py2neo version %s // Python version %s // %s'
% (neoversstring, PY2NEO_VERSION,
('%s.%s.%s' % sys.version_info[0:3]), jvers))
if len(neovers) > 3:
CMAdb.log.warning(
'Neo4j version %s is beta code - results not guaranteed.' %
str(neovers))
# Important to note that we don't want PacketListener to create its own 'io' object
# or it will screw up the ReliableUDP protocol...
listener = PacketListener(config, disp, io=io)
mandatory_modules = ['discoverylistener']
for mandatory in mandatory_modules:
importlib.import_module(mandatory)
#pylint is confused here...
# pylint: disable=E1133
for optional in config['optional_modules']:
importlib.import_module(optional)
if opt.doTrace:
import trace
tracer = trace.Trace(count=False, trace=True)
if CMAdb.debug:
CMAdb.log.debug('Starting up traced listener.listen(); debug=%d' %
opt.debug)
if opt.foreground:
print >> sys.stderr, (
'cma: Starting up traced listener.listen() in foreground; debug=%d'
% opt.debug)
tracer.run('listener.listen()')
else:
if CMAdb.debug:
CMAdb.log.debug(
'Starting up untraced listener.listen(); debug=%d' % opt.debug)
if opt.foreground:
print >> sys.stderr, (
'cma: Starting up untraced listener.listen() in foreground; debug=%d'
% opt.debug)
# This is kind of a kludge, we should really look again at
# at initializition and so on.
# This module *ought* to be optional.
# that would involve adding some Drone callbacks for creation of new Drones
BestPractices(config, io, CMAdb.store, CMAdb.log, opt.debug)
listener.listen()
return 0
