def __init__(self, designation, port=None, startaddr=None , primary_ip_addr=None, domain=CMAconsts.globaldomain , status= '(unknown)', reason='(initialization)', roles=None, key_id=''): '''Initialization function for the Drone class. We mainly initialize a few attributes from parameters as noted above... The first time around we also initialize a couple of class-wide query strings for a few queries we know we'll need later. We also behave as though we're a dict from the perspective of JSON attributes. These discovery strings are converted into pyConfigContext objects and are then searchable like dicts themselves - however updating these dicts has no direct impact on the underlying JSON strings stored in the database. The reason for treating these as a dict is so we can easily change the implementation to put JSON strings in separate nodes, or perhaps eventually in a separate data store. This is necessary because the performance of putting lots of really large strings in Neo4j is absolutely horrible. Putting large strings in is dumb and what Neo4j does with them is even dumber... The result is at least DUMB^2 -not 2*DUMB ;-) ''' SystemNode.__init__(self, domain=domain, designation=designation) if roles is None: roles = ['host', 'drone'] self.addrole(roles) self._io = CMAdb.io self.lastjoin = 'None' self.status = status self.reason = reason self.key_id = key_id self.startaddr = str(startaddr) self.primary_ip_addr = str(primary_ip_addr) self.time_status_ms = int(round(time.time() * 1000)) self.time_status_iso8601 = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) if port is not None: self.port = int(port) else: self.port = None self.monitors_activated = False if Drone.IPownerquery_1 is None: Drone.IPownerquery_1 = (Drone.IPownerquery_1_txt % (CMAconsts.REL_ipowner, CMAconsts.REL_nicowner)) Drone.OwnedIPsQuery_subtxt = (Drone.OwnedIPsQuery_txt \ % (CMAconsts.REL_nicowner, CMAconsts.REL_ipowner)) Drone.OwnedIPsQuery = Drone.OwnedIPsQuery_subtxt self.set_crypto_identity() if Store.is_abstract(self) and not CMAdb.store.readonly: #print 'Creating BP rules for', self.designation from bestpractices import BestPractices bprules = CMAdb.io.config['bprulesbydomain'] rulesetname = bprules[domain] if domain in bprules else bprules[CMAconsts.globaldomain] for rule in BestPractices.gen_bp_rules_by_ruleset(CMAdb.store, rulesetname): #print >> sys.stderr, 'ADDING RELATED RULE SET for', \ #self.designation, rule.bp_class, rule CMAdb.store.relate(self, CMAconsts.REL_bprulefor, rule, properties={'bp_class': rule.bp_class})
def grab_category_scores(store, categories=None, debug=False): '''Program to create and return some python Dicts with security scores and totals by category and totals by drone/category Categories is None or a list of desired categories. ''' cypher = '''START drone=node:Drone('*:*') RETURN drone''' BestPractices(CMAdb.io.config, CMAdb.io, store, CMAdb.log, debug=debug) dtype_totals = {} # scores organized by (category, discovery-type) drone_totals = {} # scores organized by (category, discovery-type, drone) rule_totals = {} # scores organized by (category, discovery-type, rule) for drone in store.load_cypher_nodes(cypher, Drone): designation = drone.designation discoverytypes = drone.bp_discoverytypes_list() for dtype in discoverytypes: dattr = Drone.bp_discoverytype_result_attrname(dtype) statuses = getattr(drone, dattr) for rule_obj in BestPractices.eval_objects[dtype]: rulesobj = rule_obj.fetch_rules(drone, None, dtype) _, scores, rulescores = BestPractices.compute_scores(drone, rulesobj, statuses) for category in scores: if category not in categories and categories: continue # Accumulate scores by (category, discovery_type) setup_dict2(dtype_totals, category, dtype) dtype_totals[category][dtype] += scores[category] # Accumulate scores by (category, discovery_type, drone) setup_dict3(drone_totals, category, dtype, designation) drone_totals[category][dtype][designation] += scores[category] # Accumulate scores by (category, discovery_type, ruleid) for ruleid in rulescores[category]: setup_dict3(rule_totals, category, dtype, ruleid) rule_totals[category][dtype][ruleid] += rulescores[category][ruleid] return dtype_totals, drone_totals, rule_totals
def __init__(self, designation, port=None, startaddr=None , primary_ip_addr=None, domain=CMAconsts.globaldomain , status= '(unknown)', reason='(initialization)', roles=None, key_id=''): '''Initialization function for the Drone class. We mainly initialize a few attributes from parameters as noted above... The first time around we also initialize a couple of class-wide CypherQuery objects for a couple of queries we know we'll need later. ''' SystemNode.__init__(self, domain=domain, designation=designation) if roles is None: roles = ['host', 'drone'] self.addrole(roles) self._io = CMAdb.io self.lastjoin = 'None' self.status = status self.reason = reason self.key_id = key_id self.startaddr = str(startaddr) self.primary_ip_addr = str(primary_ip_addr) self.time_status_ms = int(round(time.time() * 1000)) self.time_status_iso8601 = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) if port is not None: self.port = int(port) else: self.port = None self.monitors_activated = False if Drone.IPownerquery_1 is None: Drone.IPownerquery_1 = neo4j.CypherQuery(CMAdb.cdb.db, Drone.IPownerquery_1_txt % (CMAconsts.REL_ipowner, CMAconsts.REL_nicowner)) Drone.OwnedIPsQuery_subtxt = Drone.OwnedIPsQuery_txt \ % (CMAconsts.REL_nicowner, CMAconsts.REL_ipowner) Drone.OwnedIPsQuery = neo4j.CypherQuery(CMAdb.cdb.db, Drone.OwnedIPsQuery_subtxt) self.set_crypto_identity() if Store.is_abstract(self) and not CMAdb.store.readonly: from bestpractices import BestPractices bprules = CMAdb.io.config['bprulesbydomain'] rulesetname = bprules[domain] if domain in bprules else bprules[CMAconsts.globaldomain] for rule in BestPractices.gen_bp_rules_by_ruleset(CMAdb.store, rulesetname): CMAdb.store.relate(self, CMAconsts.REL_bprulefor, rule, properties={'bp_class': rule.bp_class})
def execute(store, executor_context, otherargs, flagoptions): 'Load all best practice files we find in the specified directory.' executor_context = executor_context flagoptions = flagoptions basedon = None if len(otherargs) not in (0,2,3): return usage() elif len(otherargs) >= 2: bpdir = otherargs[0] rulesetname = otherargs[1] if len(otherargs) > 2: basedon = otherargs[2] else: bpdir = BPINSTALL_DIR rulesetname = CMAconsts.BASERULESETNAME qcount = 0 for q in BestPractices.load_directory(store, bpdir, rulesetname, basedon): qcount += 1 q = q store.commit() return 0 if qcount > 0 else 1
def __init__(self, designation, port=None, startaddr=None , primary_ip_addr=None, domain=CMAconsts.globaldomain , status= '(unknown)', reason='(initialization)', roles=None, key_id=''): '''Initialization function for the Drone class. We mainly initialize a few attributes from parameters as noted above... The first time around we also initialize a couple of class-wide query strings for a few queries we know we'll need later. We also behave as though we're a dict from the perspective of JSON attributes. These discovery strings are converted into pyConfigContext objects and are then searchable like dicts themselves - however updating these dicts has no direct impact on the underlying JSON strings stored in the database. The reason for treating these as a dict is so we can easily change the implementation to put JSON strings in separate nodes, or perhaps eventually in a separate data store. This is necessary because the performance of putting lots of really large strings in Neo4j is absolutely horrible. Putting large strings in is dumb and what Neo4j does with them is even dumber... The result is at least DUMB^2 -not 2*DUMB ;-) ''' SystemNode.__init__(self, domain=domain, designation=designation) if roles is None: roles = ['host', 'drone'] self.addrole(roles) self._io = CMAdb.io self.lastjoin = 'None' self._active_nic_count = None self.status = status self.reason = reason self.key_id = key_id self.startaddr = str(startaddr) self.primary_ip_addr = str(primary_ip_addr) self.time_status_ms = int(round(time.time() * 1000)) self.time_status_iso8601 = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) if port is not None: self.port = int(port) else: self.port = None self.monitors_activated = False if Drone.IPownerquery_1 is None: Drone.IPownerquery_1 = (Drone.IPownerquery_1_txt % (CMAconsts.REL_ipowner, CMAconsts.REL_nicowner)) Drone.OwnedIPsQuery_subtxt = (Drone.OwnedIPsQuery_txt \ % (CMAconsts.REL_nicowner, CMAconsts.REL_ipowner)) Drone.OwnedIPsQuery = Drone.OwnedIPsQuery_subtxt self.set_crypto_identity() if Store.is_abstract(self) and not CMAdb.store.readonly: #print 'Creating BP rules for', self.designation from bestpractices import BestPractices bprules = CMAdb.io.config['bprulesbydomain'] rulesetname = bprules[domain] if domain in bprules else bprules[CMAconsts.globaldomain] for rule in BestPractices.gen_bp_rules_by_ruleset(CMAdb.store, rulesetname): #print >> sys.stderr, 'ADDING RELATED RULE SET for', \ #self.designation, rule.bp_class, rule CMAdb.store.relate(self, CMAconsts.REL_bprulefor, rule, properties={'bp_class': rule.bp_class})
def main(): 'Main program for the CMA (Collective Management Authority)' py2neo_major_version = int(PY2NEO_VERSION.partition('.')[0]) if py2neo_major_version not in SUPPORTED_PY2NEO_VERSIONS: raise EnvironmentError('py2neo version %s not supported' % PY2NEO_VERSION) DefaultPort = 1984 # VERY Linux-specific - but useful and apparently correct ;-) PrimaryIPcmd = \ "ip address show primary scope global | grep '^ *inet' | sed -e 's%^ *inet *%%' -e 's%/.*%%'" ipfd = os.popen(PrimaryIPcmd, 'r') OurAddrStr = ('%s:%d' % (ipfd.readline().rstrip(), DefaultPort)) ipfd.close() parser = optparse.OptionParser( prog='CMA', version=AssimCtypes.VERSION_STRING, description= 'Collective Management Authority for the Assimilation System', usage='cma.py [--bind address:port]') parser.add_option( '-b', '--bind', action='store', default=None, dest='bind', metavar='address:port-to-bind-to', help='Address:port to listen to - for nanoprobes to connect to') parser.add_option( '-d', '--debug', action='store', default=0, dest='debug', help= 'enable debug for CMA and libraries - value is debug level for C libraries.' ) parser.add_option('-s', '--status', action='store_true', default=False, dest='status', help='Return status of running CMA') parser.add_option('-k', '--kill', action='store_true', default=False, dest='kill', help='Shut down running CMA.') parser.add_option('-e', '--erasedb', action='store_true', default=False, dest='erasedb', help='Erase Neo4J before starting') parser.add_option('-f', '--foreground', action='store_true', default=False, dest='foreground', help='keep the CMA from going into the background') parser.add_option('-p', '--pidfile', action='store', default='/var/run/assimilation/cma', dest='pidfile', metavar='pidfile-pathname', help='full pathname of where to locate our pid file') parser.add_option('-T', '--trace', action='store_true', default=False, dest='doTrace', help='Trace CMA execution') parser.add_option('-u', '--user', action='store', default=CMAUSERID, dest='userid', metavar='userid', help='userid to run the CMA as') opt = parser.parse_args()[0] from AssimCtypes import daemonize_me, assimilation_openlog, are_we_already_running, \ kill_pid_service, pidrunningstat_to_status, remove_pid_file, rmpid_and_exit_on_signal if opt.status: rc = pidrunningstat_to_status(are_we_already_running( opt.pidfile, None)) return rc if opt.kill: if kill_pid_service(opt.pidfile, 15) < 0: print >> sys.stderr, "Unable to stop CMA." return 1 return 0 opt.debug = int(opt.debug) # This doesn't seem to work no matter where I invoke it... # But if we don't fork in daemonize_me() ('C' code), it works great... # def cleanup(): # remove_pid_file(opt.pidfile) # atexit.register(cleanup) # signal.signal(signal.SIGTERM, lambda sig, stack: sys.exit(0)) # signal.signal(signal.SIGINT, lambda sig, stack: sys.exit(0)) from cmadb import CMAdb CMAdb.running_under_docker() make_pid_dir(opt.pidfile, opt.userid) make_key_dir(CRYPTKEYDIR, opt.userid) cryptwarnings = pyCryptCurve25519.initkeys() for warn in cryptwarnings: print >> sys.stderr, ("WARNING: %s" % warn) #print >> sys.stderr, 'All known key ids:' keyids = pyCryptFrame.get_key_ids() keyids.sort() for keyid in keyids: if not keyid.startswith(CMA_KEY_PREFIX): try: # @FIXME This is not an ideal way to associate identities with hosts # in a multi-tenant environment # @FIXME - don't think I need to do the associate_identity at all any more... hostname, notused_post = keyid.split('@@', 1) notused_post = notused_post pyCryptFrame.associate_identity(hostname, keyid) except ValueError: pass #print >> sys.stderr, '> %s/%s' % (keyid, pyCryptFrame.get_identity(keyid)) daemonize_me(opt.foreground, '/', opt.pidfile, 20) rmpid_and_exit_on_signal(opt.pidfile, signal.SIGTERM) # Next statement can't appear before daemonize_me() or bind() fails -- not quite sure why... assimilation_openlog("cma") from packetlistener import PacketListener from messagedispatcher import MessageDispatcher from dispatchtarget import DispatchTarget from monitoring import MonitoringRule from AssimCclasses import pyNetAddr, pySignFrame, pyReliableUDP, \ pyPacketDecoder from AssimCtypes import CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR, CONFIGNAME_CMADISCOVER, \ CONFIGNAME_CMAFAIL, CONFIGNAME_CMAPORT, CONFIGNAME_OUTSIG, CONFIGNAME_COMPRESSTYPE, \ CONFIGNAME_COMPRESS, proj_class_incr_debug, LONG_LICENSE_STRING, MONRULEINSTALL_DIR if opt.debug: print >> sys.stderr, ('Setting debug to %s' % opt.debug) for debug in range(opt.debug): debug = debug print >> sys.stderr, ('Incrementing C-level debug by one.') proj_class_incr_debug(None) # Input our monitoring rule templates # They only exist in flat files and in memory - they aren't in the database MonitoringRule.load_tree(MONRULEINSTALL_DIR) print >> sys.stderr, ('Monitoring rules loaded from %s' % MONRULEINSTALL_DIR) execobserver_constraints = { 'nodetype': [ 'Drone', 'IPaddrNode', 'MonitorAction', 'NICNode', 'ProcessNode', 'SystemNode', ] } ForkExecObserver(constraints=execobserver_constraints, scriptdir=NOTIFICATION_SCRIPT_DIR) print >> sys.stderr, ('Fork/Event observer dispatching from %s' % NOTIFICATION_SCRIPT_DIR) if opt.bind is not None: OurAddrStr = opt.bind OurAddr = pyNetAddr(OurAddrStr) if OurAddr.port() == 0: OurAddr.setport(DefaultPort) try: configinfo = ConfigFile(filename=CMAINITFILE) except IOError: configinfo = ConfigFile() if opt.bind is not None: bindaddr = pyNetAddr(opt.bind) if bindaddr.port() == 0: bindaddr.setport(ConfigFile[CONFIGNAME_CMAPORT]) configinfo[CONFIGNAME_CMAINIT] = bindaddr configinfo[CONFIGNAME_CMADISCOVER] = OurAddr configinfo[CONFIGNAME_CMAFAIL] = OurAddr configinfo[CONFIGNAME_CMAADDR] = OurAddr if (CONFIGNAME_COMPRESSTYPE in configinfo): configinfo[CONFIGNAME_COMPRESS] \ = pyCompressFrame(compression_method=configinfo[CONFIGNAME_COMPRESSTYPE]) configinfo[CONFIGNAME_OUTSIG] = pySignFrame(1) config = configinfo.complete_config() addr = config[CONFIGNAME_CMAINIT] # pylint is confused: addr is a pyNetAddr, not a pyConfigContext # pylint: disable=E1101 if addr.port() == 0: addr.setport(DefaultPort) ourport = addr.port() for elem in (CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR, CONFIGNAME_CMADISCOVER, CONFIGNAME_CMAFAIL): if elem in config: config[elem] = pyNetAddr(str(config[elem]), port=ourport) io = pyReliableUDP(config, pyPacketDecoder()) io.setrcvbufsize( 10 * 1024 * 1024) # No harm in asking - it will get us the best we can get... io.setsendbufsize( 1024 * 1024) # Most of the traffic volume is inbound from discovery drop_privileges_permanently(opt.userid) try: cmainit.CMAinit(io, cleanoutdb=opt.erasedb, debug=(opt.debug > 0)) except RuntimeError: remove_pid_file(opt.pidfile) raise for warn in cryptwarnings: CMAdb.log.warning(warn) if CMAdb.cdb.db.neo4j_version[0] not in SUPPORTED_NEO4J_VERSIONS: raise EnvironmentError('Neo4j version %s.%s.%s not supported' % CMAdb.cdb.db.neo4j_version) CMAdb.log.info('Listening on: %s' % str(config[CONFIGNAME_CMAINIT])) CMAdb.log.info('Requesting return packets sent to: %s' % str(OurAddr)) CMAdb.log.info('Socket input buffer size: %d' % io.getrcvbufsize()) CMAdb.log.info('Socket output buffer size: %d' % io.getsendbufsize()) keyids = pyCryptFrame.get_key_ids() keyids.sort() for keyid in keyids: CMAdb.log.info('KeyId %s Identity %s' % (keyid, pyCryptFrame.get_identity(keyid))) if CMAdb.debug: CMAdb.log.debug('C-library Debug was set to %s' % opt.debug) CMAdb.log.debug('TheOneRing created - id = %s' % CMAdb.TheOneRing) CMAdb.log.debug('Config Object sent to nanoprobes: %s' % config) jvmfd = os.popen('java -version 2>&1') jvers = jvmfd.readline() jvmfd.close() disp = MessageDispatcher(DispatchTarget.dispatchtable) neovers = CMAdb.cdb.db.neo4j_version neoversstring = (('%s.%s.%s' if len(neovers) == 3 else '%s.%s.%s%s') % neovers[0:3]) CMAdb.log.info('Starting CMA version %s - licensed under %s' % (AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING)) CMAdb.log.info( 'Neo4j version %s // py2neo version %s // Python version %s // %s' % (('%s.%s.%s' % CMAdb.cdb.db.neo4j_version), str(py2neo.__version__), ('%s.%s.%s' % sys.version_info[0:3]), jvers)) if opt.foreground: print >> sys.stderr, ( 'Starting CMA version %s - licensed under %s' % (AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING)) print >> sys.stderr, ( 'Neo4j version %s // py2neo version %s // Python version %s // %s' % (neoversstring, PY2NEO_VERSION, ('%s.%s.%s' % sys.version_info[0:3]), jvers)) if len(neovers) > 3: CMAdb.log.warning( 'Neo4j version %s is beta code - results not guaranteed.' % str(neovers)) # Important to note that we don't want PacketListener to create its own 'io' object # or it will screw up the ReliableUDP protocol... listener = PacketListener(config, disp, io=io) mandatory_modules = ['discoverylistener'] for mandatory in mandatory_modules: importlib.import_module(mandatory) #pylint is confused here... # pylint: disable=E1133 for optional in config['optional_modules']: importlib.import_module(optional) if opt.doTrace: import trace tracer = trace.Trace(count=False, trace=True) if CMAdb.debug: CMAdb.log.debug('Starting up traced listener.listen(); debug=%d' % opt.debug) if opt.foreground: print >> sys.stderr, ( 'cma: Starting up traced listener.listen() in foreground; debug=%d' % opt.debug) tracer.run('listener.listen()') else: if CMAdb.debug: CMAdb.log.debug( 'Starting up untraced listener.listen(); debug=%d' % opt.debug) if opt.foreground: print >> sys.stderr, ( 'cma: Starting up untraced listener.listen() in foreground; debug=%d' % opt.debug) # This is kind of a kludge, we should really look again at # at initializition and so on. # This module *ought* to be optional. # that would involve adding some Drone callbacks for creation of new Drones BestPractices(config, io, CMAdb.store, CMAdb.log, opt.debug) listener.listen() return 0