def __init__(self, config=None):
if config is None:
try:
config = ConfigFile(filename=CMAINITFILE)
except IOError:
config = ConfigFile()
self.config = config
def processpkt_netconfig(self, drone, _unused_srcaddr, jsonobj):
'''We want to trigger Switch discovery when we hear a 'netconfig' packet
Build up the parameters for the discovery
action, then send it to drone.request_discovery(...)
To build up the parameters, you use ConfigFile.agent_params()
which will pull values from the system configuration.
'''
init_params = ConfigFile.agent_params(self.config, 'discovery',
'#SWITCH', drone.designation)
data = jsonobj['data'] # the data portion of the JSON message
discovery_args = []
for devname in data.keys():
devinfo = data[devname]
if (str(devinfo['operstate']) == 'up'
and str(devinfo['carrier']) == 'True'
and str(devinfo['address']) != '00-00-00-00-00-00'
and str(devinfo['address']) != ''):
params = pyConfigContext(init_params)
params[CONFIGNAME_INSTANCE] = '#SWITCH_' + devname
params[CONFIGNAME_DEVNAME] = devname
params[CONFIGNAME_SWPROTOS] = ["lldp", "cdp"]
#print >> sys.stderr, '***#SWITCH parameters:', params
discovery_args.append(params)
if discovery_args:
drone.request_discovery(discovery_args)
def send_hbmsg(self, dest, fstype, addrlist):
'''Send a message with an attached pyNetAddr list - each including port numbers'
This is intended primarily for start or stop heartbeating messages.'''
# Now we create a collection of frames that looks like this:
#
# One FrameTypes.RSCJSON frame containing JSON Heartbeat parameters
# one frame per dest, type FrameTypes.IPPORT
#
params = ConfigFile.agent_params(CMAdb.io.config, 'heartbeats', None,
self.designation)
framelist = [
{
'frametype': FrameTypes.RSCJSON,
'framevalue': str(params)
},
]
for addr in addrlist:
if addr is None:
continue
framelist.append({
'frametype': FrameTypes.IPPORT,
'framevalue': addr
})
CMAdb.transaction.add_packet(dest, fstype, framelist)
def request_discovery(self, args): ##< A vector of arguments containing
'''Send our System a request to perform discovery
We send a DISCNAME frame with the instance name
then an optional DISCINTERVAL frame with the repeat interval
then a DISCJSON frame with the JSON data for the discovery operation.
Our argument is a vector of pyConfigContext objects with values for
'instance' Name of this discovery instance
'interval' How often to repeat this discovery action
'timeout' How long to wait before considering this discovery failed...
'''
#fs = pyFrameSet(FrameSetTypes.DODISCOVER)
frames = []
for arg in args:
agent_params = ConfigFile.agent_params(CMAdb.io.config, 'discovery',
arg[CONFIGNAME_TYPE], self.designation)
for key in ('repeat', 'warn' 'timeout', 'nice'):
if key in agent_params and key not in arg:
arg[key] = agent_params[arg]
instance = arg['instance']
frames.append({'frametype': FrameTypes.DISCNAME, 'framevalue': instance})
if 'repeat' in arg:
interval = int(arg['repeat'])
frames.append({'frametype': FrameTypes.DISCINTERVAL, 'framevalue': int(interval)})
else:
interval = 0
frames.append({'frametype': FrameTypes.DISCJSON, 'framevalue': str(arg)})
self.send_frames(FrameSetTypes.DODISCOVER, frames)
def processpkt_netconfig(self, drone, _unused_srcaddr, jsonobj):
'''We want to trigger ARP discovery when we hear a 'netconfig' packet
Build up the parameters for the discovery
action, then send it to drone.request_discovery(...)
To build up the parameters, you use ConfigFile.agent_params()
which will pull values from the system configuration.
'''
init_params = ConfigFile.agent_params(self.config, 'discovery', '#ARP',
drone.designation)
data = jsonobj['data'] # the data portion of the JSON message
discovery_args = []
for devname in data.keys():
#print >> sys.stderr, "*** devname:", devname
devinfo = data[devname]
if discovery_indicates_link_is_up(devinfo):
params = pyConfigContext(init_params)
params[CONFIGNAME_INSTANCE] = '_ARP_' + devname
params[CONFIGNAME_DEVNAME] = devname
#print >> sys.stderr, '#ARP parameters:', params
discovery_args.append(params)
if discovery_args:
drone.request_discovery(discovery_args)
def request_discovery(self, args): ##< A vector of arguments containing
'''Send our drone a request to perform discovery
We send a DISCNAME frame with the instance name
then an optional DISCINTERVAL frame with the repeat interval
then a DISCJSON frame with the JSON data for the discovery operation.
Our argument is a vector of pyConfigContext objects with values for
'instance' Name of this discovery instance
'interval' How often to repeat this discovery action
'timeout' How long to wait before considering this discovery failed...
'''
#fs = pyFrameSet(FrameSetTypes.DODISCOVER)
frames = []
for arg in args:
agent_params = ConfigFile.agent_params(CMAdb.io.config, 'discovery',
arg[CONFIGNAME_TYPE], self.designation)
for key in ('repeat', 'warn' 'timeout', 'nice'):
if key in agent_params and key not in arg:
arg[key] = agent_params[arg]
instance = arg['instance']
frames.append({'frametype': FrameTypes.DISCNAME, 'framevalue': instance})
if 'repeat' in arg:
interval = int(arg['repeat'])
frames.append({'frametype': FrameTypes.DISCINTERVAL, 'framevalue': int(interval)})
frames.append({'frametype': FrameTypes.DISCJSON, 'framevalue': str(arg)})
# This doesn't work if the client has bound to a VIP
ourip = self.select_ip() # meaning select our primary IP
ourip = pyNetAddr(ourip)
if ourip.port() == 0:
ourip.setport(self.port)
#print >> sys.stderr, ('ADDING PACKET TO TRANSACTION: %s', str(frames))
if CMAdb.debug:
CMAdb.log.debug('Sending Discovery request(%s, %s) to %s Frames: %s'
% (instance, str(interval), str(ourip), str(frames)))
CMAdb.transaction.add_packet(ourip, FrameSetTypes.DODISCOVER, frames)
def processpkt_netconfig(self, drone, unused_srcaddr, jsonobj):
'''We want to trigger Switch discovery when we hear a 'netconfig' packet
Build up the parameters for the discovery
action, then send it to drone.request_discovery(...)
To build up the parameters, you use ConfigFile.agent_params()
which will pull values from the system configuration.
'''
unused_srcaddr = unused_srcaddr # make pylint happy
init_params = ConfigFile.agent_params(self.config, 'discovery', '#SWITCH'
, drone.designation)
data = jsonobj['data'] # the data portion of the JSON message
discovery_args = []
for devname in data.keys():
devinfo = data[devname]
if (str(devinfo['operstate']) == 'up' and str(devinfo['carrier']) == 'True'
and str(devinfo['address']) != '00-00-00-00-00-00'
and str(devinfo['address']) != ''):
params = pyConfigContext(init_params)
params[CONFIGNAME_INSTANCE] = '#SWITCH_' + devname
params[CONFIGNAME_DEVNAME] = devname
params[CONFIGNAME_SWPROTOS] = ["lldp", "cdp"]
#print >> sys.stderr, '***#SWITCH parameters:', params
discovery_args.append(params)
if discovery_args:
drone.request_discovery(discovery_args)
def processpkt(self, drone, unused_srcaddr, jsonobj):
"Send commands to gather discovery data from /proc/sys"
unused_srcaddr = unused_srcaddr
data = jsonobj['data'] # The data portion of the JSON message
osfield = 'operating-system'
if osfield not in data:
self.log.warning('OS name not found in %s' % str(data))
return
osname = data[osfield]
if osname.find('Linux') == -1 and osname.find('linux') == -1:
self.log.info('ProcSysDiscovery: OS name is not Linux: %s' %
str(osname))
return
params = ConfigFile.agent_params(self.config, 'discovery', 'proc_sys',
drone.designation)
params['parameters'] = pyConfigContext(
{'ASSIM_discoverdir': '/proc/sys'})
params[CONFIGNAME_TYPE] = 'proc_sys'
params[CONFIGNAME_INSTANCE] = '_auto_proc_sys'
# Request discovery of checksums of all the binaries talking (tcp) over the network
if self.debug:
self.log.debug('REQUESTING /proc/sys DISCOVERY')
drone.request_discovery((params, ))
def _add_service_monitoring(self, drone, monitoredservice, moninfo):
'''
We start the monitoring of 'monitoredservice' using the information
in 'moninfo' - which came from MonitoringRule.constructaction()
'''
monitorclass = moninfo['monitorclass']
monitortype = moninfo['monitortype']
if 'provider' in moninfo:
monitorprovider = moninfo['provider']
classtype = "%s::%s:%s" % (monitorclass, monitorprovider,
monitortype)
else:
monitorprovider = None
classtype = "%s::%s" % (monitorclass, monitortype)
# Compute interval and timeout - based on global 'config'
params = ConfigFile.agent_params(self.config, 'monitoring', classtype,
drone.designation)
monitorinterval = params['parameters']['repeat']
monitortimeout = params['parameters']['timeout']
monitorarglist = moninfo.get('arglist', {})
monargv = moninfo.get('argv', None)
# Make up a monitor name that should be unique to us -- but reproducible
# We create the monitor name from the host name, the monitor class,
# monitoring type and a hash of the arguments to the monitoring agent
# Note that this is different from the hashed service/entity name, since we could
# have multiple ways of monitoring the same entity
d = hashlib.md5()
# pylint mistakenly thinks md5 objects don't have update member
# pylint: disable=E1101
d.update(
'%s:%s:%s:%s' %
(drone.designation, monitorclass, monitortype, monitorprovider))
if monitorarglist is not None:
names = monitorarglist.keys()
names.sort()
for name in names:
# pylint thinks md5 objects don't have update member
# pylint: disable=E1101
d.update('"%s": "%s"' % (name, monitorarglist[name]))
monitorname = (
'%s:%s:%s::%s' %
(drone.designation, monitorclass, monitortype, d.hexdigest()))
monnode = self.store.load_or_create(MonitorAction,
domain=drone.domain,
monitorname=monitorname,
monitorclass=monitorclass,
monitortype=monitortype,
interval=monitorinterval,
timeout=monitortimeout,
provider=monitorprovider,
arglist=monitorarglist,
argv=monargv)
if monitorclass == 'nagios':
monnode.nagiospath = self.config['monitoring']['nagiospath']
if not Store.is_abstract(monnode):
print >> sys.stderr, ('Previously monitored %s on %s' %
(monitortype, drone.designation))
monnode.activate(monitoredservice, drone)
def _add_service_monitoring(self, drone, monitoredservice, moninfo):
'''
We start the monitoring of 'monitoredservice' using the information
in 'moninfo' - which came from MonitoringRule.constructaction()
'''
monitorclass = moninfo['monitorclass']
monitortype = moninfo['monitortype']
if 'provider' in moninfo:
monitorprovider = moninfo['provider']
classtype = "%s::%s:%s" % (monitorclass, monitorprovider, monitortype)
else:
monitorprovider = None
classtype = "%s::%s" % (monitorclass, monitortype)
# Compute interval and timeout - based on global 'config'
params = ConfigFile.agent_params(self.config, 'monitoring', classtype, drone.designation)
monitorinterval = params['parameters']['repeat']
monitortimeout = params['parameters']['timeout']
monitorarglist = moninfo.get('arglist', {})
monargv = moninfo.get('argv', None)
# Make up a monitor name that should be unique to us -- but reproducible
# We create the monitor name from the host name, the monitor class,
# monitoring type and a hash of the arguments to the monitoring agent
# Note that this is different from the hashed service/entity name, since we could
# have multiple ways of monitoring the same entity
d = hashlib.md5()
# pylint mistakenly thinks md5 objects don't have update member
# pylint: disable=E1101
d.update('%s:%s:%s:%s'
% (drone.designation, monitorclass, monitortype, monitorprovider))
if monitorarglist is not None:
names = monitorarglist.keys()
names.sort()
for name in names:
# pylint thinks md5 objects don't have update member
# pylint: disable=E1101
d.update('"%s": "%s"' % (name, monitorarglist[name]))
monitorname = ('%s:%s:%s::%s'
% (drone.designation, monitorclass, monitortype, d.hexdigest()))
monnode = self.store.load_or_create(MonitorAction, domain=drone.domain
, monitorname=monitorname, monitorclass=monitorclass
, monitortype=monitortype, interval=monitorinterval, timeout=monitortimeout
, provider=monitorprovider, arglist=monitorarglist, argv=monargv)
if monitorclass == 'nagios':
monnode.nagiospath = self.config['monitoring']['nagiospath']
if not Store.is_abstract(monnode):
print >> sys.stderr, ('Previously monitored %s on %s'
% (monitortype, drone.designation))
monnode.activate(monitoredservice, drone)
def __init__(self, addrframesetpairs, sleepatend=0):
if isinstance(addrframesetpairs, tuple):
addrframesetpairs = addrframesetpairs
self.inframes = addrframesetpairs
self.packetswritten = []
self.packetsread = 0
self.sleepatend = sleepatend
self.index = 0
self.writecount = 0
self.config = ConfigFile().complete_config()
(self.pipe_read, self.pipe_write) = os.pipe()
os.write(self.pipe_write, ' ')
os.close(self.pipe_write)
self.pipe_write = -1
self.atend = False
self.readfails = 0
self.initpackets = len(self.inframes)
def processtcpdiscoverypkt(self, drone, unused_srcaddr, jsonobj):
"Send commands to generate checksums for this Drone's net-facing things"
unused_srcaddr = unused_srcaddr
params = ConfigFile.agent_params(self.config, 'discovery', 'checksums',
drone.designation)
params['parameters'] = pyConfigContext({
'ASSIM_sumcmds': [
'/usr/bin/sha256sum', '/usr/bin/sha224sum',
'/usr/bin/sha384sum', '/usr/bin/sha512sum', '/usr/bin/sha1sum',
'/usr/bin/md5sum', '/usr/bin/cksum', '/usr/bin/crc32'
],
'ASSIM_filelist': [
'/bin/sh', '/bin/bash', '/bin/login', '/usr/bin/passwd',
'/tmp/foobar'
]
})
params[CONFIGNAME_TYPE] = 'checksums'
params[CONFIGNAME_INSTANCE] = '_auto_checksumdiscovery'
data = jsonobj['data'] # The data portion of the JSON message
for procname in data.keys(
): # List of nanoprobe-assigned names of processes...
procinfo = data[procname]
if 'exe' not in procinfo:
continue
exename = procinfo.get('exe')
# dups (if any) are removed by the agent
params['parameters']['ASSIM_filelist'].append(exename)
if exename.endswith('/java'):
# Special case for some/many JAVA programs - find the jars...
if 'cmdline' not in procinfo:
continue
cmdline = procinfo.get('cmdline')
for j in range(0, len(cmdline)):
# The argument following -cp is the ':'-separated CLASSPATH
if cmdline[j] == '-cp' and j < len(cmdline) - 1:
jars = cmdline[j + 1].split(':')
for jar in jars:
params['parameters']['ASSIM_filelist'].append(jar)
break
# Request discovery of checksums of all the binaries talking (tcp) over the network
print >> sys.stderr, ('REQUESTING CHECKSUM MONITORING OF %d files' %
(len(params['parameters']['ASSIM_filelist'])))
drone.request_discovery((params, ))
def send_hbmsg(self, dest, fstype, addrlist):
'''Send a message with an attached pyNetAddr list - each including port numbers'
This is intended primarily for start or stop heartbeating messages.'''
# Now we create a collection of frames that looks like this:
#
# One FrameTypes.RSCJSON frame containing JSON Heartbeat parameters
# one frame per dest, type FrameTypes.IPPORT
#
params = ConfigFile.agent_params(CMAdb.io.config
, 'heartbeats', None, self.designation)
framelist = [{'frametype': FrameTypes.RSCJSON, 'framevalue': str(params)},]
for addr in addrlist:
if addr is None:
continue
framelist.append({'frametype': FrameTypes.IPPORT, 'framevalue': addr})
CMAdb.transaction.add_packet(dest, fstype, framelist)
def processpkt(self, drone, unused_srcaddr, jsonobj):
"Send commands to gather discovery data from /proc/sys"
unused_srcaddr = unused_srcaddr
data = jsonobj["data"] # The data portion of the JSON message
osfield = "operating-system"
if osfield not in data:
self.log.warning("OS name not found in %s" % str(data))
return
osname = data[osfield]
if osname.find("Linux") == -1 and osname.find("linux") == -1:
self.log.info("ProcSysDiscovery: OS name is not Linux: %s" % str(osname))
return
params = ConfigFile.agent_params(self.config, "discovery", "proc_sys", drone.designation)
params["parameters"] = pyConfigContext({"ASSIM_discoverdir": "/proc/sys"})
params[CONFIGNAME_TYPE] = "proc_sys"
params[CONFIGNAME_INSTANCE] = "_auto_proc_sys"
# Request discovery of checksums of all the binaries talking (tcp) over the network
if self.debug:
self.log.debug("REQUESTING /proc/sys DISCOVERY")
drone.request_discovery((params,))
def request_discovery(self, args): ##< A vector of arguments containing
'''Send our System a request to perform discovery
We send a DISCNAME frame with the instance name
then an optional DISCINTERVAL frame with the repeat interval
then a DISCJSON frame with the JSON data for the discovery operation.
Our argument is a vector of pyConfigContext objects with values for
'instance' Name of this discovery instance
'interval' How often to repeat this discovery action
'timeout' How long to wait before considering this discovery failed...
'''
#fs = pyFrameSet(FrameSetTypes.DODISCOVER)
frames = []
for arg in args:
agent_params = ConfigFile.agent_params(CMAdb.io.config,
'discovery',
arg[CONFIGNAME_TYPE],
self.designation)
for key in ('repeat', 'warn' 'timeout', 'nice'):
if key in agent_params and key not in arg:
arg[key] = agent_params[arg]
instance = arg['instance']
frames.append({
'frametype': FrameTypes.DISCNAME,
'framevalue': instance
})
if 'repeat' in arg:
interval = int(arg['repeat'])
frames.append({
'frametype': FrameTypes.DISCINTERVAL,
'framevalue': int(interval)
})
else:
interval = 0
frames.append({
'frametype': FrameTypes.DISCJSON,
'framevalue': str(arg)
})
self.send_frames(FrameSetTypes.DODISCOVER, frames)
def processpkt(self, drone, unused_srcaddr, jsonobj):
"Send commands to gather discovery data from /proc/sys"
unused_srcaddr = unused_srcaddr
data = jsonobj['data'] # The data portion of the JSON message
osfield='operating-system'
if osfield not in data:
self.log.warning('OS name not found in %s' % str(data))
return
osname = data[osfield]
if osname.find('Linux') == -1 and osname.find('linux') == -1:
self.log.info('ProcSysDiscovery: OS name is not Linux: %s' % str(osname))
return
params = ConfigFile.agent_params(self.config, 'discovery', 'proc_sys', drone.designation)
params['parameters'] = pyConfigContext({'ASSIM_discoverdir': '/proc/sys' })
params[CONFIGNAME_TYPE] = 'proc_sys'
params[CONFIGNAME_INSTANCE] = '_auto_proc_sys'
# Request discovery of checksums of all the binaries talking (tcp) over the network
if self.debug:
self.log.debug('REQUESTING /proc/sys DISCOVERY')
drone.request_discovery((params,))
def processtcpdiscoverypkt(self, drone, unused_srcaddr, jsonobj):
"Send commands to generate checksums for this Drone's net-facing things"
unused_srcaddr = unused_srcaddr
params = ConfigFile.agent_params(self.config, 'discovery', 'checksums',
drone.designation)
sumcmds = self.config['checksum_cmds']
filelist = self.config['checksum_files']
filelist.extend(sumcmds)
params['parameters'] = pyConfigContext()
params[CONFIGNAME_TYPE] = 'checksums'
params[CONFIGNAME_INSTANCE] = '_auto_checksumdiscovery'
data = jsonobj['data'] # The data portion of the JSON message
for procname in data.keys(): # List of of process names...
procinfo = data[procname] # (names assigned by the nanoprobe)
if 'exe' not in procinfo:
continue
exename = procinfo.get('exe')
# dups (if any) are removed by the agent
filelist.append(exename)
if exename.endswith('/java'):
# Special case for some/many JAVA programs - find the jars...
if 'cmdline' not in procinfo:
continue
cmdline = procinfo.get('cmdline')
for j in range(0, len(cmdline)):
# The argument following -cp is the ':'-separated CLASSPATH
if cmdline[j] == '-cp' and j < len(cmdline)-1:
jars = cmdline[j+1].split(':')
for jar in jars:
filelist.append(jar)
break
params['parameters']['ASSIM_sumcmds'] = sumcmds
params['parameters']['ASSIM_filelist'] = filelist
# Request checksums of all the binaries talking (tcp) over the network
print >> sys.stderr, ('REQUESTING CHECKSUM MONITORING OF %d files'
% (len(params['parameters']['ASSIM_filelist'])))
drone.request_discovery((params,))
def processtcpdiscoverypkt(self, drone, _unused_srcaddr, jsonobj):
"Send commands generating checksums for the Systems's net-facing things"
params = ConfigFile.agent_params(self.config, 'discovery', 'checksums',
drone.designation)
sumcmds = self.config['checksum_cmds']
filelist = self.config['checksum_files']
filelist.extend(sumcmds)
params['parameters'] = pyConfigContext()
params[CONFIGNAME_TYPE] = 'checksums'
params[CONFIGNAME_INSTANCE] = '_auto_checksumdiscovery'
data = jsonobj['data'] # The data portion of the JSON message
for procname in data.keys(): # List of of process names...
procinfo = data[procname] # (names assigned by the nanoprobe)
if 'exe' not in procinfo:
continue
exename = procinfo.get('exe')
# dups (if any) are removed by the agent
filelist.append(exename)
if exename.endswith('/java'):
# Special case for some/many JAVA programs - find the jars...
if 'cmdline' not in procinfo:
continue
cmdline = procinfo.get('cmdline')
for j in range(0, len(cmdline)):
# The argument following -cp is the ':'-separated CLASSPATH
if cmdline[j] == '-cp' and j < len(cmdline) - 1:
jars = cmdline[j + 1].split(':')
for jar in jars:
filelist.append(jar)
break
params['parameters']['ASSIM_sumcmds'] = sumcmds
params['parameters']['ASSIM_filelist'] = filelist
# Request checksums of all the binaries talking (tcp) over the network
print >> sys.stderr, ('REQUESTING CHECKSUM MONITORING OF %d files' %
(len(params['parameters']['ASSIM_filelist'])))
drone.request_discovery((params, ))
def processtcpdiscoverypkt(self, drone, _unused_srcaddr, jsonobj):
"Send commands generating checksums for the Systems's net-facing things"
params = ConfigFile.agent_params(self.config, "discovery", "checksums", drone.designation)
sumcmds = self.config["checksum_cmds"]
filelist = self.config["checksum_files"]
filelist.extend(sumcmds)
params["parameters"] = pyConfigContext()
params[CONFIGNAME_TYPE] = "checksums"
params[CONFIGNAME_INSTANCE] = "_auto_checksumdiscovery"
data = jsonobj["data"] # The data portion of the JSON message
for procname in data.keys(): # List of of process names...
procinfo = data[procname] # (names assigned by the nanoprobe)
if "exe" not in procinfo:
continue
exename = procinfo.get("exe")
# dups (if any) are removed by the agent
filelist.append(exename)
if exename.endswith("/java"):
# Special case for some/many JAVA programs - find the jars...
if "cmdline" not in procinfo:
continue
cmdline = procinfo.get("cmdline")
for j in range(0, len(cmdline)):
# The argument following -cp is the ':'-separated CLASSPATH
if cmdline[j] == "-cp" and j < len(cmdline) - 1:
jars = cmdline[j + 1].split(":")
for jar in jars:
filelist.append(jar)
break
params["parameters"]["ASSIM_sumcmds"] = sumcmds
params["parameters"]["ASSIM_filelist"] = filelist
# Request checksums of all the binaries talking (tcp) over the network
print >>sys.stderr, (
"REQUESTING CHECKSUM MONITORING OF %d files" % (len(params["parameters"]["ASSIM_filelist"]))
)
drone.request_discovery((params,))
def processpkt_netconfig(self, drone, _unused_srcaddr, jsonobj):
'''We want to trigger ARP discovery when we hear a 'netconfig' packet
Build up the parameters for the discovery
action, then send it to drone.request_discovery(...)
To build up the parameters, you use ConfigFile.agent_params()
which will pull values from the system configuration.
'''
init_params = ConfigFile.agent_params(self.config, 'discovery', '#ARP', drone.designation)
data = jsonobj['data'] # the data portion of the JSON message
discovery_args = []
for devname in data.keys():
#print >> sys.stderr, "*** devname:", devname
devinfo = data[devname]
if discovery_indicates_link_is_up(devinfo):
params = pyConfigContext(init_params)
params[CONFIGNAME_INSTANCE] = '_ARP_' + devname
params[CONFIGNAME_DEVNAME] = devname
#print >> sys.stderr, '#ARP parameters:', params
discovery_args.append(params)
if discovery_args:
drone.request_discovery(discovery_args)
def geninitconfig(ouraddr):
configinfo = ConfigFile()
for j in ('cmainit', 'cmaaddr', 'cmadisc', 'cmafail'):
configinfo[j] = ouraddr
configinfo['outsig'] = pySignFrame(1)
return configinfo.complete_config()
def dispatch(self, origaddr, frameset):
json = None
addrstr = repr(origaddr)
fstype = frameset.get_framesettype()
localtime = None
listenaddr = None
keyid = None
pubkey = None
keysize = None
#print >> sys.stderr, ("DispatchSTARTUP: received [%s] FrameSet from [%s]"
#% (FrameSetTypes.get(fstype)[0], addrstr))
if CMAdb.debug:
CMAdb.log.debug(
"DispatchSTARTUP: received [%s] FrameSet from [%s]" %
(FrameSetTypes.get(fstype)[0], addrstr))
if not self.io.connactive(origaddr):
self.io.closeconn(DEFAULT_FSP_QID, origaddr)
CMAdb.transaction.post_transaction_packets.append(
FrameSetTypes.ACKSTARTUP)
for frame in frameset.iter():
frametype = frame.frametype()
if frametype == FrameTypes.WALLCLOCK:
localtime = str(frame.getint())
elif frametype == FrameTypes.IPPORT:
listenaddr = frame.getnetaddr()
elif frametype == FrameTypes.HOSTNAME:
sysname = frame.getstr()
if sysname == CMAdb.nodename:
if origaddr.islocal():
CMAdb.log.info(
"Received STARTUP from local system (%s)" %
addrstr)
else:
addresses = ['127.0.0.1', '::ffff:127.0.0.1', '::1']
for address in addresses:
localhost = pyNetAddr(address)
self.io.addalias(localhost, origaddr)
CMAdb.log.info("Aliasing %s to %s" %
(localhost, origaddr))
elif frametype == FrameTypes.JSDISCOVER:
json = frame.getstr()
#print >> sys.stderr, 'GOT JSDISCOVER JSON: [%s] (strlen:%s,framelen:%s)' \
#% (json, len(json), frame.framelen())
elif frametype == FrameTypes.KEYID:
keyid = frame.getstr()
elif frametype == FrameTypes.PUBKEYCURVE25519:
pubkey = frame.framevalue()
keysize = frame.framelen()
joininfo = pyConfigContext(init=json)
origaddr, isNAT = self.validate_source_ip(sysname, origaddr, joininfo,
listenaddr)
CMAdb.log.info(
'Drone %s registered from address %s (%s) port %s, key_id %s' %
(sysname, origaddr, addrstr, origaddr.port(), keyid))
drone = self.droneinfo.add(sysname,
'STARTUP packet',
port=origaddr.port(),
primary_ip_addr=str(origaddr))
drone.listenaddr = str(listenaddr) # Seems good to hang onto this...
drone.isNAT = isNAT # ditto...
if CMAdb.debug:
CMAdb.log.debug('DRONE select_ip() result: %s' %
(drone.select_ip()))
CMAdb.log.debug('DRONE listenaddr: %s' % (drone.listenaddr))
CMAdb.log.debug('DRONE port: %s (%s)' %
(drone.port, type(drone.port)))
# Did they give us the crypto info we need?
if keyid is None or pubkey is None:
if CMAdb.debug:
CMAdb.log.debug(
'Drone %s registered with keyid %s and pubkey provided: %s'
% (self, keyid, pubkey is not None))
else:
if drone.key_id == '':
if not keyid.startswith(sysname + "@@"):
CMAdb.log.warning(
"Drone %s wants to register with key_id %s -- permitted.",
sysname, keyid)
if not cryptcurve25519_save_public_key(keyid, pubkey, keysize):
raise ValueError(
"Drone %s public key (key_id %s, %d bytes) is invalid."
% (sysname, keyid, keysize))
elif drone.key_id != keyid:
raise ValueError(
"Drone %s tried to register with key_id %s instead of %s."
% (sysname, keyid, drone.key_id))
drone.set_crypto_identity(keyid=keyid)
pyCryptFrame.dest_set_key_id(origaddr, keyid)
#
# THIS IS HERE BECAUSE OF A PROTOCOL BUG...
# @FIXME Protocol bug when starting up a connection if our first (this) packet gets lost,
# then the protocol doesn't retransmit it.
# More specifically, it seems to clear it out of the queue.
# This might be CMA bug or a protocol bug. It's not clear...
# The packet goes into the queue, but if that packet is lost in transmission, then when
# we come back around here, it's not in the queue any more, even though it
# definitely wasn't ACKed.
# Once this is fixed, this "add_packet" call needs to go *after* the 'if' statement below.
#
CMAdb.transaction.add_packet(origaddr, FrameSetTypes.SETCONFIG,
(str(self.config), ),
FrameTypes.CONFIGJSON)
if (localtime is not None):
if (drone.lastjoin == localtime):
CMAdb.log.warning('Drone %s [%s] sent duplicate STARTUP' %
(sysname, origaddr))
if CMAdb.debug:
self.io.log_conn(origaddr)
return
drone.lastjoin = localtime
#print >> sys.stderr, 'DRONE from find: ', drone, type(drone), drone.port
drone.startaddr = str(origaddr)
if json is not None:
drone.logjson(origaddr, json)
if CMAdb.debug:
CMAdb.log.debug('Joining TheOneRing: %s / %s / %s' %
(drone, type(drone), drone.port))
CMAdb.cdb.TheOneRing.join(drone)
if CMAdb.debug:
CMAdb.log.debug('Requesting Discovery from %s' % str(drone))
discovery_params = []
for agent in self.config['initial_discovery']:
params = ConfigFile.agent_params(self.config, 'discovery', agent,
sysname)
params['agent'] = agent
params['instance'] = '_init_%s' % agent
discovery_params.append(params)
# Discover the permissions of all the lists of files we're configured to ask about
# Note that there are several lists to keep the amount of data in any one list
# down to a somewhat more reasonable level. 'fileattrs' output is really verbose
for pathlist_name in self.config['perm_discovery_lists']:
paths = self.config[pathlist_name]
params = ConfigFile.agent_params(self.config, 'discovery',
'fileattrs', sysname)
params['agent'] = 'fileattrs'
params['instance'] = pathlist_name
params['parameters'] = {'ASSIM_filelist': paths}
discovery_params.append(params)
if CMAdb.debug:
CMAdb.log.debug('Discovery details: %s' % str(discovery_params))
for item in discovery_params:
CMAdb.log.debug('Discovery item details: %s' % str(item))
drone.request_discovery(discovery_params)
AssimEvent(drone, AssimEvent.OBJUP)
def __init__(self, config, packetio, store, log, debug):
BestPractices.__init__(self, config, packetio, store, log, debug)
from cmaconfig import ConfigFile
ConfigFile.register_callback(BestPracticesCMA.configcallback, args=None)
def main():
'Main program for the CMA (Collective Management Authority)'
py2neo_major_version = int(PY2NEO_VERSION.partition('.')[0])
if py2neo_major_version not in SUPPORTED_PY2NEO_VERSIONS:
raise EnvironmentError('py2neo version %s not supported' %
PY2NEO_VERSION)
DefaultPort = 1984
# VERY Linux-specific - but useful and apparently correct ;-)
PrimaryIPcmd = \
"ip address show primary scope global | grep '^ *inet' | sed -e 's%^ *inet *%%' -e 's%/.*%%'"
ipfd = os.popen(PrimaryIPcmd, 'r')
OurAddrStr = ('%s:%d' % (ipfd.readline().rstrip(), DefaultPort))
ipfd.close()
parser = optparse.OptionParser(
prog='CMA',
version=AssimCtypes.VERSION_STRING,
description=
'Collective Management Authority for the Assimilation System',
usage='cma.py [--bind address:port]')
parser.add_option(
'-b',
'--bind',
action='store',
default=None,
dest='bind',
metavar='address:port-to-bind-to',
help='Address:port to listen to - for nanoprobes to connect to')
parser.add_option(
'-d',
'--debug',
action='store',
default=0,
dest='debug',
help=
'enable debug for CMA and libraries - value is debug level for C libraries.'
)
parser.add_option('-s',
'--status',
action='store_true',
default=False,
dest='status',
help='Return status of running CMA')
parser.add_option('-k',
'--kill',
action='store_true',
default=False,
dest='kill',
help='Shut down running CMA.')
parser.add_option('-e',
'--erasedb',
action='store_true',
default=False,
dest='erasedb',
help='Erase Neo4J before starting')
parser.add_option('-f',
'--foreground',
action='store_true',
default=False,
dest='foreground',
help='keep the CMA from going into the background')
parser.add_option('-p',
'--pidfile',
action='store',
default='/var/run/assimilation/cma',
dest='pidfile',
metavar='pidfile-pathname',
help='full pathname of where to locate our pid file')
parser.add_option('-T',
'--trace',
action='store_true',
default=False,
dest='doTrace',
help='Trace CMA execution')
parser.add_option('-u',
'--user',
action='store',
default=CMAUSERID,
dest='userid',
metavar='userid',
help='userid to run the CMA as')
opt = parser.parse_args()[0]
from AssimCtypes import daemonize_me, assimilation_openlog, are_we_already_running, \
kill_pid_service, pidrunningstat_to_status, remove_pid_file, rmpid_and_exit_on_signal
if opt.status:
rc = pidrunningstat_to_status(are_we_already_running(
opt.pidfile, None))
return rc
if opt.kill:
if kill_pid_service(opt.pidfile, 15) < 0:
print >> sys.stderr, "Unable to stop CMA."
return 1
return 0
opt.debug = int(opt.debug)
# This doesn't seem to work no matter where I invoke it...
# But if we don't fork in daemonize_me() ('C' code), it works great...
# def cleanup():
# remove_pid_file(opt.pidfile)
# atexit.register(cleanup)
# signal.signal(signal.SIGTERM, lambda sig, stack: sys.exit(0))
# signal.signal(signal.SIGINT, lambda sig, stack: sys.exit(0))
from cmadb import CMAdb
CMAdb.running_under_docker()
make_pid_dir(opt.pidfile, opt.userid)
make_key_dir(CRYPTKEYDIR, opt.userid)
cryptwarnings = pyCryptCurve25519.initkeys()
for warn in cryptwarnings:
print >> sys.stderr, ("WARNING: %s" % warn)
#print >> sys.stderr, 'All known key ids:'
keyids = pyCryptFrame.get_key_ids()
keyids.sort()
for keyid in keyids:
if not keyid.startswith(CMA_KEY_PREFIX):
try:
# @FIXME This is not an ideal way to associate identities with hosts
# in a multi-tenant environment
# @FIXME - don't think I need to do the associate_identity at all any more...
hostname, notused_post = keyid.split('@@', 1)
notused_post = notused_post
pyCryptFrame.associate_identity(hostname, keyid)
except ValueError:
pass
#print >> sys.stderr, '> %s/%s' % (keyid, pyCryptFrame.get_identity(keyid))
daemonize_me(opt.foreground, '/', opt.pidfile, 20)
rmpid_and_exit_on_signal(opt.pidfile, signal.SIGTERM)
# Next statement can't appear before daemonize_me() or bind() fails -- not quite sure why...
assimilation_openlog("cma")
from packetlistener import PacketListener
from messagedispatcher import MessageDispatcher
from dispatchtarget import DispatchTarget
from monitoring import MonitoringRule
from AssimCclasses import pyNetAddr, pySignFrame, pyReliableUDP, \
pyPacketDecoder
from AssimCtypes import CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR, CONFIGNAME_CMADISCOVER, \
CONFIGNAME_CMAFAIL, CONFIGNAME_CMAPORT, CONFIGNAME_OUTSIG, CONFIGNAME_COMPRESSTYPE, \
CONFIGNAME_COMPRESS, proj_class_incr_debug, LONG_LICENSE_STRING, MONRULEINSTALL_DIR
if opt.debug:
print >> sys.stderr, ('Setting debug to %s' % opt.debug)
for debug in range(opt.debug):
debug = debug
print >> sys.stderr, ('Incrementing C-level debug by one.')
proj_class_incr_debug(None)
# Input our monitoring rule templates
# They only exist in flat files and in memory - they aren't in the database
MonitoringRule.load_tree(MONRULEINSTALL_DIR)
print >> sys.stderr, ('Monitoring rules loaded from %s' %
MONRULEINSTALL_DIR)
execobserver_constraints = {
'nodetype': [
'Drone',
'IPaddrNode',
'MonitorAction',
'NICNode',
'ProcessNode',
'SystemNode',
]
}
ForkExecObserver(constraints=execobserver_constraints,
scriptdir=NOTIFICATION_SCRIPT_DIR)
print >> sys.stderr, ('Fork/Event observer dispatching from %s' %
NOTIFICATION_SCRIPT_DIR)
if opt.bind is not None:
OurAddrStr = opt.bind
OurAddr = pyNetAddr(OurAddrStr)
if OurAddr.port() == 0:
OurAddr.setport(DefaultPort)
try:
configinfo = ConfigFile(filename=CMAINITFILE)
except IOError:
configinfo = ConfigFile()
if opt.bind is not None:
bindaddr = pyNetAddr(opt.bind)
if bindaddr.port() == 0:
bindaddr.setport(ConfigFile[CONFIGNAME_CMAPORT])
configinfo[CONFIGNAME_CMAINIT] = bindaddr
configinfo[CONFIGNAME_CMADISCOVER] = OurAddr
configinfo[CONFIGNAME_CMAFAIL] = OurAddr
configinfo[CONFIGNAME_CMAADDR] = OurAddr
if (CONFIGNAME_COMPRESSTYPE in configinfo):
configinfo[CONFIGNAME_COMPRESS] \
= pyCompressFrame(compression_method=configinfo[CONFIGNAME_COMPRESSTYPE])
configinfo[CONFIGNAME_OUTSIG] = pySignFrame(1)
config = configinfo.complete_config()
addr = config[CONFIGNAME_CMAINIT]
# pylint is confused: addr is a pyNetAddr, not a pyConfigContext
# pylint: disable=E1101
if addr.port() == 0:
addr.setport(DefaultPort)
ourport = addr.port()
for elem in (CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR,
CONFIGNAME_CMADISCOVER, CONFIGNAME_CMAFAIL):
if elem in config:
config[elem] = pyNetAddr(str(config[elem]), port=ourport)
io = pyReliableUDP(config, pyPacketDecoder())
io.setrcvbufsize(
10 * 1024 *
1024) # No harm in asking - it will get us the best we can get...
io.setsendbufsize(
1024 * 1024) # Most of the traffic volume is inbound from discovery
drop_privileges_permanently(opt.userid)
try:
cmainit.CMAinit(io, cleanoutdb=opt.erasedb, debug=(opt.debug > 0))
except RuntimeError:
remove_pid_file(opt.pidfile)
raise
for warn in cryptwarnings:
CMAdb.log.warning(warn)
if CMAdb.cdb.db.neo4j_version[0] not in SUPPORTED_NEO4J_VERSIONS:
raise EnvironmentError('Neo4j version %s.%s.%s not supported' %
CMAdb.cdb.db.neo4j_version)
CMAdb.log.info('Listening on: %s' % str(config[CONFIGNAME_CMAINIT]))
CMAdb.log.info('Requesting return packets sent to: %s' % str(OurAddr))
CMAdb.log.info('Socket input buffer size: %d' % io.getrcvbufsize())
CMAdb.log.info('Socket output buffer size: %d' % io.getsendbufsize())
keyids = pyCryptFrame.get_key_ids()
keyids.sort()
for keyid in keyids:
CMAdb.log.info('KeyId %s Identity %s' %
(keyid, pyCryptFrame.get_identity(keyid)))
if CMAdb.debug:
CMAdb.log.debug('C-library Debug was set to %s' % opt.debug)
CMAdb.log.debug('TheOneRing created - id = %s' % CMAdb.TheOneRing)
CMAdb.log.debug('Config Object sent to nanoprobes: %s' % config)
jvmfd = os.popen('java -version 2>&1')
jvers = jvmfd.readline()
jvmfd.close()
disp = MessageDispatcher(DispatchTarget.dispatchtable)
neovers = CMAdb.cdb.db.neo4j_version
neoversstring = (('%s.%s.%s' if len(neovers) == 3 else '%s.%s.%s%s') %
neovers[0:3])
CMAdb.log.info('Starting CMA version %s - licensed under %s' %
(AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING))
CMAdb.log.info(
'Neo4j version %s // py2neo version %s // Python version %s // %s' %
(('%s.%s.%s' % CMAdb.cdb.db.neo4j_version), str(py2neo.__version__),
('%s.%s.%s' % sys.version_info[0:3]), jvers))
if opt.foreground:
print >> sys.stderr, (
'Starting CMA version %s - licensed under %s' %
(AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING))
print >> sys.stderr, (
'Neo4j version %s // py2neo version %s // Python version %s // %s'
% (neoversstring, PY2NEO_VERSION,
('%s.%s.%s' % sys.version_info[0:3]), jvers))
if len(neovers) > 3:
CMAdb.log.warning(
'Neo4j version %s is beta code - results not guaranteed.' %
str(neovers))
# Important to note that we don't want PacketListener to create its own 'io' object
# or it will screw up the ReliableUDP protocol...
listener = PacketListener(config, disp, io=io)
mandatory_modules = ['discoverylistener']
for mandatory in mandatory_modules:
importlib.import_module(mandatory)
#pylint is confused here...
# pylint: disable=E1133
for optional in config['optional_modules']:
importlib.import_module(optional)
if opt.doTrace:
import trace
tracer = trace.Trace(count=False, trace=True)
if CMAdb.debug:
CMAdb.log.debug('Starting up traced listener.listen(); debug=%d' %
opt.debug)
if opt.foreground:
print >> sys.stderr, (
'cma: Starting up traced listener.listen() in foreground; debug=%d'
% opt.debug)
tracer.run('listener.listen()')
else:
if CMAdb.debug:
CMAdb.log.debug(
'Starting up untraced listener.listen(); debug=%d' % opt.debug)
if opt.foreground:
print >> sys.stderr, (
'cma: Starting up untraced listener.listen() in foreground; debug=%d'
% opt.debug)
# This is kind of a kludge, we should really look again at
# at initializition and so on.
# This module *ought* to be optional.
# that would involve adding some Drone callbacks for creation of new Drones
BestPractices(config, io, CMAdb.store, CMAdb.log, opt.debug)
listener.listen()
return 0
def main():
'Main program for the CMA (Collective Management Authority)'
py2neo_major_version = int(PY2NEO_VERSION.partition('.')[0])
if py2neo_major_version not in SUPPORTED_PY2NEO_VERSIONS:
raise EnvironmentError('py2neo version %s not supported' % PY2NEO_VERSION)
DefaultPort = 1984
# VERY Linux-specific - but useful and apparently correct ;-)
PrimaryIPcmd = \
"ip address show primary scope global | grep '^ *inet' | sed -e 's%^ *inet *%%' -e 's%/.*%%'"
ipfd = os.popen(PrimaryIPcmd, 'r')
OurAddrStr = ('%s:%d' % (ipfd.readline().rstrip(), DefaultPort))
ipfd.close()
parser = optparse.OptionParser(prog='CMA', version=AssimCtypes.VERSION_STRING,
description='Collective Management Authority for the Assimilation System',
usage='cma.py [--bind address:port]')
parser.add_option('-b', '--bind', action='store', default=None, dest='bind'
, metavar='address:port-to-bind-to'
, help='Address:port to listen to - for nanoprobes to connect to')
parser.add_option('-d', '--debug', action='store', default=0, dest='debug'
, help='enable debug for CMA and libraries - value is debug level for C libraries.')
parser.add_option('-s', '--status', action='store_true', default=False, dest='status'
, help='Return status of running CMA')
parser.add_option('-k', '--kill', action='store_true', default=False, dest='kill'
, help='Shut down running CMA.')
parser.add_option('-e', '--erasedb', action='store_true', default=False, dest='erasedb'
, help='Erase Neo4J before starting')
parser.add_option('-f', '--foreground', action='store_true', default=False, dest='foreground'
, help='keep the CMA from going into the background')
parser.add_option('-p', '--pidfile', action='store', default='/var/run/assimilation/cma'
, dest='pidfile', metavar='pidfile-pathname'
, help='full pathname of where to locate our pid file')
parser.add_option('-T', '--trace', action='store_true', default=False, dest='doTrace'
, help='Trace CMA execution')
parser.add_option('-u', '--user', action='store', default=CMAUSERID, dest='userid'
, metavar='userid'
, help='userid to run the CMA as')
opt = parser.parse_args()[0]
from AssimCtypes import daemonize_me, assimilation_openlog, are_we_already_running, \
kill_pid_service, pidrunningstat_to_status, remove_pid_file, rmpid_and_exit_on_signal
if opt.status:
rc = pidrunningstat_to_status(are_we_already_running(opt.pidfile, None))
return rc
if opt.kill:
if kill_pid_service(opt.pidfile, 15) < 0:
print >> sys.stderr, "Unable to stop CMA."
return 1
return 0
opt.debug = int(opt.debug)
# This doesn't seem to work no matter where I invoke it...
# But if we don't fork in daemonize_me() ('C' code), it works great...
# def cleanup():
# remove_pid_file(opt.pidfile)
# atexit.register(cleanup)
# signal.signal(signal.SIGTERM, lambda sig, stack: sys.exit(0))
# signal.signal(signal.SIGINT, lambda sig, stack: sys.exit(0))
from cmadb import CMAdb
CMAdb.running_under_docker()
make_pid_dir(opt.pidfile, opt.userid)
make_key_dir(CRYPTKEYDIR, opt.userid)
cryptwarnings = pyCryptCurve25519.initkeys()
for warn in cryptwarnings:
print >> sys.stderr, ("WARNING: %s" % warn)
#print >> sys.stderr, 'All known key ids:'
keyids = pyCryptFrame.get_key_ids()
keyids.sort()
for keyid in keyids:
if not keyid.startswith(CMA_KEY_PREFIX):
try:
# @FIXME This is not an ideal way to associate identities with hosts
# in a multi-tenant environment
# @FIXME - don't think I need to do the associate_identity at all any more...
hostname, notused_post = keyid.split('@@', 1)
notused_post = notused_post
pyCryptFrame.associate_identity(hostname, keyid)
except ValueError:
pass
#print >> sys.stderr, '> %s/%s' % (keyid, pyCryptFrame.get_identity(keyid))
daemonize_me(opt.foreground, '/', opt.pidfile, 20)
rmpid_and_exit_on_signal(opt.pidfile, signal.SIGTERM)
# Next statement can't appear before daemonize_me() or bind() fails -- not quite sure why...
assimilation_openlog("cma")
from packetlistener import PacketListener
from messagedispatcher import MessageDispatcher
from dispatchtarget import DispatchTarget
from monitoring import MonitoringRule
from AssimCclasses import pyNetAddr, pySignFrame, pyReliableUDP, \
pyPacketDecoder
from AssimCtypes import CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR, CONFIGNAME_CMADISCOVER, \
CONFIGNAME_CMAFAIL, CONFIGNAME_CMAPORT, CONFIGNAME_OUTSIG, CONFIGNAME_COMPRESSTYPE, \
CONFIGNAME_COMPRESS, proj_class_incr_debug, LONG_LICENSE_STRING, MONRULEINSTALL_DIR
if opt.debug:
print >> sys.stderr, ('Setting debug to %s' % opt.debug)
for debug in range(opt.debug):
debug = debug
print >> sys.stderr, ('Incrementing C-level debug by one.')
proj_class_incr_debug(None)
# Input our monitoring rule templates
# They only exist in flat files and in memory - they aren't in the database
MonitoringRule.load_tree(MONRULEINSTALL_DIR)
print >> sys.stderr, ('Monitoring rules loaded from %s' % MONRULEINSTALL_DIR)
execobserver_constraints = {
'nodetype': ['Drone',
'IPaddrNode',
'MonitorAction',
'NICNode',
'ProcessNode',
'SystemNode',
]
}
ForkExecObserver(constraints=execobserver_constraints, scriptdir=NOTIFICATION_SCRIPT_DIR)
print >> sys.stderr, ('Fork/Event observer dispatching from %s' % NOTIFICATION_SCRIPT_DIR)
if opt.bind is not None:
OurAddrStr = opt.bind
OurAddr = pyNetAddr(OurAddrStr)
if OurAddr.port() == 0:
OurAddr.setport(DefaultPort)
try:
configinfo = ConfigFile(filename=CMAINITFILE)
except IOError:
configinfo = ConfigFile()
if opt.bind is not None:
bindaddr = pyNetAddr(opt.bind)
if bindaddr.port() == 0:
bindaddr.setport(ConfigFile[CONFIGNAME_CMAPORT])
configinfo[CONFIGNAME_CMAINIT] = bindaddr
configinfo[CONFIGNAME_CMADISCOVER] = OurAddr
configinfo[CONFIGNAME_CMAFAIL] = OurAddr
configinfo[CONFIGNAME_CMAADDR] = OurAddr
if (CONFIGNAME_COMPRESSTYPE in configinfo):
configinfo[CONFIGNAME_COMPRESS] \
= pyCompressFrame(compression_method=configinfo[CONFIGNAME_COMPRESSTYPE])
configinfo[CONFIGNAME_OUTSIG] = pySignFrame(1)
config = configinfo.complete_config()
addr = config[CONFIGNAME_CMAINIT]
# pylint is confused: addr is a pyNetAddr, not a pyConfigContext
# pylint: disable=E1101
if addr.port() == 0:
addr.setport(DefaultPort)
ourport = addr.port()
for elem in (CONFIGNAME_CMAINIT, CONFIGNAME_CMAADDR
, CONFIGNAME_CMADISCOVER, CONFIGNAME_CMAFAIL):
if elem in config:
config[elem] = pyNetAddr(str(config[elem]), port=ourport)
io = pyReliableUDP(config, pyPacketDecoder())
io.setrcvbufsize(10*1024*1024) # No harm in asking - it will get us the best we can get...
io.setsendbufsize(1024*1024) # Most of the traffic volume is inbound from discovery
drop_privileges_permanently(opt.userid)
try:
cmainit.CMAinit(io, cleanoutdb=opt.erasedb, debug=(opt.debug > 0))
except RuntimeError:
remove_pid_file(opt.pidfile)
raise
for warn in cryptwarnings:
CMAdb.log.warning(warn)
if CMAdb.cdb.db.neo4j_version[0] not in SUPPORTED_NEO4J_VERSIONS:
raise EnvironmentError('Neo4j version %s.%s.%s not supported'
% CMAdb.cdb.db.neo4j_version)
CMAdb.log.info('Listening on: %s' % str(config[CONFIGNAME_CMAINIT]))
CMAdb.log.info('Requesting return packets sent to: %s' % str(OurAddr))
CMAdb.log.info('Socket input buffer size: %d' % io.getrcvbufsize())
CMAdb.log.info('Socket output buffer size: %d' % io.getsendbufsize())
keyids = pyCryptFrame.get_key_ids()
keyids.sort()
for keyid in keyids:
CMAdb.log.info('KeyId %s Identity %s' % (keyid, pyCryptFrame.get_identity(keyid)))
if CMAdb.debug:
CMAdb.log.debug('C-library Debug was set to %s' % opt.debug)
CMAdb.log.debug('TheOneRing created - id = %s' % CMAdb.TheOneRing)
CMAdb.log.debug('Config Object sent to nanoprobes: %s' % config)
jvmfd = os.popen('java -version 2>&1')
jvers = jvmfd.readline()
jvmfd.close()
disp = MessageDispatcher(DispatchTarget.dispatchtable)
neovers = CMAdb.cdb.db.neo4j_version
neoversstring = (('%s.%s.%s'if len(neovers) == 3 else '%s.%s.%s%s')
% neovers[0:3])
CMAdb.log.info('Starting CMA version %s - licensed under %s'
% (AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING))
CMAdb.log.info('Neo4j version %s // py2neo version %s // Python version %s // %s'
% (('%s.%s.%s' % CMAdb.cdb.db.neo4j_version)
, str(py2neo.__version__)
, ('%s.%s.%s' % sys.version_info[0:3])
, jvers))
if opt.foreground:
print >> sys.stderr, ('Starting CMA version %s - licensed under %s'
% (AssimCtypes.VERSION_STRING, LONG_LICENSE_STRING))
print >> sys.stderr, ('Neo4j version %s // py2neo version %s // Python version %s // %s'
% ( neoversstring
, PY2NEO_VERSION
, ('%s.%s.%s' % sys.version_info[0:3])
, jvers))
if len(neovers) > 3:
CMAdb.log.warning('Neo4j version %s is beta code - results not guaranteed.'
% str(neovers))
# Important to note that we don't want PacketListener to create its own 'io' object
# or it will screw up the ReliableUDP protocol...
listener = PacketListener(config, disp, io=io)
mandatory_modules = [ 'discoverylistener' ]
for mandatory in mandatory_modules:
importlib.import_module(mandatory)
#pylint is confused here...
# pylint: disable=E1133
for optional in config['optional_modules']:
importlib.import_module(optional)
if opt.doTrace:
import trace
tracer = trace.Trace(count=False, trace=True)
if CMAdb.debug:
CMAdb.log.debug(
'Starting up traced listener.listen(); debug=%d' % opt.debug)
if opt.foreground:
print >> sys.stderr, (
'cma: Starting up traced listener.listen() in foreground; debug=%d' % opt.debug)
tracer.run('listener.listen()')
else:
if CMAdb.debug:
CMAdb.log.debug(
'Starting up untraced listener.listen(); debug=%d' % opt.debug)
if opt.foreground:
print >> sys.stderr, (
'cma: Starting up untraced listener.listen() in foreground; debug=%d' % opt.debug)
# This is kind of a kludge, we should really look again at
# at initializition and so on.
# This module *ought* to be optional.
# that would involve adding some Drone callbacks for creation of new Drones
BestPractices(config, io, CMAdb.store, CMAdb.log, opt.debug)
listener.listen()
return 0
def geninitconfig(ouraddr):
configinfo = ConfigFile()
for j in ("cmainit", "cmaaddr", "cmadisc", "cmafail"):
configinfo[j] = ouraddr
configinfo["outsig"] = pySignFrame(1)
return configinfo.complete_config()
def dispatch(self, origaddr, frameset):
json = None
addrstr = repr(origaddr)
fstype = frameset.get_framesettype()
localtime = None
listenaddr = None
keyid = None
pubkey = None
keysize = None
#print >> sys.stderr, ("DispatchSTARTUP: received [%s] FrameSet from [%s]"
#% (FrameSetTypes.get(fstype)[0], addrstr))
if CMAdb.debug:
CMAdb.log.debug("DispatchSTARTUP: received [%s] FrameSet from [%s]"
% (FrameSetTypes.get(fstype)[0], addrstr))
if not self.io.connactive(origaddr):
self.io.closeconn(DEFAULT_FSP_QID, origaddr)
CMAdb.transaction.post_transaction_packets.append(FrameSetTypes.ACKSTARTUP)
for frame in frameset.iter():
frametype = frame.frametype()
if frametype == FrameTypes.WALLCLOCK:
localtime = str(frame.getint())
elif frametype == FrameTypes.IPPORT:
listenaddr = frame.getnetaddr()
elif frametype == FrameTypes.HOSTNAME:
sysname = frame.getstr()
if sysname == CMAdb.nodename:
if origaddr.islocal():
CMAdb.log.info("Received STARTUP from local system (%s)" % addrstr)
else:
addresses = ['127.0.0.1', '::ffff:127.0.0.1', '::1' ]
for address in addresses:
localhost = pyNetAddr(address)
self.io.addalias(localhost, origaddr)
CMAdb.log.info("Aliasing %s to %s" % (localhost, origaddr))
elif frametype == FrameTypes.JSDISCOVER:
json = frame.getstr()
#print >> sys.stderr, 'GOT JSDISCOVER JSON: [%s] (strlen:%s,framelen:%s)' \
#% (json, len(json), frame.framelen())
elif frametype == FrameTypes.KEYID:
keyid = frame.getstr()
elif frametype == FrameTypes.PUBKEYCURVE25519:
pubkey = frame.framevalue()
keysize = frame.framelen()
joininfo = pyConfigContext(init=json)
origaddr, isNAT = self.validate_source_ip(sysname, origaddr, joininfo, listenaddr)
CMAdb.log.info('Drone %s registered from address %s (%s) port %s, key_id %s'
% (sysname, origaddr, addrstr, origaddr.port(), keyid))
drone = self.droneinfo.add(sysname, 'STARTUP packet', port=origaddr.port()
, primary_ip_addr=str(origaddr))
drone.listenaddr = str(listenaddr) # Seems good to hang onto this...
drone.isNAT = isNAT # ditto...
# Did they give us the crypto info we need?
if keyid is None or pubkey is None:
if CMAdb.debug:
CMAdb.log.debug('Drone %s registered with keyid %s and pubkey provided: %s'
% (self, keyid, pubkey is not None))
else:
if drone.key_id == '':
if not keyid.startswith(sysname + "@@"):
CMAdb.log.warning("Drone %s wants to register with key_id %s -- permitted."
, sysname, keyid)
if not cryptcurve25519_save_public_key(keyid, pubkey, keysize):
raise ValueError("Drone %s public key (key_id %s, %d bytes) is invalid."
% (sysname, keyid, keysize))
elif drone.key_id != keyid:
raise ValueError("Drone %s tried to register with key_id %s instead of %s."
% (sysname, keyid, drone.key_id))
drone.set_crypto_identity(keyid=keyid)
pyCryptFrame.dest_set_key_id(origaddr, keyid)
#
# THIS IS HERE BECAUSE OF A PROTOCOL BUG...
# @FIXME Protocol bug when starting up a connection if our first (this) packet gets lost,
# then the protocol doesn't retransmit it.
# More specifically, it seems to clear it out of the queue.
# This might be CMA bug or a protocol bug. It's not clear...
# The packet goes into the queue, but if that packet is lost in transmission, then when
# we come back around here, it's not in the queue any more, even though it
# definitely wasn't ACKed.
# Once this is fixed, this "add_packet" call needs to go *after* the 'if' statement below.
#
CMAdb.transaction.add_packet(origaddr, FrameSetTypes.SETCONFIG, (str(self.config), )
, FrameTypes.CONFIGJSON)
if (localtime is not None):
if (drone.lastjoin == localtime):
CMAdb.log.warning('Drone %s [%s] sent duplicate STARTUP' % (sysname, origaddr))
if CMAdb.debug:
self.io.log_conn(origaddr)
return
drone.lastjoin = localtime
#print >> sys.stderr, 'DRONE from find: ', drone, type(drone), drone.port
drone.startaddr = str(origaddr)
if json is not None:
drone.logjson(origaddr, json)
if CMAdb.debug:
CMAdb.log.debug('Joining TheOneRing: %s / %s / %s' % (drone, type(drone), drone.port))
CMAdb.cdb.TheOneRing.join(drone)
if CMAdb.debug:
CMAdb.log.debug('Requesting Discovery from %s' % str(drone))
discovery_params = []
for agent in self.config['initial_discovery']:
params = ConfigFile.agent_params(self.config, 'discovery', agent, sysname)
params['agent'] = agent
params['instance'] = '_init_%s' % agent
discovery_params.append(params)
# Discover the permissions of all the lists of files we're configured to ask about
# Note that there are several lists to keep the amount of data in any one list
# down to a somewhat more reasonable level. 'fileattrs' output is really verbose
for pathlist_name in self.config['perm_discovery_lists']:
paths = self.config[pathlist_name]
params = ConfigFile.agent_params(self.config, 'discovery', 'fileattrs', sysname)
params['agent'] = 'fileattrs'
params['instance'] = pathlist_name
params['parameters'] = {'ASSIM_filelist': paths}
discovery_params.append(params)
if CMAdb.debug:
CMAdb.log.debug('Discovery details: %s' % str(discovery_params))
drone.request_discovery(discovery_params)
AssimEvent(drone, AssimEvent.OBJUP)
def _add_service_monitoring(self, drone, monitoredservice, moninfo):
'''
We start the monitoring of 'monitoredservice' using the information
in 'moninfo' - which came from MonitoringRule.constructaction()
and is based on discovery and general rules for that monitoring action
Moninfo includes the following kinds of metadata:
- identification parameters - class, provider, type
- environment variables
- command line arguments
'''
monitorclass = moninfo['monitorclass']
monitortype = moninfo['monitortype']
monitorprovider = moninfo.get('provider', None)
if monitorprovider is not None:
classtype = "%s::%s:%s" % (monitorclass, moninfo['provider'], monitortype)
else:
classtype = "%s::%s" % (monitorclass, monitortype)
# Compute interval and timeout - based on global 'config'
agent_params = ConfigFile.agent_params(self.config, 'monitoring',
classtype, drone.designation)
# This produces the following metadata:
# - class-independent parameters: repeat, timeout, etc
# - environment variables
# - command line arguments
# These are merged together with the moninfo parameters in the following way
# - Class-independent variables are taken from 'params' if conflicting
# - Environment variables are taken from 'params' if there's a conflict
# - command line arguments from params come first, followed by the
# any that come from 'moninfo'
parameters = agent_params['parameters']
paraminterval = parameters['repeat']
paramtimeout = parameters['timeout']
paramwarntime = parameters['warn']
paramargv = parameters.get('argv', [])
paramenv = parameters.get('env', {})
monargv = moninfo.get('argv', [])
monenv = moninfo.get('arglist', {})
argv = [] # pyConfigContext doesn't handle arrays well...
if paramargv is not None:
argv.extend(paramargv)
if monargv is not None:
argv.extend(monargv)
environ = {}
for envset in (monenv, paramenv):
if envset is not None:
for env in envset:
environ[env] = envset[env]
# Make up a monitor name that should be unique to us -- but reproducible
# We create the monitor name from the host name, the monitor class,
# monitoring type and a hash of the arguments to the monitoring agent
# Note that this is different from the hashed service/entity name, since we could
# have multiple ways of monitoring the same entity
d = hashlib.md5()
# pylint mistakenly thinks md5 objects don't have update member
# pylint: disable=E1101
d.update('%s:%s:%s:%s'
% (drone.designation, monitorclass, monitortype, monitorprovider))
if environ is not None:
names = environ.keys()
names.sort()
for name in names:
# pylint thinks md5 objects don't have update member
# pylint: disable=E1101
d.update('"%s": "%s"' % (name, environ[name]))
for arg in argv:
d.update('"%s"' % str(arg))
monitorname = ('%s:%s:%s::%s'
% (drone.designation, monitorclass, monitortype, d.hexdigest()))
monnode = self.store.load_or_create(MonitorAction, domain=drone.domain
, monitorname=monitorname, monitorclass=monitorclass
, monitortype=monitortype, interval=paraminterval, timeout=paramtimeout
, warntime=paramwarntime
, provider=monitorprovider
, arglist = environ if environ else None
, argv = argv if argv else None) # Neo4j restriction...
if monitorclass == 'nagios':
monnode.nagiospath = self.config['monitoring']['nagiospath']
if not Store.is_abstract(monnode):
print >> sys.stderr, ('Previously monitored %s on %s'
% (monitortype, drone.designation))
monnode.activate(monitoredservice, drone)
def _add_service_monitoring(self, drone, monitoredservice, moninfo):
'''
We start the monitoring of 'monitoredservice' using the information
in 'moninfo' - which came from MonitoringRule.constructaction()
and is based on discovery and general rules for that monitoring action
Moninfo includes the following kinds of metadata:
- identification parameters - class, provider, type
- environment variables
- command line arguments
'''
monitorclass = moninfo['monitorclass']
monitortype = moninfo['monitortype']
monitorprovider = moninfo.get('provider', None)
if monitorprovider is not None:
classtype = "%s::%s:%s" % (monitorclass, moninfo['provider'],
monitortype)
else:
classtype = "%s::%s" % (monitorclass, monitortype)
# Compute interval and timeout - based on global 'config'
agent_params = ConfigFile.agent_params(self.config, 'monitoring',
classtype, drone.designation)
# This produces the following metadata:
# - class-independent parameters: repeat, timeout, etc
# - environment variables
# - command line arguments
# These are merged together with the moninfo parameters in the following way
# - Class-independent variables are taken from 'params' if conflicting
# - Environment variables are taken from 'params' if there's a conflict
# - command line arguments from params come first, followed by the
# any that come from 'moninfo'
parameters = agent_params['parameters']
paraminterval = parameters['repeat']
paramtimeout = parameters['timeout']
paramwarntime = parameters['warn']
paramargv = parameters.get('argv', [])
paramenv = parameters.get('env', {})
monargv = moninfo.get('argv', [])
monenv = moninfo.get('arglist', {})
argv = [] # pyConfigContext doesn't handle arrays well...
if paramargv is not None:
argv.extend(paramargv)
if monargv is not None:
argv.extend(monargv)
environ = {}
for envset in (monenv, paramenv):
if envset is not None:
for env in envset:
environ[env] = envset[env]
# Make up a monitor name that should be unique to us -- but reproducible
# We create the monitor name from the host name, the monitor class,
# monitoring type and a hash of the arguments to the monitoring agent
# Note that this is different from the hashed service/entity name, since we could
# have multiple ways of monitoring the same entity
d = hashlib.md5()
# pylint mistakenly thinks md5 objects don't have update member
# pylint: disable=E1101
d.update(
'%s:%s:%s:%s' %
(drone.designation, monitorclass, monitortype, monitorprovider))
if environ is not None:
names = environ.keys()
names.sort()
for name in names:
# pylint thinks md5 objects don't have update member
# pylint: disable=E1101
d.update('"%s": "%s"' % (name, environ[name]))
for arg in argv:
d.update('"%s"' % str(arg))
monitorname = (
'%s:%s:%s::%s' %
(drone.designation, monitorclass, monitortype, d.hexdigest()))
monnode = self.store.load_or_create(
MonitorAction,
domain=drone.domain,
monitorname=monitorname,
monitorclass=monitorclass,
monitortype=monitortype,
interval=paraminterval,
timeout=paramtimeout,
warntime=paramwarntime,
provider=monitorprovider,
arglist=environ if environ else None,
argv=argv if argv else None) # Neo4j restriction...
if monitorclass == 'nagios':
monnode.nagiospath = self.config['monitoring']['nagiospath']
if not Store.is_abstract(monnode):
print >> sys.stderr, ('Previously monitored %s on %s' %
(monitortype, drone.designation))
monnode.activate(monitoredservice, drone)
def geninitconfig(ouraddr):
configinfo = ConfigFile()
for j in ('cmainit', 'cmaaddr', 'cmadisc', 'cmafail'):
configinfo[j] = ouraddr
configinfo['outsig'] = pySignFrame(1)
return configinfo.complete_config()
