def __init__(self, action):
self.proxies = [] # list of proxies needed by some stations
self.stations = [] # initial list of stations
self.stations_retry = [] # checks failed, try again
self.stations_complete = [] # checks succeeded, done
self.stations_expired = [] # tried max number of times allowed
self.stations_partial = [] # stations that are missing information
self.output_directory = ""
self.thread_count = 0
self.max_threads = 10
self.thread_ttl = 7200 # Should not take more than 2 hours
self.threads = []
self.action = action
self.types = None
self.selection = None
self.station_file = ""
self.station_file_encrypted = False
self.poll_interval = 0.10 # seconds
self.poller = Interval(self.poll_interval, self.poll, False)
self.lock_poll = thread.allocate_lock()
self.done = False
self.continuity_only = False
self.versions_only = False
self.version_logger = Logger(prefix='deviations_')
self.version_queue = Queue.Queue()
self.version_files = []
def __init__(self, action, stop_queue, max_threads=10):
threading.Thread.__init__(self, name="Manager")
self.action = action
self.stop_queue = stop_queue
self.max_threads = max_threads
self.stations = {} # dictionary of station info
self.proxies = {} # dictionary of proxy info
self.groups = {} # dictionary of group info
self.groups['NONE'] = {
'name' : 'NONE',
'type' : 'Group',
'threads' : '10',
}
self.output_directory = "."
self.types = None
self.selected_stations = None
self.excluded_stations = None
self.selected_networks = None
self.excluded_networks = None
self.group_selection = None
self.commands_only = False
self.continuity_only = False
self.list_only = False
self.versions_only = False
self.commands = []
self.station_file = ""
self.station_file_encrypted = False
self.version_logger = Logger(prefix='deviations_')
self.version_queue = Queue.Queue()
self.version_thread = None
self.version_files = {}
self.logger = Logger(prefix='StationsManager_')
self.logger.set_log_to_screen(True)
self.logger.set_log_to_file(False)
self.logger.set_log_note('Manager')
self.loops = []
self.queue = Queue.Queue()
def __init__(self, manager, group, action, station_names, max_threads, version_queue):
threading.Thread.__init__(self, name=group)
self.manager = manager
self.group = group
self.action = action
self.commands = []
self.commands_only = False
self.continuity_only = False
self.list_only = False
self.versions_only = False
self.version_queue = version_queue
self.version_files = {}
self.max_threads = max_threads
self.thread_ttl = 7200 # Should not take more than 2 hours
self.threads = []
self.output_directory = '.'
# Group based station tracking dictionaries
self.stations_fresh = station_names
self.stations_retry = [] # checks failed, try again
self.stations_complete = [] # checks succeeded, done
self.stations_expired = [] # tried max number of times allowed
self.stations_partial = [] # stations that are missing information
self.fresh = True
self.done = False
self.running = False
self.queue = Queue.Queue()
self.logger = Logger()
self.logger.set_log_to_screen(True)
self.logger.set_log_to_file(False)
class Loop:
def __init__(self, action):
self.proxies = [] # list of proxies needed by some stations
self.stations = [] # initial list of stations
self.stations_retry = [] # checks failed, try again
self.stations_complete = [] # checks succeeded, done
self.stations_expired = [] # tried max number of times allowed
self.stations_partial = [] # stations that are missing information
self.output_directory = ""
self.thread_count = 0
self.max_threads = 10
self.thread_ttl = 7200 # Should not take more than 2 hours
self.threads = []
self.action = action
self.types = None
self.selection = None
self.station_file = ""
self.station_file_encrypted = False
self.poll_interval = 0.10 # seconds
self.poller = Interval(self.poll_interval, self.poll, False)
self.lock_poll = thread.allocate_lock()
self.done = False
self.continuity_only = False
self.versions_only = False
self.version_logger = Logger(prefix='deviations_')
self.version_queue = Queue.Queue()
self.version_files = []
def is_done(self):
return self.done
def set_selection(self, list):
self.selection = list
def set_continuity_only(self, only=True):
self.continuity_only = only
def set_versions_only(self, only=True):
self.versions_only = only
def set_types(self, list):
self.types = list
def set_file(self, name, encrypted=False):
self.station_file = name
self.station_file_encrypted = encrypted
def set_dir(self, directory):
self.station_dir = directory
# create the archive directory if it does not exist
def init_dir(self):
self.output_directory = None
self.version_directory = None
if not self.output_directory:
self.output_directory = "%(HOME)s/stations" % os.environ
if not os.path.exists(self.output_directory):
try:
os.makedirs(self.output_directory)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(self.output_directory).st_mode != permissions:
os.chmod(self.output_directory, permissions)
except:
raise Exception, "CheckLoop::init_dir() could not create storage directory: %s" % self.output_directory
self.output_directory += "/gsn"
if not os.path.exists(self.output_directory):
try:
os.makedirs(self.output_directory)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(self.output_directory).st_mode != permissions:
os.chmod(self.output_directory, permissions)
except:
raise Exception, "CheckLoop::init_dir() could not create storage directory: %s" % self.output_directory
self.version_directory = self.output_directory + "/versions"
if not os.path.exists(self.version_directory):
try:
os.makedirs(self.version_directory)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(self.version_directory).st_mode != permissions:
os.chmod(self.version_directory, permissions)
except:
raise Exception, "CheckLoop::init_dir() could not create version directory: %s" % self.version_directory
self.version_logger.set_log_path(self.version_directory)
# ===== Long Running Threads =====================================
def start_threads(self):
self.version_thread = threading.Thread(target=self._version_log_thread)
self.version_thread.start()
def stop_threads(self):
self.version_queue.put('HALT')
self.version_thread.join()
def _version_log_thread(self):
run = True
while run:
message = self.version_queue.get()
if message == 'HALT':
run = False
else:
parts = message.split(':', 1)
if (len(parts) == 2) and (4 < len(parts[0]) < 9):
self.version_logger.set_log_note(parts[0])
self.version_logger.log(parts[1])
def read_version_file(self):
version_file = './software_versions'
if os.path.exists(version_file) and os.path.isfile(version_file):
fh = open(version_file, 'r')
for line in fh.readlines():
parts = line.split('#', 1)[0].split()
if len(parts) > 1:
hash = parts[0]
file = parts[1]
self.version_files.append((file, hash))
def prep_station(self, station, info):
if info.has_key('station'):
station.set_name(info['station'])
if info.has_key('address'):
station.set_address(info['address'])
if info.has_key('port'):
station.set_port(info['port'])
if info.has_key('username'):
station.set_username(info['username'])
if info.has_key('password'):
station.set_password(info['password'])
if info.has_key('netserv'):
list = info['netserv'].split(',')
for item in list:
station.add_netserv_log(item)
if info.has_key('server'):
list = info['server'].split(',')
for item in list:
station.add_server_log(item)
def start(self):
self.init_dir()
self.parse_configuration()
self.read_version_file()
self.start_threads()
self.poller.start()
def record(self, station):
if not station:
return
date = time.gmtime()
# by default, check output is stored in path as follows:
# $HOME/stations/gsn/<station>/<year>/<j_day>/<HHMMSS>.chk
dir = self.output_directory + '/' + station.name + time.strftime("/%Y/%j", date)
file = time.strftime("%H%M%S.chk", date)
# create the directories if they do not exist
ydir = dir.rsplit('/',1)[0]
if not os.path.exists(ydir):
try:
os.makedirs(ydir)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(ydir).st_mode != permissions:
os.chmod(ydir, permissions)
except:
station._log("could not create directory %s" % ydir)
return
if not os.path.exists(dir):
try:
os.makedirs(dir)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(dir).st_mode != permissions:
os.chmod(dir, permissions)
except:
station._log("could not create directory %s" % dir)
return
# if the target directory path exists but is not a directory, complain
elif not os.path.isdir(dir):
station._log("%s exists, and is not a directory, please resolve." % dir)
return
# write results into the summary file
try:
summary_file = dir + '/' + file
fd = open(summary_file, "a")
fd.write( station.summary )
fd.write( station.output )
fd.write( station.log_messages )
fd.close()
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH
if os.stat(summary_file).st_mode != permissions:
os.chmod(summary_file, permissions)
except Exception, e:
station._log("CheckLoop::record() failed to record data to file. Exception: %s" % str(e))
class Manager(threading.Thread):
def __init__(self, action, stop_queue, max_threads=10):
threading.Thread.__init__(self, name="Manager")
self.action = action
self.stop_queue = stop_queue
self.max_threads = max_threads
self.stations = {} # dictionary of station info
self.proxies = {} # dictionary of proxy info
self.groups = {} # dictionary of group info
self.groups['NONE'] = {
'name' : 'NONE',
'type' : 'Group',
'threads' : '10',
}
self.output_directory = "."
self.types = None
self.selected_stations = None
self.excluded_stations = None
self.selected_networks = None
self.excluded_networks = None
self.group_selection = None
self.commands_only = False
self.continuity_only = False
self.list_only = False
self.versions_only = False
self.commands = []
self.station_file = ""
self.station_file_encrypted = False
self.version_logger = Logger(prefix='deviations_')
self.version_queue = Queue.Queue()
self.version_thread = None
self.version_files = {}
self.logger = Logger(prefix='StationsManager_')
self.logger.set_log_to_screen(True)
self.logger.set_log_to_file(False)
self.logger.set_log_note('Manager')
self.loops = []
self.queue = Queue.Queue()
# ===== Mutators =====
def set_file(self, name, encrypted=False):
self.station_file = name
self.station_file_encrypted = encrypted
def set_commands_only(self, only=True):
self.commands_only = only
def set_continuity_only(self, only=True):
self.continuity_only = only
def set_list_only(self, only=True):
self.list_only = only
def set_versions_only(self, only=True):
self.versions_only = only
def clear_commands(self):
self.commands = []
def set_commands(self, commands):
self.commands = commands
def add_command(self, command):
self.commands.append(command)
def add_commands(self, commands):
self.commands.extend(commands)
def set_selected_stations(self, stations):
self.selected_stations = stations
def set_excluded_stations(self, stations):
self.excluded_stations = stations
def set_selected_networks(self, networks):
self.selected_networks = networks
def set_excluded_networks(self, networks):
self.excluded_networks = networks
def set_exclusion(self, exclusion):
self.exclusion = exclusion
def set_types(self, types):
self.types = types
def set_group_selection(self, groups):
self.group_selection = groups
# ===== Entry Point =====
def run(self):
try:
self.init_dir()
self.parse_configuration()
if self.list_only:
# List all stations
max_name_len = 0
groups = {'NONE' : {}}
for group in self.groups.keys():
groups[group] = {}
for station in self.stations.keys():
group = 'NONE'
info = self.stations[station]
if info.has_key('group'):
group = info['group']
groups[group][info['name']] = info
max_name_len = max(max_name_len, len(station))
group_names = self.groups.keys()
if self.group_selection is not None:
group_names = self.group_selection
for group in sorted(group_names):
if group == 'NONE':
print "No Group (%d stations):" % len(groups[group].keys())
elif not groups.has_key(group):
print "Group '%s' does not exist" % group
continue
else:
print "Group '%s' (%d stations):" % (group, len(groups[group].keys()))
for station in sorted(groups[group].keys()):
info = groups[group][station]
name = station.ljust(max_name_len)
print " %s [%s]" % (name, info['type'])
raise ExceptionList()
self.read_version_file()
#max_len = max(map(len, self.version_files.keys()))
#print "Version Files:"
#for key in sorted(self.version_files.keys()):
# print "%s: %s" % (key.rjust(max_len),self.version_files[key][::-1])
self.start_threads()
group_names = self.groups.keys()
if self.group_selection is not None:
group_names = self.group_selection
station_groups = {}
for station_name in self.stations.keys():
group = 'NONE'
if self.stations[station_name].has_key('group'):
group = self.stations[station_name]['group']
add_station = True
if (group_names != None) and (group not in group_names):
add_station = False
if add_station:
if not station_groups.has_key(group):
station_groups[group] = []
station_groups[group].append(station_name)
self.logger.log("Groups: %s" % str(station_groups.keys()))
for group in sorted(station_groups.keys()):
try:
max_threads = self.max_threads
if self.groups[group].has_key('threads'):
max_threads = int(self.groups[group]['threads'])
loop = ThreadLoop(self, group, self.action, sorted(station_groups[group]), max_threads, self.version_queue)
loop.set_version_files(self.version_files)
loop.set_versions_only(self.versions_only)
loop.set_commands_only(self.commands_only)
loop.set_commands(self.commands)
loop.set_continuity_only(self.continuity_only)
loop.set_list_only(self.list_only)
loop.set_output_directory(self.output_directory)
loop.logger.set_log_note("Loop:%s" % group)
self.logger.log("Starting new ThreadLoop for group '%s'" % group)
loop.start()
self.loops.append(loop)
except Exception, e:
self.logger.log("Exception while creating Loop:%s: %s" % (group, str(e)))
(ex_f, ex_s, trace) = sys.exc_info()
traceback.print_tb(trace)
while len(self.loops) > 0:
try:
message,loop = self.queue.get()
name = "Anonymous"
if loop is not None:
name = loop.name
self.logger.log("Received message from queue (Loop %s says '%s')" % (name, message))
if (message == 'DONE') and (loop is not None):
loop.join()
self.logger.log("LoopThread joined: %s" % name)
for l in self.loops:
if (not l.fresh) and (not l.running):
self.loops.remove(l)
self.logger.log("Removing Loop:%s. %d loop(s) remaining." % (str(l.group), len(self.loops)))
for l in self.loops:
running_threads = []
for t in l.threads:
running_threads.append(t.name)
self.logger.log(" Loop:%s has %d station thread(s) running [%s]." % (str(l.group), len(l.threads), ", ".join(running_threads)))
except KeyboardInterrupt, e:
self.logger.log("Thread Summary [%d]: %s" % (threading.activeCount(), str(threading.enumerate())))
class ThreadLoop(threading.Thread):
def __init__(self, manager, group, action, station_names, max_threads, version_queue):
threading.Thread.__init__(self, name=group)
self.manager = manager
self.group = group
self.action = action
self.commands = []
self.commands_only = False
self.continuity_only = False
self.list_only = False
self.versions_only = False
self.version_queue = version_queue
self.version_files = {}
self.max_threads = max_threads
self.thread_ttl = 7200 # Should not take more than 2 hours
self.threads = []
self.output_directory = '.'
# Group based station tracking dictionaries
self.stations_fresh = station_names
self.stations_retry = [] # checks failed, try again
self.stations_complete = [] # checks succeeded, done
self.stations_expired = [] # tried max number of times allowed
self.stations_partial = [] # stations that are missing information
self.fresh = True
self.done = False
self.running = False
self.queue = Queue.Queue()
self.logger = Logger()
self.logger.set_log_to_screen(True)
self.logger.set_log_to_file(False)
def set_output_directory(self, dir):
self.output_directory = dir
def set_commands(self, commands):
self.commands = commands
def set_commands_only(self, only=True):
self.commands_only = only
def set_continuity_only(self, only=True):
self.continuity_only = only
def set_list_only(self, only=True):
self.list_only = only
def set_version_files(self, files):
self.version_files = files
def set_versions_only(self, only=True):
self.versions_only = only
def is_done(self):
alive = False
try:
alive = self.is_alive()
except:
alive = self.isAlive()
return not alive
# ===== Preparation methods =====
def prep_station(self, station, info):
if info.has_key('type'):
station.type = info['type']
if info.has_key('name'):
station.set_name(info['name'])
if info.has_key('group'):
station.set_group(info['group'])
if info.has_key('address'):
station.set_address(info['address'])
if info.has_key('port'):
station.set_port(info['port'])
if info.has_key('username'):
station.set_username(info['username'])
if info.has_key('password'):
station.set_password(info['password'])
if info.has_key('prompt'):
station.prompt_shell = info['prompt']
if info.has_key('skip-continuity'):
station.skip_continuity_check(info['skip-continuity'].upper() == 'TRUE')
if info.has_key('netserv'):
list = info['netserv'].split(',')
for item in list:
station.add_netserv_log(item)
if info.has_key('server'):
list = info['server'].split(',')
for item in list:
station.add_server_log(item)
if info.has_key('sync-multiplier'):
station.sync_multiplier = int(info['sync-multiplier'])
station.info = info
def prep_proxy(self, proxy, info, station=None):
if info.has_key('type'):
proxy.type = info['type']
if info.has_key('name'):
proxy.set_name(info['name'])
if info.has_key('address'):
proxy.set_address(info['address'])
if info.has_key('port'):
proxy.set_port(info['port'])
if info.has_key('username'):
proxy.set_username(info['username'])
if info.has_key('password'):
proxy.set_password(info['password'])
if info.has_key('prompt'):
proxy.prompt_shell = info['prompt']
if info.has_key('local-address'):
proxy.local_address = info['local-address']
else:
proxy.local_address = "127.0.0.1"
if info.has_key('local-port'):
proxy.local_port = info['local-port']
if info.has_key('sync-multiplier'):
proxy.sync_multiplier = int(info['sync-multiplier'])
if station is not None:
proxy.station_address = station.address
proxy.station_port = station.port
proxy.group = station.group
proxy.info=info
# Recursively determines a station's proxy chain
def find_proxies(self, station, station_info, proxies=None):
if proxies is None:
proxies = {}
if station_info.has_key('proxy'):
if self.manager.proxies.has_key(station_info['proxy']):
station.proxy_info = self.manager.proxies[station_info['proxy']]
# Search for nested proxies
if proxies.has_key(station.proxy_info['name']):
raise Exception("Encountered a proxy loop, first repeated on proxy '%(proxy)s'" % station_info)
station.proxy = Proxy(station.proxy_info['name'])
self.prep_proxy(station.proxy, station.proxy_info, station)
# Look for any proxy upon which this proxy depends (nested proxy support)
self.find_proxies(station.proxy, station.proxy_info, proxies)
else:
raise Exception("Could not locate proxy '%(proxy)s' associated with station '%(name)s'" % station_info)
# Recursively starts a station's proxy chain
def start_proxies(self, station, dir, depth):
if station.proxy is not None:
# The depth logic can be a little confusing. We need this in order
# to track how many ~ characters are required in order to reach
# the SSH shell from this level. By passing a list, we can modify
# its contents before returning control to our parent, thereby
# giving increment control to the child. This ensures that the
# parent proxy has a higher depth value than its child proxy, and
# on down the chain. The parents is dependent on the child proxy
# to establish the next portion of the path before it can connect.
# We connect the child proxy before calling the parent proxy's start
# method which establishes all connections in the correct order.
proxy = station.proxy
proxy.log_file_name(dir + '/proxy.log')
proxy.log_to_file()
proxy.log_to_screen()
proxy.set_output_directory(self.output_directory + '/' + station.name)
proxy.logger.set_log_note("%s:%s" % (proxy.name, station.name))
self.start_proxies(proxy, dir, depth)
proxy.depth = depth[0]
station._log("Starting proxy '%s:%s' thread at depth %d..." % (proxy.name, station.name, depth[0]))
depth[0] = depth[0] + 1
proxy.start()
def summarize(self):
# build a summary
self.logger.log("All stations have been processed.")
def halt(self):
self.stations_fresh = []
for thread in self.threads:
self.logger.log("Halting thread '%s'..." % thread.name)
thread.halt(now=True)
thread.join()
def record(self, station):
if not station:
return
date = time.gmtime()
# by default, check output is stored in path as follows:
# $HOME/stations/gsn/<station>/<year>/<j_day>/<HHMMSS>.chk
dir = self.output_directory + '/' + station.name + time.strftime("/%Y/%j", date)
file = time.strftime("%H%M%S.chk", date)
# create the directories if they do not exist
ydir = dir.rsplit('/',1)[0]
if not os.path.exists(ydir):
try:
os.makedirs(ydir)
Permissions("EEEEEEEIE", 1).process([ydir])
except:
station._log("could not create directory %s" % ydir)
return
if not os.path.exists(dir):
try:
os.makedirs(dir)
Permissions("EEEEEEEIE", 1).process([dir])
except:
station._log("could not create directory %s" % dir)
return
# if the target directory path exists but is not a directory, complain
elif not os.path.isdir(dir):
station._log("%s exists, and is not a directory, please resolve." % dir)
return
# write results into the summary file
try:
summary_file = dir + '/' + file
fd = open(summary_file, "a")
fd.write( station.summary )
fd.write( station.output )
fd.write( station.log_messages )
fd.close()
Permissions("EEIEEIEII", 1).process([summary_file])
except Exception, e:
station._log("CheckLoop::record() failed to record data to file. Exception: %s" % str(e))