class Loop: def __init__(self, action): self.proxies = [] # list of proxies needed by some stations self.stations = [] # initial list of stations self.stations_retry = [] # checks failed, try again self.stations_complete = [] # checks succeeded, done self.stations_expired = [] # tried max number of times allowed self.stations_partial = [] # stations that are missing information self.output_directory = "" self.thread_count = 0 self.max_threads = 10 self.thread_ttl = 7200 # Should not take more than 2 hours self.threads = [] self.action = action self.types = None self.selection = None self.station_file = "" self.station_file_encrypted = False self.poll_interval = 0.10 # seconds self.poller = Interval(self.poll_interval, self.poll, False) self.lock_poll = thread.allocate_lock() self.done = False self.continuity_only = False self.versions_only = False self.version_logger = Logger(prefix='deviations_') self.version_queue = Queue.Queue() self.version_files = [] def is_done(self): return self.done def set_selection(self, list): self.selection = list def set_continuity_only(self, only=True): self.continuity_only = only def set_versions_only(self, only=True): self.versions_only = only def set_types(self, list): self.types = list def set_file(self, name, encrypted=False): self.station_file = name self.station_file_encrypted = encrypted def set_dir(self, directory): self.station_dir = directory # create the archive directory if it does not exist def init_dir(self): self.output_directory = None self.version_directory = None if not self.output_directory: self.output_directory = "%(HOME)s/stations" % os.environ if not os.path.exists(self.output_directory): try: os.makedirs(self.output_directory) permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH if os.stat(self.output_directory).st_mode != permissions: os.chmod(self.output_directory, permissions) except: raise Exception, "CheckLoop::init_dir() could not create storage directory: %s" % self.output_directory self.output_directory += "/gsn" if not os.path.exists(self.output_directory): try: os.makedirs(self.output_directory) permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH if os.stat(self.output_directory).st_mode != permissions: os.chmod(self.output_directory, permissions) except: raise Exception, "CheckLoop::init_dir() could not create storage directory: %s" % self.output_directory self.version_directory = self.output_directory + "/versions" if not os.path.exists(self.version_directory): try: os.makedirs(self.version_directory) permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH if os.stat(self.version_directory).st_mode != permissions: os.chmod(self.version_directory, permissions) except: raise Exception, "CheckLoop::init_dir() could not create version directory: %s" % self.version_directory self.version_logger.set_log_path(self.version_directory) # ===== Long Running Threads ===================================== def start_threads(self): self.version_thread = threading.Thread(target=self._version_log_thread) self.version_thread.start() def stop_threads(self): self.version_queue.put('HALT') self.version_thread.join() def _version_log_thread(self): run = True while run: message = self.version_queue.get() if message == 'HALT': run = False else: parts = message.split(':', 1) if (len(parts) == 2) and (4 < len(parts[0]) < 9): self.version_logger.set_log_note(parts[0]) self.version_logger.log(parts[1]) def read_version_file(self): version_file = './software_versions' if os.path.exists(version_file) and os.path.isfile(version_file): fh = open(version_file, 'r') for line in fh.readlines(): parts = line.split('#', 1)[0].split() if len(parts) > 1: hash = parts[0] file = parts[1] self.version_files.append((file, hash)) def prep_station(self, station, info): if info.has_key('station'): station.set_name(info['station']) if info.has_key('address'): station.set_address(info['address']) if info.has_key('port'): station.set_port(info['port']) if info.has_key('username'): station.set_username(info['username']) if info.has_key('password'): station.set_password(info['password']) if info.has_key('netserv'): list = info['netserv'].split(',') for item in list: station.add_netserv_log(item) if info.has_key('server'): list = info['server'].split(',') for item in list: station.add_server_log(item) def start(self): self.init_dir() self.parse_configuration() self.read_version_file() self.start_threads() self.poller.start() def record(self, station): if not station: return date = time.gmtime() # by default, check output is stored in path as follows: # $HOME/stations/gsn/<station>/<year>/<j_day>/<HHMMSS>.chk dir = self.output_directory + '/' + station.name + time.strftime("/%Y/%j", date) file = time.strftime("%H%M%S.chk", date) # create the directories if they do not exist ydir = dir.rsplit('/',1)[0] if not os.path.exists(ydir): try: os.makedirs(ydir) permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH if os.stat(ydir).st_mode != permissions: os.chmod(ydir, permissions) except: station._log("could not create directory %s" % ydir) return if not os.path.exists(dir): try: os.makedirs(dir) permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH if os.stat(dir).st_mode != permissions: os.chmod(dir, permissions) except: station._log("could not create directory %s" % dir) return # if the target directory path exists but is not a directory, complain elif not os.path.isdir(dir): station._log("%s exists, and is not a directory, please resolve." % dir) return # write results into the summary file try: summary_file = dir + '/' + file fd = open(summary_file, "a") fd.write( station.summary ) fd.write( station.output ) fd.write( station.log_messages ) fd.close() permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH if os.stat(summary_file).st_mode != permissions: os.chmod(summary_file, permissions) except Exception, e: station._log("CheckLoop::record() failed to record data to file. Exception: %s" % str(e))
class Manager(threading.Thread): def __init__(self, action, stop_queue, max_threads=10): threading.Thread.__init__(self, name="Manager") self.action = action self.stop_queue = stop_queue self.max_threads = max_threads self.stations = {} # dictionary of station info self.proxies = {} # dictionary of proxy info self.groups = {} # dictionary of group info self.groups['NONE'] = { 'name' : 'NONE', 'type' : 'Group', 'threads' : '10', } self.output_directory = "." self.types = None self.selected_stations = None self.excluded_stations = None self.selected_networks = None self.excluded_networks = None self.group_selection = None self.commands_only = False self.continuity_only = False self.list_only = False self.versions_only = False self.commands = [] self.station_file = "" self.station_file_encrypted = False self.version_logger = Logger(prefix='deviations_') self.version_queue = Queue.Queue() self.version_thread = None self.version_files = {} self.logger = Logger(prefix='StationsManager_') self.logger.set_log_to_screen(True) self.logger.set_log_to_file(False) self.logger.set_log_note('Manager') self.loops = [] self.queue = Queue.Queue() # ===== Mutators ===== def set_file(self, name, encrypted=False): self.station_file = name self.station_file_encrypted = encrypted def set_commands_only(self, only=True): self.commands_only = only def set_continuity_only(self, only=True): self.continuity_only = only def set_list_only(self, only=True): self.list_only = only def set_versions_only(self, only=True): self.versions_only = only def clear_commands(self): self.commands = [] def set_commands(self, commands): self.commands = commands def add_command(self, command): self.commands.append(command) def add_commands(self, commands): self.commands.extend(commands) def set_selected_stations(self, stations): self.selected_stations = stations def set_excluded_stations(self, stations): self.excluded_stations = stations def set_selected_networks(self, networks): self.selected_networks = networks def set_excluded_networks(self, networks): self.excluded_networks = networks def set_exclusion(self, exclusion): self.exclusion = exclusion def set_types(self, types): self.types = types def set_group_selection(self, groups): self.group_selection = groups # ===== Entry Point ===== def run(self): try: self.init_dir() self.parse_configuration() if self.list_only: # List all stations max_name_len = 0 groups = {'NONE' : {}} for group in self.groups.keys(): groups[group] = {} for station in self.stations.keys(): group = 'NONE' info = self.stations[station] if info.has_key('group'): group = info['group'] groups[group][info['name']] = info max_name_len = max(max_name_len, len(station)) group_names = self.groups.keys() if self.group_selection is not None: group_names = self.group_selection for group in sorted(group_names): if group == 'NONE': print "No Group (%d stations):" % len(groups[group].keys()) elif not groups.has_key(group): print "Group '%s' does not exist" % group continue else: print "Group '%s' (%d stations):" % (group, len(groups[group].keys())) for station in sorted(groups[group].keys()): info = groups[group][station] name = station.ljust(max_name_len) print " %s [%s]" % (name, info['type']) raise ExceptionList() self.read_version_file() #max_len = max(map(len, self.version_files.keys())) #print "Version Files:" #for key in sorted(self.version_files.keys()): # print "%s: %s" % (key.rjust(max_len),self.version_files[key][::-1]) self.start_threads() group_names = self.groups.keys() if self.group_selection is not None: group_names = self.group_selection station_groups = {} for station_name in self.stations.keys(): group = 'NONE' if self.stations[station_name].has_key('group'): group = self.stations[station_name]['group'] add_station = True if (group_names != None) and (group not in group_names): add_station = False if add_station: if not station_groups.has_key(group): station_groups[group] = [] station_groups[group].append(station_name) self.logger.log("Groups: %s" % str(station_groups.keys())) for group in sorted(station_groups.keys()): try: max_threads = self.max_threads if self.groups[group].has_key('threads'): max_threads = int(self.groups[group]['threads']) loop = ThreadLoop(self, group, self.action, sorted(station_groups[group]), max_threads, self.version_queue) loop.set_version_files(self.version_files) loop.set_versions_only(self.versions_only) loop.set_commands_only(self.commands_only) loop.set_commands(self.commands) loop.set_continuity_only(self.continuity_only) loop.set_list_only(self.list_only) loop.set_output_directory(self.output_directory) loop.logger.set_log_note("Loop:%s" % group) self.logger.log("Starting new ThreadLoop for group '%s'" % group) loop.start() self.loops.append(loop) except Exception, e: self.logger.log("Exception while creating Loop:%s: %s" % (group, str(e))) (ex_f, ex_s, trace) = sys.exc_info() traceback.print_tb(trace) while len(self.loops) > 0: try: message,loop = self.queue.get() name = "Anonymous" if loop is not None: name = loop.name self.logger.log("Received message from queue (Loop %s says '%s')" % (name, message)) if (message == 'DONE') and (loop is not None): loop.join() self.logger.log("LoopThread joined: %s" % name) for l in self.loops: if (not l.fresh) and (not l.running): self.loops.remove(l) self.logger.log("Removing Loop:%s. %d loop(s) remaining." % (str(l.group), len(self.loops))) for l in self.loops: running_threads = [] for t in l.threads: running_threads.append(t.name) self.logger.log(" Loop:%s has %d station thread(s) running [%s]." % (str(l.group), len(l.threads), ", ".join(running_threads))) except KeyboardInterrupt, e: self.logger.log("Thread Summary [%d]: %s" % (threading.activeCount(), str(threading.enumerate())))