Ejemplo n.º 1
0
class Manager(threading.Thread):
    def __init__(self, action, stop_queue, max_threads=10):
        threading.Thread.__init__(self, name="Manager")

        self.action = action
        self.stop_queue = stop_queue
        self.max_threads = max_threads

        self.stations = {} # dictionary of station info
        self.proxies  = {} # dictionary of proxy info
        self.groups   = {} # dictionary of group info
        self.groups['NONE'] = {
            'name'    : 'NONE',
            'type'    : 'Group',
            'threads' : '10',
        }

        self.output_directory = "."
        self.types = None
        self.selected_stations = None
        self.excluded_stations = None
        self.selected_networks = None
        self.excluded_networks = None
        self.group_selection = None

        self.commands_only = False
        self.continuity_only = False
        self.list_only = False
        self.versions_only = False

        self.commands = []

        self.station_file = ""
        self.station_file_encrypted = False

        self.version_logger = Logger(prefix='deviations_')
        self.version_queue  = Queue.Queue()
        self.version_thread = None
        self.version_files  = {}

        self.logger = Logger(prefix='StationsManager_')
        self.logger.set_log_to_screen(True)
        self.logger.set_log_to_file(False)
        self.logger.set_log_note('Manager')

        self.loops = []
        self.queue = Queue.Queue()


# ===== Mutators =====
    def set_file(self, name, encrypted=False):
        self.station_file = name
        self.station_file_encrypted = encrypted

    def set_commands_only(self, only=True):
        self.commands_only = only

    def set_continuity_only(self, only=True):
        self.continuity_only = only

    def set_list_only(self, only=True):
        self.list_only = only

    def set_versions_only(self, only=True):
        self.versions_only = only

    def clear_commands(self):
        self.commands = []

    def set_commands(self, commands):
        self.commands = commands

    def add_command(self, command):
        self.commands.append(command)

    def add_commands(self, commands):
        self.commands.extend(commands)

    def set_selected_stations(self, stations):
        self.selected_stations = stations

    def set_excluded_stations(self, stations):
        self.excluded_stations = stations

    def set_selected_networks(self, networks):
        self.selected_networks = networks

    def set_excluded_networks(self, networks):
        self.excluded_networks = networks

    def set_exclusion(self, exclusion):
        self.exclusion = exclusion

    def set_types(self, types):
        self.types = types

    def set_group_selection(self, groups):
        self.group_selection = groups


# ===== Entry Point =====
    def run(self):
        try:
            self.init_dir()
            self.parse_configuration()
            if self.list_only:
                # List all stations
                max_name_len = 0
                groups = {'NONE' : {}}
                for group in self.groups.keys():
                    groups[group] = {}
                for station in self.stations.keys():
                    group = 'NONE'
                    info = self.stations[station]
                    if info.has_key('group'):
                        group = info['group']
                    groups[group][info['name']] = info
                    max_name_len = max(max_name_len, len(station))
                group_names = self.groups.keys()
                if self.group_selection is not None:
                    group_names = self.group_selection
                for group in sorted(group_names):
                    if group == 'NONE':
                        print "No Group (%d stations):" % len(groups[group].keys())
                    elif not groups.has_key(group):
                        print "Group '%s' does not exist" % group
                        continue
                    else:
                        print "Group '%s' (%d stations):" % (group, len(groups[group].keys()))
                    for station in sorted(groups[group].keys()):
                        info = groups[group][station]
                        name = station.ljust(max_name_len)
                        print "  %s [%s]" % (name, info['type'])
                raise ExceptionList()
            self.read_version_file()
            #max_len = max(map(len, self.version_files.keys()))
            #print "Version Files:"
            #for key in sorted(self.version_files.keys()):
            #    print "%s: %s" % (key.rjust(max_len),self.version_files[key][::-1])
            self.start_threads()

            group_names = self.groups.keys()
            if self.group_selection is not None:
                group_names = self.group_selection

            station_groups = {}

            for station_name in self.stations.keys():
                group = 'NONE'
                if self.stations[station_name].has_key('group'):
                    group = self.stations[station_name]['group']
                add_station = True
                if (group_names != None) and (group not in group_names):
                    add_station = False
                if add_station:
                    if not station_groups.has_key(group):
                        station_groups[group] = []
                    station_groups[group].append(station_name)
            
            self.logger.log("Groups: %s" % str(station_groups.keys()))
            for group in sorted(station_groups.keys()):
                try:
                    max_threads = self.max_threads
                    if self.groups[group].has_key('threads'):
                        max_threads = int(self.groups[group]['threads'])
                    loop = ThreadLoop(self, group, self.action, sorted(station_groups[group]), max_threads, self.version_queue)
                    loop.set_version_files(self.version_files)
                    loop.set_versions_only(self.versions_only)
                    loop.set_commands_only(self.commands_only)
                    loop.set_commands(self.commands)
                    loop.set_continuity_only(self.continuity_only)
                    loop.set_list_only(self.list_only)
                    loop.set_output_directory(self.output_directory)
                    loop.logger.set_log_note("Loop:%s" % group)
                    self.logger.log("Starting new ThreadLoop for group '%s'" % group)
                    loop.start()
                    self.loops.append(loop)
                except Exception, e:
                    self.logger.log("Exception while creating Loop:%s: %s" % (group, str(e)))
                    (ex_f, ex_s, trace) = sys.exc_info()
                    traceback.print_tb(trace)

            while len(self.loops) > 0:
                try:
                    message,loop = self.queue.get()
                    name = "Anonymous"
                    if loop is not None:
                        name = loop.name
                    self.logger.log("Received message from queue (Loop %s says '%s')" % (name, message))
                    if (message == 'DONE') and (loop is not None):
                        loop.join()
                        self.logger.log("LoopThread joined: %s" % name)

                    for l in self.loops:
                        if (not l.fresh) and (not l.running):
                            self.loops.remove(l)
                            self.logger.log("Removing Loop:%s. %d loop(s) remaining." % (str(l.group), len(self.loops)))

                    for l in self.loops:
                        running_threads = []
                        for t in l.threads:
                            running_threads.append(t.name)
                        self.logger.log("  Loop:%s has %d station thread(s) running [%s]." % (str(l.group), len(l.threads), ", ".join(running_threads)))
                except KeyboardInterrupt, e:
                    self.logger.log("Thread Summary [%d]: %s" % (threading.activeCount(), str(threading.enumerate())))
Ejemplo n.º 2
0
class ThreadLoop(threading.Thread):
    def __init__(self, manager, group, action, station_names, max_threads, version_queue):
        threading.Thread.__init__(self, name=group)

        self.manager = manager
        self.group = group
        self.action = action

        self.commands = []
        self.commands_only = False
        self.continuity_only = False
        self.list_only = False
        self.versions_only = False
        self.version_queue = version_queue
        self.version_files = {}

        self.max_threads  = max_threads
        self.thread_ttl   = 7200 # Should not take more than 2 hours
        self.threads      = []

        self.output_directory = '.'

        # Group based station tracking dictionaries
        self.stations_fresh    = station_names
        self.stations_retry    = [] # checks failed, try again
        self.stations_complete = [] # checks succeeded, done
        self.stations_expired  = [] # tried max number of times allowed
        self.stations_partial  = [] # stations that are missing information

        self.fresh = True
        self.done = False
        self.running = False

        self.queue = Queue.Queue()

        self.logger = Logger()
        self.logger.set_log_to_screen(True)
        self.logger.set_log_to_file(False)

    def set_output_directory(self, dir):
        self.output_directory = dir

    def set_commands(self, commands):
        self.commands = commands

    def set_commands_only(self, only=True):
        self.commands_only = only

    def set_continuity_only(self, only=True):
        self.continuity_only = only

    def set_list_only(self, only=True):
        self.list_only = only

    def set_version_files(self, files):
        self.version_files = files

    def set_versions_only(self, only=True):
        self.versions_only = only

    def is_done(self):
        alive = False
        try:
            alive = self.is_alive()
        except:
            alive = self.isAlive()
        return not alive

# ===== Preparation methods =====
    def prep_station(self, station, info):
        if info.has_key('type'):
            station.type = info['type']
        if info.has_key('name'):
            station.set_name(info['name'])
        if info.has_key('group'):
            station.set_group(info['group'])
        if info.has_key('address'):
            station.set_address(info['address'])
        if info.has_key('port'):
            station.set_port(info['port'])
        if info.has_key('username'):
            station.set_username(info['username'])
        if info.has_key('password'):
            station.set_password(info['password'])
        if info.has_key('prompt'):
            station.prompt_shell = info['prompt']
        if info.has_key('skip-continuity'):
            station.skip_continuity_check(info['skip-continuity'].upper() == 'TRUE')
        if info.has_key('netserv'):
            list = info['netserv'].split(',')
            for item in list:
                station.add_netserv_log(item)
        if info.has_key('server'):
            list = info['server'].split(',')
            for item in list:
                station.add_server_log(item)
        if info.has_key('sync-multiplier'):
            station.sync_multiplier = int(info['sync-multiplier'])
        station.info = info

    def prep_proxy(self, proxy, info, station=None):
        if info.has_key('type'):
            proxy.type = info['type']
        if info.has_key('name'):
            proxy.set_name(info['name'])
        if info.has_key('address'):
            proxy.set_address(info['address'])
        if info.has_key('port'):
            proxy.set_port(info['port'])
        if info.has_key('username'):
            proxy.set_username(info['username'])
        if info.has_key('password'):
            proxy.set_password(info['password'])
        if info.has_key('prompt'):
            proxy.prompt_shell = info['prompt']
        if info.has_key('local-address'):
            proxy.local_address = info['local-address']
        else:
            proxy.local_address = "127.0.0.1"
        if info.has_key('local-port'):
            proxy.local_port = info['local-port']
        if info.has_key('sync-multiplier'):
            proxy.sync_multiplier = int(info['sync-multiplier'])
        if station is not None:
            proxy.station_address = station.address
            proxy.station_port = station.port
            proxy.group = station.group
        proxy.info=info


  # Recursively determines a station's proxy chain
    def find_proxies(self, station, station_info, proxies=None):
        if proxies is None:
            proxies = {}
        if station_info.has_key('proxy'):
            if self.manager.proxies.has_key(station_info['proxy']):
                station.proxy_info = self.manager.proxies[station_info['proxy']]
                # Search for nested proxies
                if proxies.has_key(station.proxy_info['name']):
                    raise Exception("Encountered a proxy loop, first repeated on proxy '%(proxy)s'" % station_info)
                station.proxy = Proxy(station.proxy_info['name'])
                self.prep_proxy(station.proxy, station.proxy_info, station)
                # Look for any proxy upon which this proxy depends (nested proxy support)
                self.find_proxies(station.proxy, station.proxy_info, proxies)
            else:
                raise Exception("Could not locate proxy '%(proxy)s' associated with station '%(name)s'" % station_info)

  # Recursively starts a station's proxy chain
    def start_proxies(self, station, dir, depth):
        if station.proxy is not None:
            # The depth logic can be a little confusing. We need this in order
            # to track how many ~ characters are required in order to reach
            # the SSH shell from this level. By passing a list, we can modify
            # its contents before returning control to our parent, thereby
            # giving increment control to the child. This ensures that the
            # parent proxy has a higher depth value than its child proxy, and
            # on down the chain. The parents is dependent on the child proxy
            # to establish the next portion of the path before it can connect.
            # We connect the child proxy before calling the parent proxy's start
            # method which establishes all connections in the correct order.
            proxy = station.proxy
            proxy.log_file_name(dir + '/proxy.log')
            proxy.log_to_file()
            proxy.log_to_screen()
            proxy.set_output_directory(self.output_directory + '/' + station.name)
            proxy.logger.set_log_note("%s:%s" % (proxy.name, station.name))
            self.start_proxies(proxy, dir, depth)
            proxy.depth = depth[0]
            station._log("Starting proxy '%s:%s' thread at depth %d..." % (proxy.name, station.name, depth[0]))
            depth[0] = depth[0] + 1
            proxy.start()

    def summarize(self):
        # build a summary
        self.logger.log("All stations have been processed.")

    def halt(self):
        self.stations_fresh = []
        for thread in self.threads:
            self.logger.log("Halting thread '%s'..." % thread.name)
            thread.halt(now=True)
            thread.join()

    def record(self, station):
        if not station:
            return
        date = time.gmtime()
        # by default, check output is stored in path as follows:
        # $HOME/stations/gsn/<station>/<year>/<j_day>/<HHMMSS>.chk
        dir = self.output_directory + '/' + station.name + time.strftime("/%Y/%j", date)
        file = time.strftime("%H%M%S.chk", date)

        # create the directories if they do not exist
        ydir = dir.rsplit('/',1)[0]
        if not os.path.exists(ydir):
            try:
                os.makedirs(ydir)
                Permissions("EEEEEEEIE", 1).process([ydir])
            except:
                station._log("could not create directory %s" % ydir)
                return
        if not os.path.exists(dir):
            try:
                os.makedirs(dir)
                Permissions("EEEEEEEIE", 1).process([dir])
            except:
                station._log("could not create directory %s" % dir)
                return

        # if the target directory path exists but is not a directory, complain
        elif not os.path.isdir(dir):
            station._log("%s exists, and is not a directory, please resolve." % dir)
            return

        # write results into the summary file
        try:
            summary_file = dir + '/' + file
            fd = open(summary_file, "a")
            fd.write( station.summary )
            fd.write( station.output )
            fd.write( station.log_messages )
            fd.close()
            Permissions("EEIEEIEII", 1).process([summary_file])
        except Exception, e:
            station._log("CheckLoop::record() failed to record data to file. Exception: %s" % str(e))