def wait_catalog(self): """Blocks current execution until the catalog service is available. If it is not available remotely, start up a local service. """ start = dt.datetime.now() settings = systemsettings() while True: try: if self.catalog is None: self.catalog = RedisClient(settings.name) self.catalog.ping() break except OverlayNotAvailable as e: self.start_local_catalog() self.catalog = None except redis.RedisError as e: self.catalog = None delta = (dt.datetime.now() - start).total_seconds() if delta > 1: logging.info('CLIENT_DELAY,%f', delta)
def check_except(name): log = logging.getLogger(name) total_uptime = 0. total_downtime = 0. run_time = 0. counter = 0 up = False uptxt = 'DOWN' ts = dt.now() log.info('START,%f,%f,%f', total_uptime, total_downtime, run_time) while True: ts = dt.now() time.sleep(.5) try: client = RedisClient(name) status = client.ping() if counter == 0: client.set('testcounter', 1) else: client.incr('testcounter') counter += 1 testcounter = int(client.get('testcounter')) assert (counter == testcounter) except redis.RedisError as e: print(' REDIS ERROR ===> ', e.__name__) status = False except OverlayNotAvailable as e: print(' OVERLAY not available') status = False delta = (dt.now() - ts).total_seconds() if status == up: run_time += delta else: print('STATUS Change from %s' % uptxt) log.info('%s,%f,%f,%f', uptxt, total_uptime, total_downtime, run_time) run_time = 0. if status: uptxt = 'UP' total_uptime += delta else: uptxt = 'DOWN' total_downtime += delta print('%s,%f' % (uptxt, run_time)) up = status
def check_except(name): log = logging.getLogger(name) total_uptime = 0. total_downtime = 0. run_time = 0. counter = 0 up = False uptxt = 'DOWN' ts = dt.now() log.info('START,%f,%f,%f', total_uptime, total_downtime, run_time) while True: ts = dt.now() time.sleep(.5) try: client = RedisClient(name) status = client.ping() if counter == 0: client.set('testcounter', 1) else: client.incr('testcounter') counter += 1 testcounter = int(client.get('testcounter')) assert (counter == testcounter) except redis.RedisError as e: print(' REDIS ERROR ===> ', e.__name__) status = False except OverlayNotAvailable as e: print(' OVERLAY not available') status = False delta = (dt.now()-ts).total_seconds() if status == up: run_time += delta else: print('STATUS Change from %s' % uptxt) log.info('%s,%f,%f,%f', uptxt, total_uptime, total_downtime, run_time) run_time = 0. if status: uptxt = 'UP' total_uptime += delta else: uptxt = 'DOWN' total_downtime += delta print('%s,%f' % (uptxt, run_time)) up = status
catalog.rpush('jcqueue', jcID) catalog.hmset(wrapKey('jc', jcID), config) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('name', default='default') parser.add_argument('--initcatalog', action='store_true') parser.add_argument('--lattice', action='store_true') parser.add_argument('--centroid', action='store_true') args = parser.parse_args() confile = args.name + '.json' settings = systemsettings() settings.applyConfig(confile) catalog = RedisClient(args.name) if args.initcatalog: settings.envSetup() initialize.initializecatalog(catalog) if args.lattice: numresources = int(catalog.get('numresources')) bootstrap_lattice(catalog, numresources) if args.centroid: start_coords = [(2586,315), (531,20), (3765,63), (3305,668), (1732,139)] for fileno, frame in start_coords: initialize.manual_de_job(catalog, fileno, frame)
# return self.feal_list[trnum] ############################# if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('name', default='default') parser.add_argument('--centroid', action='store_true') args = parser.parse_args() confile = args.name + '.json' settings = systemsettings() settings.applyConfig(confile) catalog = RedisClient(args.name) # TO Recalculate PCA Vectors from DEShaw (~30-40 mins at 10% of data) # calcDEShaw_PCA(catalog) # sys.exit(0) if args.centroid: centroid_bootstrap(catalog) # #================ # hcf = {k: np.array([np.array(feal[i]) for i in v['elm']]) for k,v in hc5.items()} # fmean = {k: np.mean(v, axis=0) for k,v in hcf.items()} # def find_hc(hclist, index):
parser.add_argument('--centroid', action='store_true') # parser.add_argument('--initarchive', action='store_true') # parser.add_argument('--seeddata', action='store_true') # parser.add_argument('--reindex', nargs='?', const=10, type=int) # parser.add_argument('--loadindex', type=int) # parser.add_argument('--num', type=int, default=50) # parser.add_argument('--winsize', type=int, default=200) # parser.add_argument('--slide', type=int, default=100) args = parser.parse_args() confile = args.name + '.json' settings = systemsettings() settings.applyConfig(confile) catalog = RedisClient(args.name) # TO Recalculate PCA Vectors from DEShaw (~30-40 mins at 10% of data) # calcDEShaw_PCA(catalog) # sys.exit(0) if args.centroid: centroid_bootstrap(catalog) if args.initcatalog or args.all: settings.envSetup() initializecatalog(catalog) if args.onejob: makejobconfig(catalog)
def run(self): args = self.parser.parse_args() settings = systemsettings() self.experiment_number = settings.EXPERIMENT_NUMBER logging.info("APPLICATION: %s", settings.APPL_LABEL) logging.info("WORKDIR: %s", settings.WORKDIR) # Read in Slurm params (TODO: Move to abstract slurm call) if self.job_id is None: self.job_id = os.getenv('JOB_NAME') self.slurm_id = os.getenv('SLURM_JOB_ID') logging.debug('EnVars') for i in ['SBATCH_JOBID', 'SBATCH_JOB_NAME', 'SLURM_JOB_ID', 'SLURM_JOBID', 'SLURM_JOB_NAME']: logging.debug(' %s : %s', i, os.getenv(i)) logging.info("JOB NAME : %s", str(self.job_id)) logging.info("SLURM JOB: %s", str(self.slurm_id)) if args.debug: logging.debug("DEBUGGING: %s", self.name) if args.single: logging.debug("Macrothread running in single exection Mode (only 1 manager will execute).") self.singleuse = True if args.init: sys.exit(0) # Both Worker & Manager need catalog to run; load it here and import schema retry = 3 connected = False while retry > 0: retry -= 1 logging.info('Trying to estabish connection to the Catalog Service') try: self.catalog = RedisClient(settings.name) if self.catalog.isconnected and self.catalog.ping(): logging.info('Catalog service is connected') connected = True break logging.info("Catalog service is not running. Trying to start the service now") self.start_local_catalog() except (redis.RedisError, OverlayNotAvailable) as e: self.catalog = None self.start_local_catalog() if not connected: # If the catalog is unavailable. Fail this thread and re-schedule it if args.workinput: relaunch_cmd = "python3 %s -c %s -w" % (self.fname, self.config, args.workinput) else: self.slurmParams['cpus-per-task'] = 1 relaunch_cmd = "python3 %s -c %s" % (self.fname, self.config) self.slurmParams['job-name'] = self.job_id slurm.sbatch(taskid =self.slurmParams['job-name'], options = self.slurmParams, modules = self.modules, cmd = relaunch_cmd) # NOTE: This should be handled in an exception (need to figure out which one) # And then raise a custom OverlayConnectionError here return # LOAD Some self-bootstraping meta-data (if not alread loaded): mthread_key = 'macrothread:' + self.name if not self.catalog.exists(mthread_key): self.catalog.hmset(mthread_key, {'fname': self.fname}) self.catalog.loadSchema() # Should this be called from within the catalog module? # Load meta-data about registered mactrothreads self.data['macrothread'] = {} for key in self.catalog.keys('macrothread'): mt_name = key.split(':')[1] self.data['macrothread'][mt_thread] = self.catalog.hgetall(key) # Load current STATE from Catalog logging.info("Loading Thread State for from catalog:") # Load Standard set of simple params (init and simulation vals) # By default these are immutable. For any vals which may change or update # during execution, they should be explicitly set in the _mut or _append self.load(list(settings.state.keys())) self.load(list(settings.sim_params.keys())) # Load additional State values self.load(self._mut, self._immut, self._append) if args.workinput: logging.debug("Running worker.") self.worker(args.workinput) else: self.manager() if self.localcatalogserver: logging.debug("This thread is running the catalog. Waiting on local service to terminate...") self.localcatalogserver.join() self.localcatalogserver = None
class macrothread(object): """ The Macrothread Abstract Class """ __metaclass__ = abc.ABCMeta def __init__(self, fname, name): settings = systemsettings() self.name = name self.fname = fname # TODO: Better handling of command line args self.parser = None self.parser = self.addArgs() args, child_args = self.parser.parse_known_args() self.config = args.config # apply global settings (TODO: Move to system pro-proc) # Default to json ini file: if not self.config.endswith('json'): self.config += '.json' settings.applyConfig(self.config) # Down Stream Node list self._downnodes = [] self.exclude_notify = False # Thread State self._mut = [] self._immut = [] self._append = [] self._state = {} self.data = {} self.origin = {} self.upstream = None self.downstream = None # Elasticity self.delay = None # Data Stre self.catalog = None self.localcatalogserver = None # Runtime Modules to load -- Assumes Redis as baseline catalog (for now) self.modules = set(['redis']) # Default Runtime parameters to pass to slurm manager # These vary from mt to mt (and among workers) and can be updated # through the prepare method self.slurmParams = {'time':'6:00:0', 'nodes':1, 'cpus-per-task':1, 'partition':settings.PARTITION, 'job-name':self.name, 'workdir' : os.getcwd()} self.slurm_id = None self.job_id = None self.seq_num = None # To help debugging self.singleuse = False self.experiment_number = 0 # For file logging (if flagged) if args.filelog: self.filelog = logging.getLogger(name) self.filelog.setLevel(logging.INFO) filename = 'ctl.log' fh = logging.FileHandler(filename) fh.setLevel(logging.INFO) fmt = logging.Formatter('%(message)s') fh.setFormatter(fmt) self.filelog.addHandler(fh) self.filelog.propagate = False logging.info("File Logging is enabled. Logging some output to: %s", filename) else: self.filelog = None # Job ID Management # Sequence ALL JOBS --- good for up to 10,000 managers and 100 concurrently launched workers # TODO: Can make this larger, if nec'y def toMID(self, jobid): return '%sm-%04d.00' % (self.name[0], int(jobid)) def fromMID(self): if self.job_id is None: return None if re.match(r'\D[m,w]-\d{4}.{2}', self.job_id): job = self.job_id.split('-')[1] m, w = job.split('.') return int(m) else: return None def toWID(self, jobid, worknum): return '%sw-%04d.%02d' % (self.name[0], jobid, worknum) def seqNumFromID(self): """Returns the logical sequenced number for this macrothread """ if self.job_id is None: return '000000' if re.match(r'\D[m,w]-\d{4}.{2}', self.job_id): job = self.job_id.split('-')[1] m, w = job.split('.') return '%06d' % (100*(int(m)) + int(w)) else: return '000000' # Methods to add data elements to thread state def addMut(self, key, value=None): self._mut.append(key) if value is not None: self.data[key] = value def addImmut(self, key, value=None): self._immut.append(key) def addAppend(self, key, value=None): self._append.append(key) self.origin[key] = None if value is not None: self.data[key] = value self.origin[key] = copy.deepcopy(value) # TODO: Eventually we may want multiple up/down streams def setStream(self, upstream, downstream): """ Sets up/downstream data items """ if upstream is None: logging.info("Upstream data `%s` not defined", upstream) self.upstream = upstream if downstream is None: logging.warning("Downstream data `%s` not defined in schema", downstream) self.downstream = downstream def register_downnode(self, mthread): """ registers a downstream macrothread """ self._downnodes.append(mthread) # Implementation methods for macrothreads @abc.abstractmethod def term(self): raise NotImplementedError("This method has not been implemented.") @abc.abstractmethod def split(self): raise NotImplementedError("This method has not been implemented.") @abc.abstractmethod def execute(self, item): raise NotImplementedError("This method has not been implemented.") def fetch(self, item): """ Retrieve data element associated with given item reference (defaults to returning item ref) """ return item def preparejob(self, job): """ Called just prior to submitting a job on the Queue -- allows inherited macrothread to affect how a job is scheduled (e.g. dynamically setting Slurm params) in the Manager """ pass def configElasPolicy(self): """ Set rescheduling / delay policies for the manager thread """ self.delay = 60 def notify(self, mthread): """ Notifies the Manager process of the given macrothread to begin executing (within the next 60 seconds). If the manager is not running, this will launch it locally """ settings = systemsettings() if self.singleuse: logging.info("Notification invoked, but running in single use (not notifying)") return logging.info("Notifying downstream `%s` to executing in 60 seconds.", mthread) fname = self.catalog.hget('macrothread:ctl', 'fname') # fname = self.data['macrothread'][mthread]['fname'] mngr_prefix = self.fname[0] + 'm' job_queue = slurm.currentqueue() found_jobs = [] for j in job_queue: if j['name'].startswith(mngr_prefix): found_jobs.append(j) if len(found_jobs) == 0: logging.warning('Downstream manager not running. Recovering thread locally...') stdout = proc.check_output('python3 %s -c %s' % (fname, settings.name), shell=True) logging.info('Manager RECOVERED!\n %s\n------------', stdout) else: if len(found_jobs) > 1: logging.warning('Multple Downsteam manager found. Notifying earliest submitted') downstream_mngr = min(found_jobs, key=lambda x: x['submit']) else: downstream_mngr = found_jobs[0] jobid = int(downstream_mngr['jobid']) logging.info("Notifying %s Manager (jobid = %s) to start in 60 seconds", mthread, jobid) slurm.notify_start(jobid, delay=60) # Catalog access methods def setCatalog(self, catalog): self.catalog = catalog def getCatalog(self): return self.catalog def load(self, *keylist): """ Load state from remote catalog to local cache """ keys = [] for k in keylist: if isinstance(k, list): keys.extend(k) elif isinstance(k, str): keys.append(k) else: logging.error("Key Loading error: %s", str(k)) # pass expeected data types (interim solution) data = self.catalog.load(keys) logging.debug("Loaded state for %s:", self.name) for k,v in data.items(): self.data[k] = v logging.debug(" %-15s %s", k, str(v)[:15]) logging.debug("Setting origin for append-only") for k in keys: if k in self._append: self.origin[k] = copy.deepcopy(data[k]) logging.debug(" Setting Append only origin for: %s", k) def save(self, *keylist): """ Save state to remote catalog """ state = {} keys = [] for k in keylist: if isinstance(k, list): keys.extend(k) elif isinstance(k, str): keys.append(k) else: logging.error("Key Loading error: %s", str(k)) for key in keys: if key not in self.data: logging.error("KEY ERROR. %s not found in current state", key) else: state[key] = self.data[key] self.catalog.save(state) def append(self, *keylist): """ Save state to remote catalog """ state = {} keys = [] for k in keylist: if isinstance(k, list): keys.extend(k) elif isinstance(k, str): keys.append(k) else: logging.error("Key Loading error: %s", str(k)) for key in keys: if key not in self.data: logging.error("KEY ERROR. %s not found in current state", key) else: if type(self.data[key]) in [int, float, np.ndarray]: state[key] = self.data[key] - self.origin[key] elif isinstance(self.data[key], list): state[key] = [x for x in self.data[key] if x not in self.origin[key]] elif isinstance(self.data[key], dict): state[key] = {k:v for k,v in self.data[key].items() if k not in self.origin[key].keys()} else: logging.error("CANNOT APPEND data `%s` of type of %s", key, str(type(self.data[key]))) return self.catalog.append(state) # Manager Algorithm def manager(self, fork=False): logging.debug("\n==========================\n MANAGER: %s", self.name) # Check global termination: term_flag = self.data['terminate'] if term_flag and term_flag.lower() in ['halt', 'stop', 'now']: logging.info('RECEIVED TERMINATION FLAG. Shutting down') sys.exit(0) # Load Data from Thread's State and Upstream thread if self.upstream: logging.debug("Loading upstream data: %s", self.upstream) self.load(self.upstream) # Check for termination if self.term(): logging.info('TERMINATION condition for ' + self.name) return 0 # Set Elasticity Policy self.configElasPolicy() # Note: Manager can become a service daemon. Thus, we allow the manager # to run along with the monitor process and assume the manager overhead # is small enough to not interfere. Eventually, this will be threaded # differently by preventing the local service (within this object's # context) from running while the manager performs its split() function # worker dispatching. The worker (below) starts a local service # for reading, reads in the state, stops it, performs its work, and then # starts it for writing and remain alive to monitor...... # Hence, we'll eventually change this next line to False or some other # state value or we'll just let this manager become the monitor and # provide the service which means it will need to immediate re-schedule # itself # self.catalogPersistanceState = True # if self.localcatalogserver and not self.catalogPersistanceState: # self.catalog.stop() # self.localcatalogserver = None # TODO: Det if manager should load entire input data set, make this abstract, or # push into UDF portion # Defer can return either a list of items to push back or a "split" value to # perform an in-line data trim on the key-store DB (optimiation) immed, defer = self.split() # Manager oversee's id assignment. idlabel = 'id_%s' % self.name self.catalog.incr(idlabel) nextid = self.catalog.get(idlabel) # first ID check nextid = 0 if nextid is None else int(nextid) myid = self.fromMID() if myid is None: myid = int(nextid - 1) # No Jobs to run.... Delay and then rerun later if len(immed) == 0: delay = int(self.delay) logging.debug("MANAGER %s: No Available input data. Delaying %d seconds and rescheduling...." % (self.name, delay)) self.slurmParams['begin'] = 'now+%d' % delay # Dispatch Workers else: workernum = 1 delay = 180 + self.delay # Set baseline slurm params and modules (to allow for dynamic disatching) baseline_param = copy.deepcopy(self.slurmParams) baseline_mods = copy.deepcopy(self.modules) for i in immed: logging.debug("%s: scheduling worker, input=%s", self.name, i) self.preparejob(i) self.slurmParams['job-name'] = self.toWID(myid, workernum) slurm.sbatch(taskid=self.slurmParams['job-name'], options = self.slurmParams, modules = self.modules, cmd = "python3 %s -c %s -w %s" % (self.fname, self.config, str(i))) workernum += 1 # Reset params and mods self.slurmParams = copy.deepcopy(baseline_param) self.modules = copy.deepcopy(baseline_mods) # Single use exit: if self.singleuse: logging.debug("SINGLE USE INVOKED. No more managers will run ") return 0 # Elas Policy to control manager rescheduling self.slurmParams['begin'] = 'now+%d' % delay self.slurmParams['job-name'] = self.toMID(nextid) self.slurmParams['cpus-per-task'] = 1 slurm.sbatch(taskid =self.slurmParams['job-name'], options = self.slurmParams, modules = self.modules, cmd = "python3 %s -c %s" % (self.fname, self.config)) # TODO: Alternate manager rescheduling: Trigger Based # use after:job_id[:jobid...] w/ #SBATCH --dependency=<dependency_list> # Consume upstream input data logging.debug('Consuming Upstream Data....') if isinstance(defer, list): logging.debug('Deferring a list and removing %d items tasked to run immed', len(immed)) self.catalog.removeItems(self.upstream, immed) elif defer is not None: logging.debug('Slicing %d items', defer) self.catalog.slice(self.upstream, defer) # Other interal thread state is saved back to catalog self.save(self._mut) logging.debug("==========================") return 0 def worker(self, i): logging.debug("\n--------------------------\n WORKER: %s", self.name) # TODO: Optimization Notifications # catalog.notify(i, "active") logging.info("WORKER Fetching Input parameters/data for input: %s", str(i)) jobInput = self.fetch(i) logging.debug("Starting WORKER Execution ---->>") result = self.execute(jobInput) logging.debug("<<---- WORKER Execution Complete") # Ensure returned results are a list if type(result) != list: result = [result] # catalog.notify(i, "complete") for r in result: # TODO: Notification of downstream data to catalog # Option A -> passive model (for now) # Option B -> active. do you want to do something with job??? # If data is appended, make it passive # If data should be prioritized, then you need to be active # Boils down to scheduling model # catalog.notify(r, 'ready') # TODO: Add in notification for congestion control # Need feedback to upstreaam to adjust split param # trigger flow bet streams is tuning action (e.g. assume 1:1 to start) logging.debug (" WORKER output: %s", r) if self.downstream is not None and len(result) > 0: self.catalog.append({self.downstream: result}) logging.debug("Updating all append-only state items") self.append(self._append) logging.debug("Saving all mutable state items") self.save(self._mut) logging.debug("--------------------------") def addArgs(self): if self.parser is None: parser = argparse.ArgumentParser() parser.add_argument('-w', '--workinput') parser.add_argument('-c', '--config', default='default.conf') parser.add_argument('-i', '--init', action='store_true') parser.add_argument('-d', '--debug', action='store_true') parser.add_argument('-s', '--single', action='store_true') parser.add_argument('-f', '--filelog', action='store_true') self.parser = parser return self.parser def start_local_catalog(self, isaggressive=False): settings = systemsettings() service = RedisService(settings.name, isaggressive=False) self.localcatalogserver = service.start() logging.info("Catalog service started as a background thread. Waiting on it...") timeout = settings.CATALOG_STARTUP_DELAY while timeout > 0: if service.ping(): return True logging.info("Waiting on the Server. Will timeout in %d secs.", timeout) time.sleep(1) timeout -= 1 self.localcatalogserver = None return False def wait_catalog(self): """Blocks current execution until the catalog service is available. If it is not available remotely, start up a local service. """ start = dt.datetime.now() settings = systemsettings() while True: try: if self.catalog is None: self.catalog = RedisClient(settings.name) self.catalog.ping() break except OverlayNotAvailable as e: self.start_local_catalog() self.catalog = None except redis.RedisError as e: self.catalog = None delta = (dt.datetime.now() - start).total_seconds() if delta > 1: logging.info('CLIENT_DELAY,%f', delta) def run(self): args = self.parser.parse_args() settings = systemsettings() self.experiment_number = settings.EXPERIMENT_NUMBER logging.info("APPLICATION: %s", settings.APPL_LABEL) logging.info("WORKDIR: %s", settings.WORKDIR) # Read in Slurm params (TODO: Move to abstract slurm call) if self.job_id is None: self.job_id = os.getenv('JOB_NAME') self.slurm_id = os.getenv('SLURM_JOB_ID') logging.debug('EnVars') for i in ['SBATCH_JOBID', 'SBATCH_JOB_NAME', 'SLURM_JOB_ID', 'SLURM_JOBID', 'SLURM_JOB_NAME']: logging.debug(' %s : %s', i, os.getenv(i)) logging.info("JOB NAME : %s", str(self.job_id)) logging.info("SLURM JOB: %s", str(self.slurm_id)) if args.debug: logging.debug("DEBUGGING: %s", self.name) if args.single: logging.debug("Macrothread running in single exection Mode (only 1 manager will execute).") self.singleuse = True if args.init: sys.exit(0) # Both Worker & Manager need catalog to run; load it here and import schema retry = 3 connected = False while retry > 0: retry -= 1 logging.info('Trying to estabish connection to the Catalog Service') try: self.catalog = RedisClient(settings.name) if self.catalog.isconnected and self.catalog.ping(): logging.info('Catalog service is connected') connected = True break logging.info("Catalog service is not running. Trying to start the service now") self.start_local_catalog() except (redis.RedisError, OverlayNotAvailable) as e: self.catalog = None self.start_local_catalog() if not connected: # If the catalog is unavailable. Fail this thread and re-schedule it if args.workinput: relaunch_cmd = "python3 %s -c %s -w" % (self.fname, self.config, args.workinput) else: self.slurmParams['cpus-per-task'] = 1 relaunch_cmd = "python3 %s -c %s" % (self.fname, self.config) self.slurmParams['job-name'] = self.job_id slurm.sbatch(taskid =self.slurmParams['job-name'], options = self.slurmParams, modules = self.modules, cmd = relaunch_cmd) # NOTE: This should be handled in an exception (need to figure out which one) # And then raise a custom OverlayConnectionError here return # LOAD Some self-bootstraping meta-data (if not alread loaded): mthread_key = 'macrothread:' + self.name if not self.catalog.exists(mthread_key): self.catalog.hmset(mthread_key, {'fname': self.fname}) self.catalog.loadSchema() # Should this be called from within the catalog module? # Load meta-data about registered mactrothreads self.data['macrothread'] = {} for key in self.catalog.keys('macrothread'): mt_name = key.split(':')[1] self.data['macrothread'][mt_thread] = self.catalog.hgetall(key) # Load current STATE from Catalog logging.info("Loading Thread State for from catalog:") # Load Standard set of simple params (init and simulation vals) # By default these are immutable. For any vals which may change or update # during execution, they should be explicitly set in the _mut or _append self.load(list(settings.state.keys())) self.load(list(settings.sim_params.keys())) # Load additional State values self.load(self._mut, self._immut, self._append) if args.workinput: logging.debug("Running worker.") self.worker(args.workinput) else: self.manager() if self.localcatalogserver: logging.debug("This thread is running the catalog. Waiting on local service to terminate...") self.localcatalogserver.join() self.localcatalogserver = None