def __init__(self, config, section): self._config = config self._section = section topics = config.get(section, 'topics').split() services = "" if config.has_option(section, 'services'): services = config.get(section, 'services').split() self._listener = ListenerContainer(topics=topics, services=services) self._publisher = publisher.NoisyPublisher("geo_gatherer") self._subject = config.get(section, "publish_topic") self._pattern = config.get(section, 'pattern') self._providing_server = None if config.has_option(section, 'providing_server'): self._providing_server = config.get(section, 'providing_server') self._parser = Parser(self._pattern) try: self._timeliness = dt.timedelta( seconds=config.getint(section, "timeliness")) except (NoOptionError, ValueError): self._timeliness = dt.timedelta(seconds=20) self._timeout = None self.metadata = {} self.received_files = set() self.wanted_files = set() self.all_files = set() self.critical_files = set() self.delayed_files = OrderedDict() self.logger = logging.getLogger("geo_gatherer") self._loop = False
def __init__(self, config, section): self._config = config self._section = section topics = config.get(section, 'topics').split() try: nameservers = config.get(section, 'nameserver') nameservers = nameservers.split() except (NoOptionError, ValueError): nameservers = [] try: addresses = config.get(section, 'addresses') addresses = addresses.split() except (NoOptionError, ValueError): addresses = None try: publish_port = config.get(section, 'publish_port') except NoOptionError: publish_port = 0 try: services = config.get(section, 'services').split() except (NoOptionError, ValueError): services = "" self._listener = ListenerContainer(topics=topics, addresses=addresses services=services) self._publisher = publisher.NoisyPublisher("segment_gatherer", port=publish_port, nameservers=nameservers) self._subject = config.get(section, "publish_topic") self._pattern = config.get(section, 'pattern') self._parser = Parser(self._pattern) try: self._time_tolerance = config.getint(section, "time_tolerance") except NoOptionError: self._time_tolerance = 30 try: self._timeliness = dt.timedelta(seconds=config.getint(section, "timeliness")) except (NoOptionError, ValueError): self._timeliness = dt.timedelta(seconds=1200) try: self._num_files_premature_publish = \ config.getint(section, "num_files_premature_publish") except (NoOptionError, ValueError): self._num_files_premature_publish = -1 self.slots = OrderedDict() self.time_name = config.get(section, 'time_name') self.logger = logging.getLogger("segment_gatherer") self._loop = False self._providing_server = None if config.has_option(section, 'providing_server'): self._providing_server = config.get(section, 'providing_server')
def __init__(self, config, section): self._config = config self._section = section topics = config.get(section, 'topics').split() self._listener = ListenerContainer(topics=topics) self._publisher = publisher.NoisyPublisher("segment_gatherer") self._subject = config.get(section, "publish_topic") self._pattern = config.get(section, 'pattern') self._parser = Parser(self._pattern) try: self._timeliness = dt.timedelta( seconds=config.getint(section, "timeliness")) except (NoOptionError, ValueError): self._timeliness = dt.timedelta(seconds=1200) try: self._num_files_premature_publish = \ config.getint(section, "num_files_premature_publish") except (NoOptionError, ValueError): self._num_files_premature_publish = -1 self.slots = OrderedDict() self.time_name = config.get(section, 'time_name') self.logger = logging.getLogger("segment_gatherer") self._loop = False
def _setup_messaging(self): """Setup messaging""" self._subject = self._config['posttroll']['publish_topic'] topics = self._config['posttroll'].get('topics') addresses = self._config['posttroll'].get('addresses') publish_port = self._config['posttroll'].get('publish_port', 0) nameservers = self._config['posttroll'].get('nameservers', []) self._listener = ListenerContainer(topics=topics, addresses=addresses) self._publisher = publisher.NoisyPublisher("segment_gatherer", port=publish_port, nameservers=nameservers) self._publisher.start()
def _setup_messaging(self): """Set up messaging.""" self._subject = self._config['posttroll']['publish_topic'] topics = self._config['posttroll'].get('topics') addresses = self._config['posttroll'].get('addresses') publish_port = self._config['posttroll'].get('publish_port', 0) nameservers = self._config['posttroll'].get('nameservers', []) services = self._config['posttroll'].get('services', "") self._listener = ListenerContainer(topics=topics, addresses=addresses, services=services) # Name each segment_gatherer with the section/patterns name. # This way the user can subscribe to a specific segment_gatherer service instead of all. publish_service_name = self._generate_publish_service_name() self._publisher = publisher.NoisyPublisher(publish_service_name, port=publish_port, nameservers=nameservers) self._publisher.start()
" %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S')) if opts.verbose: loglevel = logging.DEBUG else: loglevel = logging.INFO handler.setLevel(loglevel) logging.getLogger('').setLevel(loglevel) logging.getLogger('').addHandler(handler) logging.getLogger("posttroll").setLevel(logging.INFO) logger = logging.getLogger("gatherer") decoder = get_metadata granule_triggers = [] pub = publisher.NoisyPublisher("gatherer") # TODO: get this from the product config files regions = [ get_area_def(region) for region in config.get("default", "regions").split() ] for section in config.sections(): if section == "default": continue timeliness = timedelta(minutes=config.getint(section, "timeliness")) try: duration = timedelta(seconds=config.getfloat(section, "duration")) except NoOptionError:
def main(): """Main() for gatherer. """ global LOGGER global PUB opts = arg_parse() CONFIG.read(opts.config) print("Setting timezone to UTC") os.environ["TZ"] = "UTC" time.tzset() handlers = [] if opts.log: handlers.append( logging.handlers.TimedRotatingFileHandler(opts.log, "midnight", backupCount=7)) handlers.append(logging.StreamHandler()) if opts.verbose: loglevel = logging.DEBUG else: loglevel = logging.INFO for handler in handlers: handler.setFormatter( logging.Formatter( "[%(levelname)s: %(asctime)s :" " %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S')) handler.setLevel(loglevel) logging.getLogger('').setLevel(loglevel) logging.getLogger('').addHandler(handler) logging.getLogger("posttroll").setLevel(logging.INFO) LOGGER = logging.getLogger("gatherer") if opts.config_item: for section in opts.config_item: if section not in CONFIG.sections(): LOGGER.warning( "No config item called %s found in config file.", section) for section in CONFIG.sections(): if section not in opts.config_item: CONFIG.remove_section(section) if len(CONFIG.sections()) == 0: LOGGER.error("No valid config item provided") return decoder = get_metadata PUB = publisher.NoisyPublisher("gatherer") granule_triggers = setup(decoder) PUB.start() for granule_trigger in granule_triggers: granule_trigger.start() try: while True: time.sleep(1) for granule_trigger in granule_triggers: if not granule_trigger.is_alive(): raise RuntimeError except KeyboardInterrupt: LOGGER.info("Shutting down...") except RuntimeError: LOGGER.critical('Something went wrong!') except OSError: LOGGER.critical('Something went wrong!') finally: LOGGER.warning('Ending publication the gathering of granules...') for granule_trigger in granule_triggers: granule_trigger.stop() PUB.stop()