def __init__(self, config, section):
self._config = config
self._section = section
topics = config.get(section, 'topics').split()
services = ""
if config.has_option(section, 'services'):
services = config.get(section, 'services').split()
self._listener = ListenerContainer(topics=topics, services=services)
self._publisher = publisher.NoisyPublisher("geo_gatherer")
self._subject = config.get(section, "publish_topic")
self._pattern = config.get(section, 'pattern')
self._providing_server = None
if config.has_option(section, 'providing_server'):
self._providing_server = config.get(section, 'providing_server')
self._parser = Parser(self._pattern)
try:
self._timeliness = dt.timedelta(
seconds=config.getint(section, "timeliness"))
except (NoOptionError, ValueError):
self._timeliness = dt.timedelta(seconds=20)
self._timeout = None
self.metadata = {}
self.received_files = set()
self.wanted_files = set()
self.all_files = set()
self.critical_files = set()
self.delayed_files = OrderedDict()
self.logger = logging.getLogger("geo_gatherer")
self._loop = False
def __init__(self, config, section):
self._config = config
self._section = section
topics = config.get(section, 'topics').split()
try:
nameservers = config.get(section, 'nameserver')
nameservers = nameservers.split()
except (NoOptionError, ValueError):
nameservers = []
try:
addresses = config.get(section, 'addresses')
addresses = addresses.split()
except (NoOptionError, ValueError):
addresses = None
try:
publish_port = config.get(section, 'publish_port')
except NoOptionError:
publish_port = 0
try:
services = config.get(section, 'services').split()
except (NoOptionError, ValueError):
services = ""
self._listener = ListenerContainer(topics=topics, addresses=addresses
services=services)
self._publisher = publisher.NoisyPublisher("segment_gatherer",
port=publish_port,
nameservers=nameservers)
self._subject = config.get(section, "publish_topic")
self._pattern = config.get(section, 'pattern')
self._parser = Parser(self._pattern)
try:
self._time_tolerance = config.getint(section, "time_tolerance")
except NoOptionError:
self._time_tolerance = 30
try:
self._timeliness = dt.timedelta(seconds=config.getint(section,
"timeliness"))
except (NoOptionError, ValueError):
self._timeliness = dt.timedelta(seconds=1200)
try:
self._num_files_premature_publish = \
config.getint(section, "num_files_premature_publish")
except (NoOptionError, ValueError):
self._num_files_premature_publish = -1
self.slots = OrderedDict()
self.time_name = config.get(section, 'time_name')
self.logger = logging.getLogger("segment_gatherer")
self._loop = False
self._providing_server = None
if config.has_option(section, 'providing_server'):
self._providing_server = config.get(section, 'providing_server')
def __init__(self, config, section):
self._config = config
self._section = section
topics = config.get(section, 'topics').split()
self._listener = ListenerContainer(topics=topics)
self._publisher = publisher.NoisyPublisher("segment_gatherer")
self._subject = config.get(section, "publish_topic")
self._pattern = config.get(section, 'pattern')
self._parser = Parser(self._pattern)
try:
self._timeliness = dt.timedelta(
seconds=config.getint(section, "timeliness"))
except (NoOptionError, ValueError):
self._timeliness = dt.timedelta(seconds=1200)
try:
self._num_files_premature_publish = \
config.getint(section, "num_files_premature_publish")
except (NoOptionError, ValueError):
self._num_files_premature_publish = -1
self.slots = OrderedDict()
self.time_name = config.get(section, 'time_name')
self.logger = logging.getLogger("segment_gatherer")
self._loop = False
def _setup_messaging(self):
"""Setup messaging"""
self._subject = self._config['posttroll']['publish_topic']
topics = self._config['posttroll'].get('topics')
addresses = self._config['posttroll'].get('addresses')
publish_port = self._config['posttroll'].get('publish_port', 0)
nameservers = self._config['posttroll'].get('nameservers', [])
self._listener = ListenerContainer(topics=topics, addresses=addresses)
self._publisher = publisher.NoisyPublisher("segment_gatherer",
port=publish_port,
nameservers=nameservers)
self._publisher.start()
def _setup_messaging(self):
"""Set up messaging."""
self._subject = self._config['posttroll']['publish_topic']
topics = self._config['posttroll'].get('topics')
addresses = self._config['posttroll'].get('addresses')
publish_port = self._config['posttroll'].get('publish_port', 0)
nameservers = self._config['posttroll'].get('nameservers', [])
services = self._config['posttroll'].get('services', "")
self._listener = ListenerContainer(topics=topics,
addresses=addresses,
services=services)
# Name each segment_gatherer with the section/patterns name.
# This way the user can subscribe to a specific segment_gatherer service instead of all.
publish_service_name = self._generate_publish_service_name()
self._publisher = publisher.NoisyPublisher(publish_service_name,
port=publish_port,
nameservers=nameservers)
self._publisher.start()
" %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S'))
if opts.verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
logger = logging.getLogger("gatherer")
decoder = get_metadata
granule_triggers = []
pub = publisher.NoisyPublisher("gatherer")
# TODO: get this from the product config files
regions = [
get_area_def(region)
for region in config.get("default", "regions").split()
]
for section in config.sections():
if section == "default":
continue
timeliness = timedelta(minutes=config.getint(section, "timeliness"))
try:
duration = timedelta(seconds=config.getfloat(section, "duration"))
except NoOptionError:
def main():
"""Main() for gatherer.
"""
global LOGGER
global PUB
opts = arg_parse()
CONFIG.read(opts.config)
print("Setting timezone to UTC")
os.environ["TZ"] = "UTC"
time.tzset()
handlers = []
if opts.log:
handlers.append(
logging.handlers.TimedRotatingFileHandler(opts.log,
"midnight",
backupCount=7))
handlers.append(logging.StreamHandler())
if opts.verbose:
loglevel = logging.DEBUG
else:
loglevel = logging.INFO
for handler in handlers:
handler.setFormatter(
logging.Formatter(
"[%(levelname)s: %(asctime)s :"
" %(name)s] %(message)s", '%Y-%m-%d %H:%M:%S'))
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
LOGGER = logging.getLogger("gatherer")
if opts.config_item:
for section in opts.config_item:
if section not in CONFIG.sections():
LOGGER.warning(
"No config item called %s found in config file.", section)
for section in CONFIG.sections():
if section not in opts.config_item:
CONFIG.remove_section(section)
if len(CONFIG.sections()) == 0:
LOGGER.error("No valid config item provided")
return
decoder = get_metadata
PUB = publisher.NoisyPublisher("gatherer")
granule_triggers = setup(decoder)
PUB.start()
for granule_trigger in granule_triggers:
granule_trigger.start()
try:
while True:
time.sleep(1)
for granule_trigger in granule_triggers:
if not granule_trigger.is_alive():
raise RuntimeError
except KeyboardInterrupt:
LOGGER.info("Shutting down...")
except RuntimeError:
LOGGER.critical('Something went wrong!')
except OSError:
LOGGER.critical('Something went wrong!')
finally:
LOGGER.warning('Ending publication the gathering of granules...')
for granule_trigger in granule_triggers:
granule_trigger.stop()
PUB.stop()
