def start_up():
try:
config.init_config()
except config.cfg.ConfigFilesNotFoundError as ex:
_LOG.exception(ex.message)
conf = config.get_config()
application = api = falcon.API()
api.add_route('/', VersionResource())
#http correlation endpoint
api.add_route('/v1/tenant/{tenant_id}/publish', PublishMessageResource())
#syslog correlation endpoint
server = SyslogServer(("0.0.0.0", 5140), syslog.MessageHandler(conf))
server.start()
syslog_server_proc = Process(target=start_io)
syslog_server_proc.start()
_LOG.info('Syslog server started as process: {}'.format(
syslog_server_proc.pid))
celery.conf.CELERYBEAT_SCHEDULE = {
'worker_stats': {
'task': 'stats.publish',
'schedule': timedelta(seconds=publish_stats.WORKER_STATUS_INTERVAL)
},
}
#include blank argument to celery in order for beat to start correctly
celery_proc = Process(target=celery.worker_main, args=[['', '--beat']])
celery_proc.start()
_LOG.info('Celery started as process: {}'.format(celery_proc.pid))
return application
def test_loading(self):
init_config(['--config-file', '../etc/meniscus/meniscus.conf'])
conf = get_config()
conf.register_group(test_group)
conf.register_opts(CFG_TEST_OPTIONS, group=test_group)
self.assertTrue(conf.test.should_pass)
def test_loading(self):
try:
init_config(['--config-file', '../etc/meniscus/meniscus.conf'])
except:
init_config(['--config-file', './etc/meniscus/meniscus.conf'])
conf = get_config()
conf.register_group(test_group)
conf.register_opts(CFG_TEST_OPTIONS, group=test_group)
self.assertTrue(conf.test.should_pass)
def test_loading(self):
try:
init_config(['--config-file', '../etc/meniscus/meniscus.conf'])
except:
print('ass {}'.format(os.getcwd()))
init_config(['--config-file', './etc/meniscus/meniscus.conf'])
conf = get_config()
conf.register_group(test_group)
conf.register_opts(CFG_TEST_OPTIONS, group=test_group)
self.assertTrue(conf.test.should_pass)
def db_handler():
try:
config.init_config()
except config.cfg.ConfigFilesNotFoundError:
#TODO(dmend) Log config error
pass
conf = config.get_config()
_handler = datasource_handler(conf)
_handler.connect()
return _handler
def start_up():
try:
config.init_config()
except config.cfg.ConfigFilesNotFoundError as ex:
_LOG.exception(ex.message)
application = api = falcon.API()
api.add_route('/', VersionResource())
#http correlation endpoint
api.add_route('/v1/tenant/{tenant_id}/publish', PublishMessageResource())
#syslog correlation endpoint
server = receiver.new_correlation_input_server()
server_proc = Process(target=server.start)
server_proc.start()
_LOG.info(
'ZeroMQ reception server started as process: {}'.format(
server_proc.pid)
)
celery.conf.CELERYBEAT_SCHEDULE = {
'worker_stats': {
'task': 'stats.publish',
'schedule': timedelta(seconds=publish_stats.WORKER_STATUS_INTERVAL)
},
}
#include blank argument to celery in order for beat to start correctly
celery_proc = Process(target=celery.worker_main, args=[['', '--beat']])
celery_proc.start()
_LOG.info(
'Celery started as process: {}'.format(celery_proc.pid)
)
es_flusher = ElasticSearchStreamBulker()
flush_proc = Process(target=es_flusher.start)
flush_proc.start()
_LOG.info(
'ElasticSearchStreamBulker started as process: {}'.format(
flush_proc.pid)
)
return application
def start_up():
try:
config.init_config()
except config.cfg.ConfigFilesNotFoundError as ex:
_LOG.exception(ex.message)
conf = config.get_config()
application = api = falcon.API()
api.add_route('/', VersionResource())
#http correlation endpoint
api.add_route('/v1/tenant/{tenant_id}/publish', PublishMessageResource())
#syslog correlation endpoint
server = SyslogServer(
("0.0.0.0", 5140), syslog.MessageHandler(conf))
server.start()
syslog_server_proc = Process(target=start_io)
syslog_server_proc.start()
_LOG.info(
'Syslog server started as process: {}'.format(syslog_server_proc.pid)
)
celery.conf.CELERYBEAT_SCHEDULE = {
'worker_stats': {
'task': 'stats.publish',
'schedule': timedelta(seconds=publish_stats.WORKER_STATUS_INTERVAL)
},
}
#include blank argument to celery in order for beat to start correctly
#celery_proc = Process(target=celery.worker_main, args=[['', '--beat']])
celery_proc = Process(target=celery.worker_main)
celery_proc.start()
_LOG.info(
'Celery started as process: {}'.format(celery_proc.pid)
)
return application
_WATCHLIST_GROUP = cfg.OptGroup(name='watchlist_settings',
title='Watchlist Settings')
get_config().register_group(_WATCHLIST_GROUP)
_WATCHLIST_THRESHOLDS = [
cfg.IntOpt('failure_tolerance_seconds',
default=60,
help="""default duration for monitoring failed workers"""),
cfg.IntOpt('watchlist_count_threshold',
default=5,
help="""count of reported failures""")
]
get_config().register_opts(_WATCHLIST_THRESHOLDS, group=_WATCHLIST_GROUP)
try:
init_config()
conf = get_config()
except cfg.ConfigFilesNotFoundError:
conf = get_config()
FAILURE_TOLERANCE_SECONDS = conf.watchlist_settings.failure_tolerance_seconds
WATCHLIST_COUNT_THRESHOLD = conf.watchlist_settings.watchlist_count_threshold
def _add_watchlist_item(db, worker_id):
"""
adds item to watchlist
"""
watch_item = WatchlistItem(worker_id)
db.put('watchlist', watch_item.format())
default='cache-config',
help="""The name of the cache to store worker config values"""
),
cfg.StrOpt('cache_tenant',
default='cache-tenant',
help="""The name of the cache to store worker config values"""
),
cfg.StrOpt('cache_token',
default='cache-token',
help="""The name of the cache to store worker config values"""
)
]
get_config().register_opts(_CACHE_OPTIONS, group=_cache_group)
try:
init_config()
conf = get_config()
except cfg.ConfigFilesNotFoundError:
conf = get_config()
DEFAULT_EXPIRES = conf.cache.default_expires
CONFIG_EXPIRES = conf.cache.config_expires
CACHE_CONFIG = conf.cache.cache_config
CACHE_TENANT = conf.cache.cache_tenant
CACHE_TOKEN = conf.cache.cache_token
class Cache(object):
def __init__(self):
self.cache = NativeProxy()
# Normalization configuration options
_NORMALIZATION_GROUP = cfg.OptGroup(name='liblognorm',
title='Liblognorm options')
config.get_config().register_group(_NORMALIZATION_GROUP)
_NORMALIZATION = [
cfg.StrOpt('rules_dir',
default=None,
help="""directory to load rules from""")
]
config.get_config().register_opts(_NORMALIZATION, group=_NORMALIZATION_GROUP)
try:
config.init_config()
except config.cfg.ConfigFilesNotFoundError as ex:
_LOG.exception(ex.message)
def get_normalizer(conf=config.get_config()):
"""This returns both a normalizer as well as a list of loaded rules"""
normalization_conf = conf.liblognorm
normalizer = LogNormalizer()
loaded_rules = list()
if normalization_conf.rules_dir:
loaded_rules = load_rules(normalizer, normalization_conf.rules_dir)
return (normalizer, loaded_rules)
def load_rules(normalizer, path):
def setUp(self):
init_config(['--config-file', 'meniscus.cfg'])
conf = get_config()
self.handler = datasource_handler(conf)
self.handler.connect()
help="""base HDFS directory to use"""
),
cfg.IntOpt('transaction_expire',
default=300,
help="""length of time for a write to hdfs before expiring"""
),
cfg.IntOpt('transfer_frequency',
default=60,
help="""frequency to write records to hdfs"""
)
]
config.get_config().register_opts(_hdfs_options, group=_hdfs_group)
try:
config.init_config()
except config.cfg.ConfigFilesNotFoundError as ex:
_LOG.exception(ex.message)
conf = config.get_config()
TRANSACTION_EXPIRE = conf.hdfs_sink.transaction_expire
FREQUENCY = conf.hdfs_sink.transfer_frequency
SINK = 'hdfs'
class HdfsTransaction(transaction.BatchMessageTransaction):
def _process_locked_records(self):
write_dir = "{0}/{1}".format(
def setUp(self):
init_config(['--config-file', 'meniscus.cfg'])
conf = get_config()
self.handler = datasource_handler(conf)
self.handler.connect()
