class Log_usingElasticSearch(BaseLog):
@use_settings
def __init__(self, host, index, type="log", max_size=1000, batch_size=100, settings=None):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
self.es = Cluster(settings).get_or_create_index(
schema=convert.json2value(convert.value2json(SCHEMA), paths=True),
limit_replicas=True,
settings=settings
)
self.queue = self.es.threaded_queue(max_size=max_size, batch_size=batch_size)
def write(self, template, params):
try:
if params.get("template"):
# DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE
self.queue.add({"value": params})
else:
if len(template) > 2000:
template = template[:1997] + "..."
self.queue.add({"value": {"template": template, "params": params}})
return self
except Exception, e:
raise e # OH NO!
class TextLog_usingElasticSearch(TextLog):
@use_settings
def __init__(self, host, index, type="log", max_size=1000, batch_size=100, settings=None):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
self.es = Cluster(settings).get_or_create_index(
schema=convert.json2value(convert.value2json(SCHEMA), leaves=True),
limit_replicas=True,
tjson=False,
settings=settings
)
self.es.add_alias("debug")
self.queue = self.es.threaded_queue(max_size=max_size, batch_size=batch_size)
def write(self, template, params):
if params.get("template"):
# DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE
self.queue.add({"value": params})
else:
template = strings.limit(template, 2000)
self.queue.add({"value": {"template": template, "params": params}}, timeout=3*MINUTE)
return self
def stop(self):
try:
self.queue.add(Thread.STOP) # BE PATIENT, LET REST OF MESSAGE BE SENT
except Exception, e:
pass
try:
self.queue.close()
except Exception, f:
pass
class Log_usingElasticSearch(BaseLog):
@use_settings
def __init__(self,
host,
index,
type="log",
max_size=1000,
batch_size=100,
settings=None):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
self.es = Cluster(settings).get_or_create_index(
schema=convert.json2value(convert.value2json(SCHEMA), paths=True),
limit_replicas=True,
settings=settings)
self.queue = self.es.threaded_queue(max_size=max_size,
batch_size=batch_size)
def write(self, template, params):
try:
if params.get("template"):
# DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE
self.queue.add({"value": params})
else:
if len(template) > 2000:
template = template[:1997] + "..."
self.queue.add(
{"value": {
"template": template,
"params": params
}})
return self
except Exception, e:
raise e # OH NO!
def __init__(self,
host,
index,
type=DATA_TYPE,
max_size=10,
batch_size=10,
kwargs=None):
"""
settings ARE FOR THE ELASTICSEARCH INDEX
"""
es = Cluster(kwargs).get_or_create_index(schema=json2value(
convert.value2json(SCHEMA), leaves=True),
limit_replicas=True,
typed=False,
kwargs=kwargs)
es.add_alias(index)
self.queue = es.threaded_queue(max_size=max_size,
batch_size=batch_size,
period=1)
self.es = jx_elasticsearch.new_instance(es.settings)
