class Log_usingElasticSearch(BaseLog): @use_settings def __init__(self, host, index, type="log", max_size=1000, batch_size=100, settings=None): """ settings ARE FOR THE ELASTICSEARCH INDEX """ self.es = Cluster(settings).get_or_create_index( schema=convert.json2value(convert.value2json(SCHEMA), paths=True), limit_replicas=True, settings=settings ) self.queue = self.es.threaded_queue(max_size=max_size, batch_size=batch_size) def write(self, template, params): try: if params.get("template"): # DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE self.queue.add({"value": params}) else: if len(template) > 2000: template = template[:1997] + "..." self.queue.add({"value": {"template": template, "params": params}}) return self except Exception, e: raise e # OH NO!
class TextLog_usingElasticSearch(TextLog): @use_settings def __init__(self, host, index, type="log", max_size=1000, batch_size=100, settings=None): """ settings ARE FOR THE ELASTICSEARCH INDEX """ self.es = Cluster(settings).get_or_create_index( schema=convert.json2value(convert.value2json(SCHEMA), leaves=True), limit_replicas=True, tjson=False, settings=settings ) self.es.add_alias("debug") self.queue = self.es.threaded_queue(max_size=max_size, batch_size=batch_size) def write(self, template, params): if params.get("template"): # DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE self.queue.add({"value": params}) else: template = strings.limit(template, 2000) self.queue.add({"value": {"template": template, "params": params}}, timeout=3*MINUTE) return self def stop(self): try: self.queue.add(Thread.STOP) # BE PATIENT, LET REST OF MESSAGE BE SENT except Exception, e: pass try: self.queue.close() except Exception, f: pass
class Log_usingElasticSearch(BaseLog): @use_settings def __init__(self, host, index, type="log", max_size=1000, batch_size=100, settings=None): """ settings ARE FOR THE ELASTICSEARCH INDEX """ self.es = Cluster(settings).get_or_create_index( schema=convert.json2value(convert.value2json(SCHEMA), paths=True), limit_replicas=True, settings=settings) self.queue = self.es.threaded_queue(max_size=max_size, batch_size=batch_size) def write(self, template, params): try: if params.get("template"): # DETECTED INNER TEMPLATE, ASSUME TRACE IS ON, SO DO NOT NEED THE OUTER TEMPLATE self.queue.add({"value": params}) else: if len(template) > 2000: template = template[:1997] + "..." self.queue.add( {"value": { "template": template, "params": params }}) return self except Exception, e: raise e # OH NO!
def __init__(self, host, index, type=DATA_TYPE, max_size=10, batch_size=10, kwargs=None): """ settings ARE FOR THE ELASTICSEARCH INDEX """ es = Cluster(kwargs).get_or_create_index(schema=json2value( convert.value2json(SCHEMA), leaves=True), limit_replicas=True, typed=False, kwargs=kwargs) es.add_alias(index) self.queue = es.threaded_queue(max_size=max_size, batch_size=batch_size, period=1) self.es = jx_elasticsearch.new_instance(es.settings)