def query(self, command): """ WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED :param command: COMMAND FOR SQLITE :return: list OF RESULTS """ if self.closed: Log.error("database is closed") signal = _allocate_lock() signal.acquire() result = Data() trace = extract_stack(1) if self.get_trace else None if self.get_trace: current_thread = Thread.current() with self.locker: for t in self.available_transactions: if t.thread is current_thread: Log.error(DOUBLE_TRANSACTION_ERROR) self.queue.add(CommandItem(command, result, signal, trace, None)) signal.acquire() if result.exception: Log.error("Problem with Sqlite call", cause=result.exception) return result
def __init__(self, db, parent=None): self.db = db self.locker = Lock("transaction " + text_type(id(self)) + " todo lock") self.todo = [] self.complete = 0 self.end_of_life = False self.exception = None self.parent = parent self.thread = parent.thread if parent else Thread.current()
def loop(is_ready, please_stop): with locker: while not got_signal: locker.wait(till=Till(seconds=0.01)) is_ready.go() Log.note("{{thread}} is ready", thread=Thread.current().name) Log.note("outside loop") locker.wait() Log.note("thread is expected to get here")
def time_delta_pusher(please_stop, appender, queue, interval): """ appender - THE FUNCTION THAT ACCEPTS A STRING queue - FILLED WITH LOG ENTRIES {"template":template, "params":params} TO WRITE interval - timedelta USE IN A THREAD TO BATCH LOGS BY TIME INTERVAL """ next_run = time() + interval while not please_stop: Thread.current().cprofiler.disable() (Till(till=next_run) | please_stop).wait() Thread.current().cprofiler.enable() next_run = time() + interval logs = queue.pop_all() if not logs: continue lines = [] for log in logs: try: if log is THREAD_STOP: please_stop.go() next_run = time() else: expanded = expand_template(log.get("template"), log.get("params")) lines.append(expanded) except Exception as e: location = log.get('params', {}).get('location', {}) Log.warning("Trouble formatting log from {{location}}", location=location, cause=e) # SWALLOW ERROR, GOT TO KEEP RUNNING try: appender(u"\n".join(lines) + u"\n") except Exception as e: sys.stderr.write( str("Trouble with appender: ") + str(e.__class__.__name__) + str("\n"))
def transaction(self): thread = Thread.current() parent = None with self.locker: for t in self.available_transactions: if t.thread is thread: parent = t output = Transaction(self, parent=parent, thread=thread) self.available_transactions.append(output) return output
def note(cls, template, default_params={}, stack_depth=0, log_context=None, **more_params): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if not isinstance(template, text_type): Log.error("Log.note was expecting a unicode template") if len(template) > 10000: template = template[:10000] params = dict(unwrap(default_params), **more_params) log_params = set_default( { "template": template, "params": params, "timestamp": datetime.utcnow(), "machine": machine_metadata }, log_context, {"context": exceptions.NOTE}) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if cls.trace: log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace( "{{", "{{params.") f = sys._getframe(stack_depth + 1) log_params.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename.split(os.sep)[-1]), "method": text_type(f.f_code.co_name) } thread = _Thread.current() log_params.thread = {"name": thread.name, "id": thread.id} else: log_template = "{{timestamp|datetime}} - " + template.replace( "{{", "{{params.") cls.main_log.write(log_template, log_params)
def note( cls, template, default_params={}, stack_depth=0, log_context=None, **more_params ): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if not isinstance(template, text_type): Log.error("Log.note was expecting a unicode template") if len(template) > 10000: template = template[:10000] params = dict(unwrap(default_params), **more_params) log_params = set_default({ "template": template, "params": params, "timestamp": datetime.utcnow(), "machine": machine_metadata }, log_context, {"context": exceptions.NOTE}) if not template.startswith("\n") and template.find("\n") > -1: template = "\n" + template if cls.trace: log_template = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" ({{location.method}}) - " + template.replace("{{", "{{params.") f = sys._getframe(stack_depth + 1) log_params.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename.split(os.sep)[-1]), "method": text_type(f.f_code.co_name) } thread = _Thread.current() log_params.thread = {"name": thread.name, "id": thread.id} else: log_template = "{{timestamp|datetime}} - " + template.replace("{{", "{{params.") cls.main_log.write(log_template, log_params)
def one_request(request, please_stop): and_op = request.where['and'] files = [] for a in and_op: if a['in'].path: files = a['in'].path elif a.eq.path: files = [a.eq.path] with Timer("Make TUID request from {{timestamp|datetime}}", {"timestamp": request.meta.request_time}): try: result = http.post_json("http://localhost:5000/tuid", json=request, timeout=30) if result is None or len(result.data) != len(files): Log.note("incomplete response for {{thread}}", thread=Thread.current().name) except Exception as e: Log.warning("Request failure", cause=e)
def one_request(request, please_stop): and_op = request.where['and'] files = [] for a in and_op: if a['in'].path: files = a['in'].path elif a.eq.path: files = [a.eq.path] with Timer("Make TUID request from {{timestamp|datetime}}", {"timestamp": request.meta.request_time}): try: result = http.post_json( "http://localhost:5000/tuid", json=request, timeout=30 ) if result is None or len(result.data) != len(files): Log.note("incomplete response for {{thread}}", thread=Thread.current().name) except Exception as e: Log.warning("Request failure", cause=e)
def time_delta_pusher(please_stop, appender, queue, interval): """ appender - THE FUNCTION THAT ACCEPTS A STRING queue - FILLED WITH LOG ENTRIES {"template":template, "params":params} TO WRITE interval - timedelta USE IN A THREAD TO BATCH LOGS BY TIME INTERVAL """ next_run = time() + interval while not please_stop: profiler = Thread.current().cprofiler profiler.disable() (Till(till=next_run) | please_stop).wait() profiler.enable() next_run = time() + interval logs = queue.pop_all() if not logs: continue lines = [] for log in logs: try: if log is THREAD_STOP: please_stop.go() next_run = time() else: expanded = expand_template(log.get("template"), log.get("params")) lines.append(expanded) except Exception as e: location = log.get('params', {}).get('location', {}) Log.warning("Trouble formatting log from {{location}}", location=location, cause=e) # SWALLOW ERROR, GOT TO KEEP RUNNING try: appender(u"\n".join(lines) + u"\n") except Exception as e: sys.stderr.write(str("Trouble with appender: ") + str(e.__class__.__name__) + str("\n"))
def _annotate( cls, item, timestamp, stack_depth ): """ :param itemt: A LogItemTHE TYPE OF MESSAGE :param stack_depth: FOR TRACKING WHAT LINE THIS CAME FROM :return: """ item.timestamp = timestamp item.machine = machine_metadata item.template = strings.limit(item.template, 10000) item.format = strings.limit(item.format, 10000) if item.format == None: format = text_type(item) else: format = item.format.replace("{{", "{{params.") if not format.startswith(CR) and format.find(CR) > -1: format = CR + format if cls.trace: log_format = item.format = "{{machine.name}} (pid {{machine.pid}}) - {{timestamp|datetime}} - {{thread.name}} - \"{{location.file}}:{{location.line}}\" - ({{location.method}}) - " + format f = sys._getframe(stack_depth + 1) item.location = { "line": f.f_lineno, "file": text_type(f.f_code.co_filename), "method": text_type(f.f_code.co_name) } thread = _Thread.current() item.thread = {"name": thread.name, "id": thread.id} else: log_format = item.format = "{{timestamp|datetime}} - " + format cls.main_log.write(log_format, item.__data__())