def _load_functions(self): global _load_extension_warning_sent library_loc = File.new_instance(sys.modules[__name__].__file__, "../..") full_path = File.new_instance( library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath try: trace = extract_stack(0)[0] if self.upgrade: if os.name == 'nt': file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions.so") else: file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions") full_path = file.abspath self.db.enable_load_extension(True) self.db.execute(SQL_SELECT + "load_extension" + sql_iso(quote_value(full_path))) except Exception as e: if not _load_extension_warning_sent: _load_extension_warning_sent = True Log.warning( "Could not load {{file}}, doing without. (no SQRT for you!)", file=full_path, cause=e)
def query(self, command): """ WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED :param command: COMMAND FOR SQLITE :return: list OF RESULTS """ if self.closed: Log.error("database is closed") signal = _allocate_lock() signal.acquire() result = Data() trace = extract_stack(1) if self.get_trace else None if self.get_trace: current_thread = Thread.current() with self.locker: for t in self.available_transactions: if t.thread is current_thread: Log.error(DOUBLE_TRANSACTION_ERROR) self.queue.add(CommandItem(command, result, signal, trace, None)) signal.acquire() if result.exception: Log.error("Problem with Sqlite call", cause=result.exception) return result
def execute_tests(self, subtest, tjson=False): subtest = wrap(subtest) subtest.name = extract_stack()[1]['method'] if subtest.disable: return self.fill_container(subtest, tjson=tjson) self.send_queries(subtest)
def columns(self): """ RETURN ALL COLUMNS FROM ORIGIN OF FACT TABLE """ if any("verify_no_private_attachments" in t['method'] for t in extract_stack()): pass return self.namespace.get_columns(literal_field(self.alias))
def error( cls, template, # human readable template default_params={}, # parameters for template cause=None, # pausible cause stack_depth=0, **more_params): """ raise an exception with a trace for the cause too :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ if not is_text(template): sys.stderr.write(str("Log.error was expecting a unicode template")) Log.error("Log.error was expecting a unicode template") if default_params and isinstance( listwrap(default_params)[0], BaseException): cause = default_params default_params = {} params = Data(dict(default_params, **more_params)) add_to_trace = False if cause == None: causes = None elif is_list(cause): causes = [] for c in listwrap( cause ): # CAN NOT USE LIST-COMPREHENSION IN PYTHON3 (EXTRA STACK DEPTH FROM THE IN-LINED GENERATOR) causes.append(Except.wrap(c, stack_depth=1)) causes = FlatList(causes) elif isinstance(cause, BaseException): causes = Except.wrap(cause, stack_depth=1) else: causes = None Log.error("can only accept Exception, or list of exceptions") trace = exceptions.extract_stack(stack_depth + 1) if add_to_trace: cause[0].trace.extend(trace[1:]) e = Except(context=exceptions.ERROR, template=template, params=params, cause=causes, trace=trace) raise_from_none(e)
def query(self, query): if self.db.closed: Log.error("database is closed") signal = _allocate_lock() signal.acquire() result = Data() trace = extract_stack(1) if self.db.get_trace else None self.db.queue.add(CommandItem(query, result, signal, trace, self)) signal.acquire() if result.exception: Log.error("Problem with Sqlite call", cause=result.exception) return result
def test_cardinality(self): pre_test = { "data": [{ "a": "b" }, { "a": "c" }], "query": { "from": TEST_TABLE }, # DUMMY QUERY "expecting_list": { "meta": { "format": "list" }, "data": [{ "a": "b" }, { "a": "c" }] } } self.utils.execute_tests(pre_test) test = { "query": { "from": "meta.columns", "select": "cardinality", "where": { "and": [{ "eq": { "table": TEST_TABLE } }, { "eq": { "name": "a" } }] } }, "expecting_list": { "meta": { "format": "value" }, "data": [2] } } subtest = wrap(pre_test) subtest.name = text_type(extract_stack()[1]['method']) self.utils.send_queries(subtest)
def execute(self, command): """ COMMANDS WILL BE EXECUTED IN THE ORDER THEY ARE GIVEN BUT CAN INTERLEAVE WITH OTHER TREAD COMMANDS :param command: COMMAND FOR SQLITE :return: None """ if DEBUG: # EXECUTE IMMEDIATELY FOR BETTER STACK TRACE return self.query(command) if self.get_trace: trace = extract_stack(1) else: trace = None self.queue.add((command, None, None, trace))
def execute(self, command): """ COMMANDS WILL BE EXECUTED IN THE ORDER THEY ARE GIVEN BUT CAN INTERLEAVE WITH OTHER TREAD COMMANDS :param command: COMMAND FOR SQLITE :return: Signal FOR IF YOU WANT TO BE NOTIFIED WHEN DONE """ if DEBUG_EXECUTE: # EXECUTE IMMEDIATELY FOR BETTER STACK TRACE self.query(command) return DONE if self.get_trace: trace = extract_stack(1) else: trace = None is_done = Signal() self.queue.add((command, None, is_done, trace)) return is_done
def _worker(self, please_stop): global _load_extension_warning_sent if DEBUG: Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version) if Sqlite.canonical: self.db = Sqlite.canonical else: self.db = sqlite3.connect(coalesce(self.filename, ':memory:')) library_loc = File.new_instance(sys.modules[__name__].__file__, "../..") full_path = File.new_instance(library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath try: trace = extract_stack(0)[0] file = File.new_instance(trace["file"], "../../vendor/sqlite/libsqlitefunctions.so") full_path = file.abspath self.db.enable_load_extension(True) self.db.execute("SELECT load_extension(" + self.quote_value(full_path) + ")") except Exception, e: if not _load_extension_warning_sent: _load_extension_warning_sent = True Log.warning("Could not load {{file}}}, doing without. (no SQRT for you!)", file=full_path, cause=e)
def warning(cls, template, default_params={}, cause=None, stack_depth=0, log_context=None, **more_params): """ :param template: *string* human readable string with placeholders for parameters :param default_params: *dict* parameters to fill in template :param cause: *Exception* for chaining :param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller :param log_context: *dict* extra key:value pairs for your convenience :param more_params: *any more parameters (which will overwrite default_params) :return: """ timestamp = datetime.utcnow() if not is_text(template): Log.error("Log.warning was expecting a unicode template") if isinstance(default_params, BaseException): cause = default_params default_params = {} if "values" in more_params.keys(): Log.error("Can not handle a logging parameter by name `values`") params = Data(dict(default_params, **more_params)) cause = unwraplist([Except.wrap(c) for c in listwrap(cause)]) trace = exceptions.extract_stack(stack_depth + 1) e = Except(exceptions.WARNING, template=template, params=params, cause=cause, trace=trace) Log._annotate(e, timestamp, stack_depth + 1)
def test_2edge_and_sort(self): test = { "data": [{ "a": "c", "b": 0, "value": 1 }, { "a": "c", "b": 0, "value": 3 }, { "a": "c", "b": 1, "value": 4 }, { "a": "c", "b": 1, "value": 6 }, { "a": "a", "b": 1, "value": 7 }, { "a": "a", "value": 20 }, { "b": 1, "value": 21 }, { "value": 22 }, { "a": "a", "b": 0, "value": 8 }, { "a": "a", "b": 0, "value": 9 }, { "a": "a", "b": 1, "value": 10 }, { "a": "a", "b": 1, "value": 11 }], "query": { "from": TEST_TABLE, "edges": ["a", "b"], "sort": [{ "a": "desc" }, { "b": "desc" }] }, "expecting_list": { "meta": { "format": "list" }, "data": [{ "a": "c", "b": 1, "count": 2 }, { "a": "c", "b": 0, "count": 2 }, { "a": "a", "b": 1, "count": 3 }, { "a": "a", "b": 0, "count": 2 }, { "a": "a", "count": 1 }, { "b": 1, "count": 1 }, { "count": 1 }] }, "expecting_table": { "meta": { "format": "table" }, "header": ["a", "b", "count"], "data": [["c", 1, 2], ["c", 0, 2], ["a", 1, 3], ["a", 0, 2], ["a", NULL, 1], [NULL, 1, 1], [NULL, NULL, 1]] }, "expecting_cube": { "meta": { "format": "cube" }, "edges": [{ "name": "b", "domain": { "type": "set", "partitions": [{ "value": 0 }, { "value": 1 }] } }, { "name": "a", "domain": { "type": "set", "partitions": [{ "value": "a" }, { "value": "c" }] } }], "data": { "count": [[2, 2, 0], [3, 2, 1], [1, 0, 1]] } } } subtest = wrap(test) subtest.name = extract_stack()[0]['method'] self.utils.fill_container(test) test = wrap(test) self.utils.send_queries({ "query": test.query, "expecting_list": test.expecting_list }) self.utils.send_queries({ "query": test.query, "expecting_table": test.expecting_table }) try: self.utils.send_queries({ "query": test.query, "expecting_cube": test.expecting_cube }) Log.error("expecting error regarding sorting edges") except Exception, e: pass
def __exit__(self, exc_type, exc_val, exc_tb): Thread.run("delete file " + self.name, delete_daemon, file=self, caller_stack=extract_stack(1))
def execute_tests(self, subtest, typed=True, places=6): subtest = wrap(subtest) subtest.name = text_type(extract_stack()[1]['method']) self.fill_container(subtest, typed=typed) self.send_queries(subtest, places=places)
def execute(self, command): trace = extract_stack(1) if self.db.get_trace else None with self.locker: self.todo.append(CommandItem(command, None, None, trace, self))
def execute_es_tests(self, subtest, tjson=False, places=6): subtest = wrap(subtest) subtest.name = extract_stack()[1]['method'] self.fill_container(subtest, tjson=tjson) self.send_queries(subtest, places=places)
def _worker(self, please_stop): global _load_extension_warning_sent if DEBUG: Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version) if Sqlite.canonical: self.db = Sqlite.canonical else: self.db = sqlite3.connect(coalesce(self.filename, ':memory:')) library_loc = File.new_instance(sys.modules[__name__].__file__, "../..") full_path = File.new_instance( library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath try: trace = extract_stack(0)[0] if self.upgrade: if os.name == 'nt': file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions.so") else: file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions") full_path = file.abspath self.db.enable_load_extension(True) self.db.execute("SELECT load_extension(" + self.quote_value(full_path) + ")") except Exception as e: if not _load_extension_warning_sent: _load_extension_warning_sent = True Log.warning( "Could not load {{file}}}, doing without. (no SQRT for you!)", file=full_path, cause=e) try: while not please_stop: command, result, signal, trace = self.queue.pop( till=please_stop) if DEBUG_INSERT and command.strip().lower().startswith( "insert"): Log.note("Running command\n{{command|indent}}", command=command) if DEBUG and not command.strip().lower().startswith("insert"): Log.note("Running command\n{{command|indent}}", command=command) with Timer("Run command", debug=DEBUG): if signal is not None: try: curr = self.db.execute(command) self.db.commit() result.meta.format = "table" result.header = [d[0] for d in curr.description ] if curr.description else None result.data = curr.fetchall() if DEBUG and result.data: text = convert.table2csv(list(result.data)) Log.note("Result:\n{{data}}", data=text) except Exception as e: e = Except.wrap(e) result.exception = Except( ERROR, "Problem with\n{{command|indent}}", command=command, cause=e) finally: signal.go() else: try: self.db.execute(command) self.db.commit() except Exception as e: e = Except.wrap(e) e.cause = Except(type=ERROR, template="Bad call to Sqlite", trace=trace) Log.warning("Failure to execute", cause=e) except Exception as e: if not please_stop: Log.error("Problem with sql thread", e) finally: if DEBUG: Log.note("Database is closed") self.db.commit() self.db.close()
def execute(self, command): if self.end_of_life: Log.error("Transaction is dead") trace = extract_stack(1) if self.db.get_trace else None with self.locker: self.todo.append(CommandItem(command, None, None, trace, self))
def _worker(self, please_stop): global _load_extension_warning_sent if DEBUG: Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version) if Sqlite.canonical: self.db = Sqlite.canonical else: self.db = sqlite3.connect(coalesce(self.filename, ':memory:')) library_loc = File.new_instance(sys.modules[__name__].__file__, "../..") full_path = File.new_instance(library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath try: trace = extract_stack(0)[0] if self.upgrade: if os.name == 'nt': file = File.new_instance(trace["file"], "../../vendor/sqlite/libsqlitefunctions.so") else: file = File.new_instance(trace["file"], "../../vendor/sqlite/libsqlitefunctions") full_path = file.abspath self.db.enable_load_extension(True) self.db.execute("SELECT load_extension(" + self.quote_value(full_path) + ")") except Exception as e: if not _load_extension_warning_sent: _load_extension_warning_sent = True Log.warning("Could not load {{file}}}, doing without. (no SQRT for you!)", file=full_path, cause=e) try: while not please_stop: command, result, signal, trace = self.queue.pop(till=please_stop) if DEBUG_INSERT and command.strip().lower().startswith("insert"): Log.note("Running command\n{{command|indent}}", command=command) if DEBUG and not command.strip().lower().startswith("insert"): Log.note("Running command\n{{command|indent}}", command=command) with Timer("Run command", debug=DEBUG): if signal is not None: try: curr = self.db.execute(command) self.db.commit() result.meta.format = "table" result.header = [d[0] for d in curr.description] if curr.description else None result.data = curr.fetchall() if DEBUG and result.data: text = convert.table2csv(list(result.data)) Log.note("Result:\n{{data}}", data=text) except Exception as e: e = Except.wrap(e) result.exception = Except(ERROR, "Problem with\n{{command|indent}}", command=command, cause=e) finally: signal.go() else: try: self.db.execute(command) self.db.commit() except Exception as e: e = Except.wrap(e) e.cause = Except( type=ERROR, template="Bad call to Sqlite", trace=trace ) Log.warning("Failure to execute", cause=e) except Exception as e: if not please_stop: Log.error("Problem with sql thread", e) finally: if DEBUG: Log.note("Database is closed") self.db.commit() self.db.close()