def _load_functions(self):
global _load_extension_warning_sent
library_loc = File.new_instance(sys.modules[__name__].__file__,
"../..")
full_path = File.new_instance(
library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath
try:
trace = extract_stack(0)[0]
if self.upgrade:
if os.name == 'nt':
file = File.new_instance(
trace["file"],
"../../vendor/sqlite/libsqlitefunctions.so")
else:
file = File.new_instance(
trace["file"],
"../../vendor/sqlite/libsqlitefunctions")
full_path = file.abspath
self.db.enable_load_extension(True)
self.db.execute(SQL_SELECT + "load_extension" +
sql_iso(quote_value(full_path)))
except Exception as e:
if not _load_extension_warning_sent:
_load_extension_warning_sent = True
Log.warning(
"Could not load {{file}}, doing without. (no SQRT for you!)",
file=full_path,
cause=e)
def query(self, command):
"""
WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED
:param command: COMMAND FOR SQLITE
:return: list OF RESULTS
"""
if self.closed:
Log.error("database is closed")
signal = _allocate_lock()
signal.acquire()
result = Data()
trace = extract_stack(1) if self.get_trace else None
if self.get_trace:
current_thread = Thread.current()
with self.locker:
for t in self.available_transactions:
if t.thread is current_thread:
Log.error(DOUBLE_TRANSACTION_ERROR)
self.queue.add(CommandItem(command, result, signal, trace, None))
signal.acquire()
if result.exception:
Log.error("Problem with Sqlite call", cause=result.exception)
return result
def execute_tests(self, subtest, tjson=False):
subtest = wrap(subtest)
subtest.name = extract_stack()[1]['method']
if subtest.disable:
return
self.fill_container(subtest, tjson=tjson)
self.send_queries(subtest)
def columns(self):
"""
RETURN ALL COLUMNS FROM ORIGIN OF FACT TABLE
"""
if any("verify_no_private_attachments" in t['method']
for t in extract_stack()):
pass
return self.namespace.get_columns(literal_field(self.alias))
def error(
cls,
template, # human readable template
default_params={}, # parameters for template
cause=None, # pausible cause
stack_depth=0,
**more_params):
"""
raise an exception with a trace for the cause too
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
if not is_text(template):
sys.stderr.write(str("Log.error was expecting a unicode template"))
Log.error("Log.error was expecting a unicode template")
if default_params and isinstance(
listwrap(default_params)[0], BaseException):
cause = default_params
default_params = {}
params = Data(dict(default_params, **more_params))
add_to_trace = False
if cause == None:
causes = None
elif is_list(cause):
causes = []
for c in listwrap(
cause
): # CAN NOT USE LIST-COMPREHENSION IN PYTHON3 (EXTRA STACK DEPTH FROM THE IN-LINED GENERATOR)
causes.append(Except.wrap(c, stack_depth=1))
causes = FlatList(causes)
elif isinstance(cause, BaseException):
causes = Except.wrap(cause, stack_depth=1)
else:
causes = None
Log.error("can only accept Exception, or list of exceptions")
trace = exceptions.extract_stack(stack_depth + 1)
if add_to_trace:
cause[0].trace.extend(trace[1:])
e = Except(context=exceptions.ERROR,
template=template,
params=params,
cause=causes,
trace=trace)
raise_from_none(e)
def query(self, query):
if self.db.closed:
Log.error("database is closed")
signal = _allocate_lock()
signal.acquire()
result = Data()
trace = extract_stack(1) if self.db.get_trace else None
self.db.queue.add(CommandItem(query, result, signal, trace, self))
signal.acquire()
if result.exception:
Log.error("Problem with Sqlite call", cause=result.exception)
return result
def test_cardinality(self):
pre_test = {
"data": [{
"a": "b"
}, {
"a": "c"
}],
"query": {
"from": TEST_TABLE
}, # DUMMY QUERY
"expecting_list": {
"meta": {
"format": "list"
},
"data": [{
"a": "b"
}, {
"a": "c"
}]
}
}
self.utils.execute_tests(pre_test)
test = {
"query": {
"from": "meta.columns",
"select": "cardinality",
"where": {
"and": [{
"eq": {
"table": TEST_TABLE
}
}, {
"eq": {
"name": "a"
}
}]
}
},
"expecting_list": {
"meta": {
"format": "value"
},
"data": [2]
}
}
subtest = wrap(pre_test)
subtest.name = text_type(extract_stack()[1]['method'])
self.utils.send_queries(subtest)
def execute(self, command):
"""
COMMANDS WILL BE EXECUTED IN THE ORDER THEY ARE GIVEN
BUT CAN INTERLEAVE WITH OTHER TREAD COMMANDS
:param command: COMMAND FOR SQLITE
:return: None
"""
if DEBUG: # EXECUTE IMMEDIATELY FOR BETTER STACK TRACE
return self.query(command)
if self.get_trace:
trace = extract_stack(1)
else:
trace = None
self.queue.add((command, None, None, trace))
def execute(self, command):
"""
COMMANDS WILL BE EXECUTED IN THE ORDER THEY ARE GIVEN
BUT CAN INTERLEAVE WITH OTHER TREAD COMMANDS
:param command: COMMAND FOR SQLITE
:return: None
"""
if DEBUG: # EXECUTE IMMEDIATELY FOR BETTER STACK TRACE
return self.query(command)
if self.get_trace:
trace = extract_stack(1)
else:
trace = None
self.queue.add((command, None, None, trace))
def execute(self, command):
"""
COMMANDS WILL BE EXECUTED IN THE ORDER THEY ARE GIVEN
BUT CAN INTERLEAVE WITH OTHER TREAD COMMANDS
:param command: COMMAND FOR SQLITE
:return: Signal FOR IF YOU WANT TO BE NOTIFIED WHEN DONE
"""
if DEBUG_EXECUTE: # EXECUTE IMMEDIATELY FOR BETTER STACK TRACE
self.query(command)
return DONE
if self.get_trace:
trace = extract_stack(1)
else:
trace = None
is_done = Signal()
self.queue.add((command, None, is_done, trace))
return is_done
def _worker(self, please_stop):
global _load_extension_warning_sent
if DEBUG:
Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version)
if Sqlite.canonical:
self.db = Sqlite.canonical
else:
self.db = sqlite3.connect(coalesce(self.filename, ':memory:'))
library_loc = File.new_instance(sys.modules[__name__].__file__, "../..")
full_path = File.new_instance(library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath
try:
trace = extract_stack(0)[0]
file = File.new_instance(trace["file"], "../../vendor/sqlite/libsqlitefunctions.so")
full_path = file.abspath
self.db.enable_load_extension(True)
self.db.execute("SELECT load_extension(" + self.quote_value(full_path) + ")")
except Exception, e:
if not _load_extension_warning_sent:
_load_extension_warning_sent = True
Log.warning("Could not load {{file}}}, doing without. (no SQRT for you!)", file=full_path, cause=e)
def warning(cls,
template,
default_params={},
cause=None,
stack_depth=0,
log_context=None,
**more_params):
"""
:param template: *string* human readable string with placeholders for parameters
:param default_params: *dict* parameters to fill in template
:param cause: *Exception* for chaining
:param stack_depth: *int* how many calls you want popped off the stack to report the *true* caller
:param log_context: *dict* extra key:value pairs for your convenience
:param more_params: *any more parameters (which will overwrite default_params)
:return:
"""
timestamp = datetime.utcnow()
if not is_text(template):
Log.error("Log.warning was expecting a unicode template")
if isinstance(default_params, BaseException):
cause = default_params
default_params = {}
if "values" in more_params.keys():
Log.error("Can not handle a logging parameter by name `values`")
params = Data(dict(default_params, **more_params))
cause = unwraplist([Except.wrap(c) for c in listwrap(cause)])
trace = exceptions.extract_stack(stack_depth + 1)
e = Except(exceptions.WARNING,
template=template,
params=params,
cause=cause,
trace=trace)
Log._annotate(e, timestamp, stack_depth + 1)
def test_2edge_and_sort(self):
test = {
"data": [{
"a": "c",
"b": 0,
"value": 1
}, {
"a": "c",
"b": 0,
"value": 3
}, {
"a": "c",
"b": 1,
"value": 4
}, {
"a": "c",
"b": 1,
"value": 6
}, {
"a": "a",
"b": 1,
"value": 7
}, {
"a": "a",
"value": 20
}, {
"b": 1,
"value": 21
}, {
"value": 22
}, {
"a": "a",
"b": 0,
"value": 8
}, {
"a": "a",
"b": 0,
"value": 9
}, {
"a": "a",
"b": 1,
"value": 10
}, {
"a": "a",
"b": 1,
"value": 11
}],
"query": {
"from": TEST_TABLE,
"edges": ["a", "b"],
"sort": [{
"a": "desc"
}, {
"b": "desc"
}]
},
"expecting_list": {
"meta": {
"format": "list"
},
"data": [{
"a": "c",
"b": 1,
"count": 2
}, {
"a": "c",
"b": 0,
"count": 2
}, {
"a": "a",
"b": 1,
"count": 3
}, {
"a": "a",
"b": 0,
"count": 2
}, {
"a": "a",
"count": 1
}, {
"b": 1,
"count": 1
}, {
"count": 1
}]
},
"expecting_table": {
"meta": {
"format": "table"
},
"header": ["a", "b", "count"],
"data": [["c", 1, 2], ["c", 0, 2], ["a", 1, 3], ["a", 0, 2],
["a", NULL, 1], [NULL, 1, 1], [NULL, NULL, 1]]
},
"expecting_cube": {
"meta": {
"format": "cube"
},
"edges": [{
"name": "b",
"domain": {
"type": "set",
"partitions": [{
"value": 0
}, {
"value": 1
}]
}
}, {
"name": "a",
"domain": {
"type": "set",
"partitions": [{
"value": "a"
}, {
"value": "c"
}]
}
}],
"data": {
"count": [[2, 2, 0], [3, 2, 1], [1, 0, 1]]
}
}
}
subtest = wrap(test)
subtest.name = extract_stack()[0]['method']
self.utils.fill_container(test)
test = wrap(test)
self.utils.send_queries({
"query": test.query,
"expecting_list": test.expecting_list
})
self.utils.send_queries({
"query": test.query,
"expecting_table": test.expecting_table
})
try:
self.utils.send_queries({
"query": test.query,
"expecting_cube": test.expecting_cube
})
Log.error("expecting error regarding sorting edges")
except Exception, e:
pass
def __exit__(self, exc_type, exc_val, exc_tb):
Thread.run("delete file " + self.name, delete_daemon, file=self, caller_stack=extract_stack(1))
def __exit__(self, exc_type, exc_val, exc_tb):
Thread.run("delete file " + self.name,
delete_daemon,
file=self,
caller_stack=extract_stack(1))
def execute_tests(self, subtest, typed=True, places=6):
subtest = wrap(subtest)
subtest.name = text_type(extract_stack()[1]['method'])
self.fill_container(subtest, typed=typed)
self.send_queries(subtest, places=places)
def execute(self, command):
trace = extract_stack(1) if self.db.get_trace else None
with self.locker:
self.todo.append(CommandItem(command, None, None, trace, self))
def execute_es_tests(self, subtest, tjson=False, places=6):
subtest = wrap(subtest)
subtest.name = extract_stack()[1]['method']
self.fill_container(subtest, tjson=tjson)
self.send_queries(subtest, places=places)
def _worker(self, please_stop):
global _load_extension_warning_sent
if DEBUG:
Log.note("Sqlite version {{version}}",
version=sqlite3.sqlite_version)
if Sqlite.canonical:
self.db = Sqlite.canonical
else:
self.db = sqlite3.connect(coalesce(self.filename, ':memory:'))
library_loc = File.new_instance(sys.modules[__name__].__file__,
"../..")
full_path = File.new_instance(
library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath
try:
trace = extract_stack(0)[0]
if self.upgrade:
if os.name == 'nt':
file = File.new_instance(
trace["file"],
"../../vendor/sqlite/libsqlitefunctions.so")
else:
file = File.new_instance(
trace["file"],
"../../vendor/sqlite/libsqlitefunctions")
full_path = file.abspath
self.db.enable_load_extension(True)
self.db.execute("SELECT load_extension(" +
self.quote_value(full_path) + ")")
except Exception as e:
if not _load_extension_warning_sent:
_load_extension_warning_sent = True
Log.warning(
"Could not load {{file}}}, doing without. (no SQRT for you!)",
file=full_path,
cause=e)
try:
while not please_stop:
command, result, signal, trace = self.queue.pop(
till=please_stop)
if DEBUG_INSERT and command.strip().lower().startswith(
"insert"):
Log.note("Running command\n{{command|indent}}",
command=command)
if DEBUG and not command.strip().lower().startswith("insert"):
Log.note("Running command\n{{command|indent}}",
command=command)
with Timer("Run command", debug=DEBUG):
if signal is not None:
try:
curr = self.db.execute(command)
self.db.commit()
result.meta.format = "table"
result.header = [d[0] for d in curr.description
] if curr.description else None
result.data = curr.fetchall()
if DEBUG and result.data:
text = convert.table2csv(list(result.data))
Log.note("Result:\n{{data}}", data=text)
except Exception as e:
e = Except.wrap(e)
result.exception = Except(
ERROR,
"Problem with\n{{command|indent}}",
command=command,
cause=e)
finally:
signal.go()
else:
try:
self.db.execute(command)
self.db.commit()
except Exception as e:
e = Except.wrap(e)
e.cause = Except(type=ERROR,
template="Bad call to Sqlite",
trace=trace)
Log.warning("Failure to execute", cause=e)
except Exception as e:
if not please_stop:
Log.error("Problem with sql thread", e)
finally:
if DEBUG:
Log.note("Database is closed")
self.db.commit()
self.db.close()
def execute(self, command):
if self.end_of_life:
Log.error("Transaction is dead")
trace = extract_stack(1) if self.db.get_trace else None
with self.locker:
self.todo.append(CommandItem(command, None, None, trace, self))
def _worker(self, please_stop):
global _load_extension_warning_sent
if DEBUG:
Log.note("Sqlite version {{version}}", version=sqlite3.sqlite_version)
if Sqlite.canonical:
self.db = Sqlite.canonical
else:
self.db = sqlite3.connect(coalesce(self.filename, ':memory:'))
library_loc = File.new_instance(sys.modules[__name__].__file__, "../..")
full_path = File.new_instance(library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath
try:
trace = extract_stack(0)[0]
if self.upgrade:
if os.name == 'nt':
file = File.new_instance(trace["file"], "../../vendor/sqlite/libsqlitefunctions.so")
else:
file = File.new_instance(trace["file"], "../../vendor/sqlite/libsqlitefunctions")
full_path = file.abspath
self.db.enable_load_extension(True)
self.db.execute("SELECT load_extension(" + self.quote_value(full_path) + ")")
except Exception as e:
if not _load_extension_warning_sent:
_load_extension_warning_sent = True
Log.warning("Could not load {{file}}}, doing without. (no SQRT for you!)", file=full_path, cause=e)
try:
while not please_stop:
command, result, signal, trace = self.queue.pop(till=please_stop)
if DEBUG_INSERT and command.strip().lower().startswith("insert"):
Log.note("Running command\n{{command|indent}}", command=command)
if DEBUG and not command.strip().lower().startswith("insert"):
Log.note("Running command\n{{command|indent}}", command=command)
with Timer("Run command", debug=DEBUG):
if signal is not None:
try:
curr = self.db.execute(command)
self.db.commit()
result.meta.format = "table"
result.header = [d[0] for d in curr.description] if curr.description else None
result.data = curr.fetchall()
if DEBUG and result.data:
text = convert.table2csv(list(result.data))
Log.note("Result:\n{{data}}", data=text)
except Exception as e:
e = Except.wrap(e)
result.exception = Except(ERROR, "Problem with\n{{command|indent}}", command=command, cause=e)
finally:
signal.go()
else:
try:
self.db.execute(command)
self.db.commit()
except Exception as e:
e = Except.wrap(e)
e.cause = Except(
type=ERROR,
template="Bad call to Sqlite",
trace=trace
)
Log.warning("Failure to execute", cause=e)
except Exception as e:
if not please_stop:
Log.error("Problem with sql thread", e)
finally:
if DEBUG:
Log.note("Database is closed")
self.db.commit()
self.db.close()