class Schedule(object):
@override
def __init__(
self,
interval, # TIME INTERVAL BETWEEN RUNS
starting, # THE TIME TO START THE INTERVAL COUNT
max_runtime=MAX_RUNTIME, # LIMIT HOW LONG THE PROCESS IS ALIVE
wait_for_shutdown=WAIT_FOR_SHUTDOWN, # LIMIT PAITENCE WHEN ASKING FOR SHUTDOWN, THEN SEND KILL
process=None,
):
self.duration = Duration(interval)
self.starting = coalesce(Date(starting), Date.now())
self.max_runtime = Duration(max_runtime)
self.wait_for_shutdown = Duration(wait_for_shutdown)
# Process parameters
self.process = process
# STATE
self.last_started = None
self.last_finished = None
self.run_count = 0
self.fail_count = 0
self.current = None
self.terminator = None # SIGNAL TO KILL THE PROCESS
self.next_run = self._next_run()
self.next = Till(till=self.next_run)
self.next_run.then(self.run)
def _next_run_time(self):
"""
:return: return signal for next
"""
interval = mo_math.floor((Date.now() - self.starting) / self.duration)
next_time = self.starting + (interval * self.duration)
return next_time
def run(self):
self.last_started = Date.now()
self.run_count += 1
self.current = Process(**self.process)
self.terminator = Till(seconds=self.max_runtime.seconds)
self.terminator.then(self.killer)
self.current.service_stopped.then(self.done)
def killer(self, please_stop):
self.current.stop()
(
please_stop
| self.current.service_stopped()
| Till(seconds=self.wait_for_shutdown.seconds)
).wait()
if not self.current.service_stopped:
self.fail_count += 1
self.current.kill()
self.current.join()
def done(self):
self.last_finished = Date.now()
self.terminator.remove_go(self.killer)
self.terminator = None
self.current = None
self.next_run = self._next_run_time()
self.next = Till(till=self.next_run.unix)
self.next.then(self.run)
def status(self):
if self.current is None:
status = "never started"
elif not self.current.service_stopped:
status = "running"
elif self.current.returncode == 0:
status = "done"
else:
status = "failed " + text(self.current.returncode)
return Data(
name=self.name,
next_run=self.next_run,
last_started=self.last_started,
last_finished=self.last_finished,
status=status,
)
class Sqlite(DB):
"""
Allows multi-threaded access
Loads extension functions (like SQRT)
"""
@override
def __init__(
self,
filename=None,
db=None,
get_trace=None,
upgrade=False,
load_functions=False,
debug=False,
kwargs=None,
):
"""
:param filename: FILE TO USE FOR DATABASE
:param db: AN EXISTING sqlite3 DB YOU WOULD LIKE TO USE (INSTEAD OF USING filename)
:param get_trace: GET THE STACK TRACE AND THREAD FOR EVERY DB COMMAND (GOOD FOR DEBUGGING)
:param upgrade: REPLACE PYTHON sqlite3 DLL WITH MORE RECENT ONE, WITH MORE FUNCTIONS (NOT WORKING)
:param load_functions: LOAD EXTENDED MATH FUNCTIONS (MAY REQUIRE upgrade)
:param kwargs:
"""
global _upgraded
global _sqlite3
self.settings = kwargs
if not _upgraded:
if upgrade:
_upgrade()
_upgraded = True
import sqlite3 as _sqlite3
_ = _sqlite3
self.filename = File(filename).abspath if filename else None
if known_databases.get(self.filename):
Log.error(
"Not allowed to create more than one Sqlite instance for {{file}}",
file=self.filename,
)
self.debug = debug | DEBUG
# SETUP DATABASE
self.debug and Log.note("Sqlite version {{version}}",
version=_sqlite3.sqlite_version)
try:
if db == None:
self.db = _sqlite3.connect(
database=coalesce(self.filename, ":memory:"),
check_same_thread=False,
isolation_level=None,
)
else:
self.db = db
except Exception as e:
Log.error("could not open file {{filename}}",
filename=self.filename,
cause=e)
self.upgrade = upgrade
load_functions and self._load_functions()
self.locker = Lock()
self.available_transactions = [
] # LIST OF ALL THE TRANSACTIONS BEING MANAGED
self.queue = Queue(
"sql commands"
) # HOLD (command, result, signal, stacktrace) TUPLES
self.get_trace = coalesce(get_trace, TRACE)
self.closed = False
# WORKER VARIABLES
self.transaction_stack = [
] # THE TRANSACTION OBJECT WE HAVE PARTIALLY RUN
self.last_command_item = (
None
) # USE THIS TO HELP BLAME current_transaction FOR HANGING ON TOO LONG
self.too_long = None
self.delayed_queries = []
self.delayed_transactions = []
self.worker = Thread.run("sqlite db thread", self._worker)
self.debug and Log.note(
"Sqlite version {{version}}",
version=self.query("select sqlite_version()").data[0][0],
)
def _enhancements(self):
def regex(pattern, value):
return 1 if re.match(pattern + "$", value) else 0
con = self.db.create_function("regex", 2, regex)
class Percentile(object):
def __init__(self, percentile):
self.percentile = percentile
self.acc = []
def step(self, value):
self.acc.append(value)
def finalize(self):
return percentile(self.acc, self.percentile)
con.create_aggregate("percentile", 2, Percentile)
def transaction(self):
thread = Thread.current()
parent = None
with self.locker:
for t in self.available_transactions:
if t.thread is thread:
parent = t
output = Transaction(self, parent=parent, thread=thread)
self.available_transactions.append(output)
return output
def about(self, table_name):
"""
:param table_name: TABLE IF INTEREST
:return: SOME INFORMATION ABOUT THE TABLE
(cid, name, dtype, notnull, dfft_value, pk) tuples
"""
details = self.query("PRAGMA table_info" +
sql_iso(quote_column(table_name)))
return details.data
def query(self, command):
"""
WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED
:param command: COMMAND FOR SQLITE
:return: list OF RESULTS
"""
if self.closed:
Log.error("database is closed")
signal = _allocate_lock()
signal.acquire()
result = Data()
trace = get_stacktrace(1) if self.get_trace else None
if self.get_trace:
current_thread = Thread.current()
with self.locker:
for t in self.available_transactions:
if t.thread is current_thread:
Log.error(DOUBLE_TRANSACTION_ERROR)
self.queue.add(CommandItem(command, result, signal, trace, None))
signal.acquire()
if result.exception:
Log.error("Problem with Sqlite call", cause=result.exception)
return result
def close(self):
"""
OPTIONAL COMMIT-AND-CLOSE
IF THIS IS NOT DONE, THEN THE THREAD THAT SPAWNED THIS INSTANCE
:return:
"""
self.closed = True
signal = _allocate_lock()
signal.acquire()
self.queue.add(CommandItem(COMMIT, None, signal, None, None))
signal.acquire()
self.worker.please_stop.go()
return
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def _load_functions(self):
global _load_extension_warning_sent
library_loc = File.new_instance(sys.modules[__name__].__file__,
"../..")
full_path = File.new_instance(
library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath
try:
trace = get_stacktrace(0)[0]
if self.upgrade:
if os.name == "nt":
file = File.new_instance(
trace["file"],
"../../vendor/sqlite/libsqlitefunctions.so")
else:
file = File.new_instance(
trace["file"],
"../../vendor/sqlite/libsqlitefunctions")
full_path = file.abspath
self.db.enable_load_extension(True)
self.db.execute(
text(SQL_SELECT + "load_extension" +
sql_iso(quote_value(full_path))))
except Exception as e:
if not _load_extension_warning_sent:
_load_extension_warning_sent = True
Log.warning(
"Could not load {{file}}, doing without. (no SQRT for you!)",
file=full_path,
cause=e,
)
def create_new_functions(self):
def regexp(pattern, item):
reg = re.compile(pattern)
return reg.search(item) is not None
self.db.create_function("REGEXP", 2, regexp)
def show_transactions_blocked_warning(self):
blocker = self.last_command_item
blocked = (self.delayed_queries + self.delayed_transactions)[0]
Log.warning(
"Query on thread {{blocked_thread|json}} at\n"
"{{blocked_trace|indent}}"
"is blocked by {{blocker_thread|json}} at\n"
"{{blocker_trace|indent}}"
"this message brought to you by....",
blocker_trace=format_trace(blocker.trace),
blocked_trace=format_trace(blocked.trace),
blocker_thread=blocker.transaction.thread.name
if blocker.transaction is not None else None,
blocked_thread=blocked.transaction.thread.name
if blocked.transaction is not None else None,
)
def _close_transaction(self, command_item):
query, result, signal, trace, transaction = command_item
transaction.end_of_life = True
with self.locker:
self.available_transactions.remove(transaction)
assert transaction not in self.available_transactions
old_length = len(self.transaction_stack)
old_trans = self.transaction_stack[-1]
del self.transaction_stack[-1]
assert old_length - 1 == len(self.transaction_stack)
assert old_trans
assert old_trans not in self.transaction_stack
if not self.transaction_stack:
# NESTED TRANSACTIONS NOT ALLOWED IN sqlite3
self.debug and Log.note(FORMAT_COMMAND, command=query)
self.db.execute(query)
has_been_too_long = False
with self.locker:
if self.too_long is not None:
self.too_long, too_long = None, self.too_long
# WE ARE CHEATING HERE: WE REACH INTO THE Signal MEMBERS AND REMOVE WHAT WE ADDED TO THE INTERNAL job_queue
with too_long.lock:
has_been_too_long = bool(too_long)
too_long.job_queue = None
# PUT delayed BACK ON THE QUEUE, IN THE ORDER FOUND, BUT WITH QUERIES FIRST
if self.delayed_transactions:
for c in reversed(self.delayed_transactions):
self.queue.push(c)
del self.delayed_transactions[:]
if self.delayed_queries:
for c in reversed(self.delayed_queries):
self.queue.push(c)
del self.delayed_queries[:]
if has_been_too_long:
Log.note("Transaction blockage cleared")
def _worker(self, please_stop):
try:
# MAIN EXECUTION LOOP
while not please_stop:
command_item = self.queue.pop(till=please_stop)
if command_item is None:
break
try:
self._process_command_item(command_item)
except Exception as e:
Log.warning("worker can not execute command", cause=e)
except Exception as e:
e = Except.wrap(e)
if not please_stop:
Log.warning("Problem with sql", cause=e)
finally:
self.closed = True
self.debug and Log.note("Database is closed")
self.db.close()
def _process_command_item(self, command_item):
query, result, signal, trace, transaction = command_item
with Timer("SQL Timing", verbose=self.debug):
if transaction is None:
# THIS IS A TRANSACTIONLESS QUERY, DELAY IT IF THERE IS A CURRENT TRANSACTION
if self.transaction_stack:
with self.locker:
if self.too_long is None:
self.too_long = Till(
seconds=TOO_LONG_TO_HOLD_TRANSACTION)
self.too_long.then(
self.show_transactions_blocked_warning)
self.delayed_queries.append(command_item)
return
elif self.transaction_stack and self.transaction_stack[-1] not in [
transaction,
transaction.parent,
]:
# THIS TRANSACTION IS NOT THE CURRENT TRANSACTION, DELAY IT
with self.locker:
if self.too_long is None:
self.too_long = Till(
seconds=TOO_LONG_TO_HOLD_TRANSACTION)
self.too_long.then(
self.show_transactions_blocked_warning)
self.delayed_transactions.append(command_item)
return
else:
# ENSURE THE CURRENT TRANSACTION IS UP TO DATE FOR THIS query
if not self.transaction_stack:
# sqlite3 ALLOWS ONLY ONE TRANSACTION AT A TIME
self.debug and Log.note(FORMAT_COMMAND, command=BEGIN)
self.db.execute(BEGIN)
self.transaction_stack.append(transaction)
elif transaction is not self.transaction_stack[-1]:
self.transaction_stack.append(transaction)
elif transaction.exception and query is not ROLLBACK:
result.exception = Except(
context=ERROR,
template=
"Not allowed to continue using a transaction that failed",
cause=transaction.exception,
trace=trace,
)
signal.release()
return
try:
transaction.do_all()
except Exception as e:
# DEAL WITH ERRORS IN QUEUED COMMANDS
# WE WILL UNWRAP THE OUTER EXCEPTION TO GET THE CAUSE
err = Except(
context=ERROR,
template="Bad call to Sqlite3 while " + FORMAT_COMMAND,
params={"command": e.params.current.command},
cause=e.cause,
trace=e.params.current.trace,
)
transaction.exception = result.exception = err
if query in [COMMIT, ROLLBACK]:
self._close_transaction(
CommandItem(ROLLBACK, result, signal, trace,
transaction))
signal.release()
return
try:
# DEAL WITH END-OF-TRANSACTION MESSAGES
if query in [COMMIT, ROLLBACK]:
self._close_transaction(command_item)
return
# EXECUTE QUERY
self.last_command_item = command_item
self.debug and Log.note(FORMAT_COMMAND, command=query)
curr = self.db.execute(text(query))
result.meta.format = "table"
result.header = ([d[0] for d in curr.description]
if curr.description else None)
result.data = curr.fetchall()
if self.debug and result.data:
csv = convert.table2csv(list(result.data))
Log.note("Result:\n{{data|limit(100)|indent}}", data=csv)
except Exception as e:
e = Except.wrap(e)
err = Except(
context=ERROR,
template="Bad call to Sqlite while " + FORMAT_COMMAND,
params={"command": query},
trace=trace,
cause=e,
)
result.exception = err
if transaction:
transaction.exception = err
finally:
signal.release()
def main():
try:
config = startup.read_settings()
constants.set(config.constants)
Log.start(config.debug)
# SHUNT PYTHON LOGGING TO MAIN LOGGING
capture_logging()
# SHUNT ADR LOGGING TO MAIN LOGGING
# https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add
capture_loguru()
if config.taskcluster:
inject_secrets(config)
@extend(Configuration)
def update(self, config):
"""
Update the configuration object with new parameters
:param config: dict of configuration
"""
for k, v in config.items():
if v != None:
self._config[k] = v
self._config["sources"] = sorted(
map(os.path.expanduser, set(self._config["sources"])))
# Use the NullStore by default. This allows us to control whether
# caching is enabled or not at runtime.
self._config["cache"].setdefault("stores",
{"null": {
"driver": "null"
}})
object.__setattr__(self, "cache", CustomCacheManager(self._config))
for _, store in self._config["cache"]["stores"].items():
if store.path and not store.path.endswith("/"):
# REQUIRED, OTHERWISE FileStore._create_cache_directory() WILL LOOK AT PARENT DIRECTORY
store.path = store.path + "/"
if SHOW_S3_CACHE_HIT:
s3_get = S3Store._get
@extend(S3Store)
def _get(self, key):
with Timer("get {{key}} from S3", {"key": key},
verbose=False) as timer:
output = s3_get(self, key)
if output is not None:
timer.verbose = True
return output
# UPDATE ADR CONFIGURATION
with Repeat("waiting for ADR", every="10second"):
adr.config.update(config.adr)
# DUMMY TO TRIGGER CACHE
make_push_objects(from_date=Date.today().format(),
to_date=Date.now().format(),
branch="autoland")
outatime = Till(seconds=Duration(MAX_RUNTIME).total_seconds())
outatime.then(lambda: Log.alert("Out of time, exit early"))
Schedulers(config).process(outatime)
except Exception as e:
Log.warning("Problem with etl! Shutting down.", cause=e)
finally:
Log.stop()
