class Schedule(object): @override def __init__( self, interval, # TIME INTERVAL BETWEEN RUNS starting, # THE TIME TO START THE INTERVAL COUNT max_runtime=MAX_RUNTIME, # LIMIT HOW LONG THE PROCESS IS ALIVE wait_for_shutdown=WAIT_FOR_SHUTDOWN, # LIMIT PAITENCE WHEN ASKING FOR SHUTDOWN, THEN SEND KILL process=None, ): self.duration = Duration(interval) self.starting = coalesce(Date(starting), Date.now()) self.max_runtime = Duration(max_runtime) self.wait_for_shutdown = Duration(wait_for_shutdown) # Process parameters self.process = process # STATE self.last_started = None self.last_finished = None self.run_count = 0 self.fail_count = 0 self.current = None self.terminator = None # SIGNAL TO KILL THE PROCESS self.next_run = self._next_run() self.next = Till(till=self.next_run) self.next_run.then(self.run) def _next_run_time(self): """ :return: return signal for next """ interval = mo_math.floor((Date.now() - self.starting) / self.duration) next_time = self.starting + (interval * self.duration) return next_time def run(self): self.last_started = Date.now() self.run_count += 1 self.current = Process(**self.process) self.terminator = Till(seconds=self.max_runtime.seconds) self.terminator.then(self.killer) self.current.service_stopped.then(self.done) def killer(self, please_stop): self.current.stop() ( please_stop | self.current.service_stopped() | Till(seconds=self.wait_for_shutdown.seconds) ).wait() if not self.current.service_stopped: self.fail_count += 1 self.current.kill() self.current.join() def done(self): self.last_finished = Date.now() self.terminator.remove_go(self.killer) self.terminator = None self.current = None self.next_run = self._next_run_time() self.next = Till(till=self.next_run.unix) self.next.then(self.run) def status(self): if self.current is None: status = "never started" elif not self.current.service_stopped: status = "running" elif self.current.returncode == 0: status = "done" else: status = "failed " + text(self.current.returncode) return Data( name=self.name, next_run=self.next_run, last_started=self.last_started, last_finished=self.last_finished, status=status, )
class Sqlite(DB): """ Allows multi-threaded access Loads extension functions (like SQRT) """ @override def __init__( self, filename=None, db=None, get_trace=None, upgrade=False, load_functions=False, debug=False, kwargs=None, ): """ :param filename: FILE TO USE FOR DATABASE :param db: AN EXISTING sqlite3 DB YOU WOULD LIKE TO USE (INSTEAD OF USING filename) :param get_trace: GET THE STACK TRACE AND THREAD FOR EVERY DB COMMAND (GOOD FOR DEBUGGING) :param upgrade: REPLACE PYTHON sqlite3 DLL WITH MORE RECENT ONE, WITH MORE FUNCTIONS (NOT WORKING) :param load_functions: LOAD EXTENDED MATH FUNCTIONS (MAY REQUIRE upgrade) :param kwargs: """ global _upgraded global _sqlite3 self.settings = kwargs if not _upgraded: if upgrade: _upgrade() _upgraded = True import sqlite3 as _sqlite3 _ = _sqlite3 self.filename = File(filename).abspath if filename else None if known_databases.get(self.filename): Log.error( "Not allowed to create more than one Sqlite instance for {{file}}", file=self.filename, ) self.debug = debug | DEBUG # SETUP DATABASE self.debug and Log.note("Sqlite version {{version}}", version=_sqlite3.sqlite_version) try: if db == None: self.db = _sqlite3.connect( database=coalesce(self.filename, ":memory:"), check_same_thread=False, isolation_level=None, ) else: self.db = db except Exception as e: Log.error("could not open file {{filename}}", filename=self.filename, cause=e) self.upgrade = upgrade load_functions and self._load_functions() self.locker = Lock() self.available_transactions = [ ] # LIST OF ALL THE TRANSACTIONS BEING MANAGED self.queue = Queue( "sql commands" ) # HOLD (command, result, signal, stacktrace) TUPLES self.get_trace = coalesce(get_trace, TRACE) self.closed = False # WORKER VARIABLES self.transaction_stack = [ ] # THE TRANSACTION OBJECT WE HAVE PARTIALLY RUN self.last_command_item = ( None ) # USE THIS TO HELP BLAME current_transaction FOR HANGING ON TOO LONG self.too_long = None self.delayed_queries = [] self.delayed_transactions = [] self.worker = Thread.run("sqlite db thread", self._worker) self.debug and Log.note( "Sqlite version {{version}}", version=self.query("select sqlite_version()").data[0][0], ) def _enhancements(self): def regex(pattern, value): return 1 if re.match(pattern + "$", value) else 0 con = self.db.create_function("regex", 2, regex) class Percentile(object): def __init__(self, percentile): self.percentile = percentile self.acc = [] def step(self, value): self.acc.append(value) def finalize(self): return percentile(self.acc, self.percentile) con.create_aggregate("percentile", 2, Percentile) def transaction(self): thread = Thread.current() parent = None with self.locker: for t in self.available_transactions: if t.thread is thread: parent = t output = Transaction(self, parent=parent, thread=thread) self.available_transactions.append(output) return output def about(self, table_name): """ :param table_name: TABLE IF INTEREST :return: SOME INFORMATION ABOUT THE TABLE (cid, name, dtype, notnull, dfft_value, pk) tuples """ details = self.query("PRAGMA table_info" + sql_iso(quote_column(table_name))) return details.data def query(self, command): """ WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED :param command: COMMAND FOR SQLITE :return: list OF RESULTS """ if self.closed: Log.error("database is closed") signal = _allocate_lock() signal.acquire() result = Data() trace = get_stacktrace(1) if self.get_trace else None if self.get_trace: current_thread = Thread.current() with self.locker: for t in self.available_transactions: if t.thread is current_thread: Log.error(DOUBLE_TRANSACTION_ERROR) self.queue.add(CommandItem(command, result, signal, trace, None)) signal.acquire() if result.exception: Log.error("Problem with Sqlite call", cause=result.exception) return result def close(self): """ OPTIONAL COMMIT-AND-CLOSE IF THIS IS NOT DONE, THEN THE THREAD THAT SPAWNED THIS INSTANCE :return: """ self.closed = True signal = _allocate_lock() signal.acquire() self.queue.add(CommandItem(COMMIT, None, signal, None, None)) signal.acquire() self.worker.please_stop.go() return def __enter__(self): pass def __exit__(self, exc_type, exc_val, exc_tb): self.close() def _load_functions(self): global _load_extension_warning_sent library_loc = File.new_instance(sys.modules[__name__].__file__, "../..") full_path = File.new_instance( library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath try: trace = get_stacktrace(0)[0] if self.upgrade: if os.name == "nt": file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions.so") else: file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions") full_path = file.abspath self.db.enable_load_extension(True) self.db.execute( text(SQL_SELECT + "load_extension" + sql_iso(quote_value(full_path)))) except Exception as e: if not _load_extension_warning_sent: _load_extension_warning_sent = True Log.warning( "Could not load {{file}}, doing without. (no SQRT for you!)", file=full_path, cause=e, ) def create_new_functions(self): def regexp(pattern, item): reg = re.compile(pattern) return reg.search(item) is not None self.db.create_function("REGEXP", 2, regexp) def show_transactions_blocked_warning(self): blocker = self.last_command_item blocked = (self.delayed_queries + self.delayed_transactions)[0] Log.warning( "Query on thread {{blocked_thread|json}} at\n" "{{blocked_trace|indent}}" "is blocked by {{blocker_thread|json}} at\n" "{{blocker_trace|indent}}" "this message brought to you by....", blocker_trace=format_trace(blocker.trace), blocked_trace=format_trace(blocked.trace), blocker_thread=blocker.transaction.thread.name if blocker.transaction is not None else None, blocked_thread=blocked.transaction.thread.name if blocked.transaction is not None else None, ) def _close_transaction(self, command_item): query, result, signal, trace, transaction = command_item transaction.end_of_life = True with self.locker: self.available_transactions.remove(transaction) assert transaction not in self.available_transactions old_length = len(self.transaction_stack) old_trans = self.transaction_stack[-1] del self.transaction_stack[-1] assert old_length - 1 == len(self.transaction_stack) assert old_trans assert old_trans not in self.transaction_stack if not self.transaction_stack: # NESTED TRANSACTIONS NOT ALLOWED IN sqlite3 self.debug and Log.note(FORMAT_COMMAND, command=query) self.db.execute(query) has_been_too_long = False with self.locker: if self.too_long is not None: self.too_long, too_long = None, self.too_long # WE ARE CHEATING HERE: WE REACH INTO THE Signal MEMBERS AND REMOVE WHAT WE ADDED TO THE INTERNAL job_queue with too_long.lock: has_been_too_long = bool(too_long) too_long.job_queue = None # PUT delayed BACK ON THE QUEUE, IN THE ORDER FOUND, BUT WITH QUERIES FIRST if self.delayed_transactions: for c in reversed(self.delayed_transactions): self.queue.push(c) del self.delayed_transactions[:] if self.delayed_queries: for c in reversed(self.delayed_queries): self.queue.push(c) del self.delayed_queries[:] if has_been_too_long: Log.note("Transaction blockage cleared") def _worker(self, please_stop): try: # MAIN EXECUTION LOOP while not please_stop: command_item = self.queue.pop(till=please_stop) if command_item is None: break try: self._process_command_item(command_item) except Exception as e: Log.warning("worker can not execute command", cause=e) except Exception as e: e = Except.wrap(e) if not please_stop: Log.warning("Problem with sql", cause=e) finally: self.closed = True self.debug and Log.note("Database is closed") self.db.close() def _process_command_item(self, command_item): query, result, signal, trace, transaction = command_item with Timer("SQL Timing", verbose=self.debug): if transaction is None: # THIS IS A TRANSACTIONLESS QUERY, DELAY IT IF THERE IS A CURRENT TRANSACTION if self.transaction_stack: with self.locker: if self.too_long is None: self.too_long = Till( seconds=TOO_LONG_TO_HOLD_TRANSACTION) self.too_long.then( self.show_transactions_blocked_warning) self.delayed_queries.append(command_item) return elif self.transaction_stack and self.transaction_stack[-1] not in [ transaction, transaction.parent, ]: # THIS TRANSACTION IS NOT THE CURRENT TRANSACTION, DELAY IT with self.locker: if self.too_long is None: self.too_long = Till( seconds=TOO_LONG_TO_HOLD_TRANSACTION) self.too_long.then( self.show_transactions_blocked_warning) self.delayed_transactions.append(command_item) return else: # ENSURE THE CURRENT TRANSACTION IS UP TO DATE FOR THIS query if not self.transaction_stack: # sqlite3 ALLOWS ONLY ONE TRANSACTION AT A TIME self.debug and Log.note(FORMAT_COMMAND, command=BEGIN) self.db.execute(BEGIN) self.transaction_stack.append(transaction) elif transaction is not self.transaction_stack[-1]: self.transaction_stack.append(transaction) elif transaction.exception and query is not ROLLBACK: result.exception = Except( context=ERROR, template= "Not allowed to continue using a transaction that failed", cause=transaction.exception, trace=trace, ) signal.release() return try: transaction.do_all() except Exception as e: # DEAL WITH ERRORS IN QUEUED COMMANDS # WE WILL UNWRAP THE OUTER EXCEPTION TO GET THE CAUSE err = Except( context=ERROR, template="Bad call to Sqlite3 while " + FORMAT_COMMAND, params={"command": e.params.current.command}, cause=e.cause, trace=e.params.current.trace, ) transaction.exception = result.exception = err if query in [COMMIT, ROLLBACK]: self._close_transaction( CommandItem(ROLLBACK, result, signal, trace, transaction)) signal.release() return try: # DEAL WITH END-OF-TRANSACTION MESSAGES if query in [COMMIT, ROLLBACK]: self._close_transaction(command_item) return # EXECUTE QUERY self.last_command_item = command_item self.debug and Log.note(FORMAT_COMMAND, command=query) curr = self.db.execute(text(query)) result.meta.format = "table" result.header = ([d[0] for d in curr.description] if curr.description else None) result.data = curr.fetchall() if self.debug and result.data: csv = convert.table2csv(list(result.data)) Log.note("Result:\n{{data|limit(100)|indent}}", data=csv) except Exception as e: e = Except.wrap(e) err = Except( context=ERROR, template="Bad call to Sqlite while " + FORMAT_COMMAND, params={"command": query}, trace=trace, cause=e, ) result.exception = err if transaction: transaction.exception = err finally: signal.release()
def main(): try: config = startup.read_settings() constants.set(config.constants) Log.start(config.debug) # SHUNT PYTHON LOGGING TO MAIN LOGGING capture_logging() # SHUNT ADR LOGGING TO MAIN LOGGING # https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add capture_loguru() if config.taskcluster: inject_secrets(config) @extend(Configuration) def update(self, config): """ Update the configuration object with new parameters :param config: dict of configuration """ for k, v in config.items(): if v != None: self._config[k] = v self._config["sources"] = sorted( map(os.path.expanduser, set(self._config["sources"]))) # Use the NullStore by default. This allows us to control whether # caching is enabled or not at runtime. self._config["cache"].setdefault("stores", {"null": { "driver": "null" }}) object.__setattr__(self, "cache", CustomCacheManager(self._config)) for _, store in self._config["cache"]["stores"].items(): if store.path and not store.path.endswith("/"): # REQUIRED, OTHERWISE FileStore._create_cache_directory() WILL LOOK AT PARENT DIRECTORY store.path = store.path + "/" if SHOW_S3_CACHE_HIT: s3_get = S3Store._get @extend(S3Store) def _get(self, key): with Timer("get {{key}} from S3", {"key": key}, verbose=False) as timer: output = s3_get(self, key) if output is not None: timer.verbose = True return output # UPDATE ADR CONFIGURATION with Repeat("waiting for ADR", every="10second"): adr.config.update(config.adr) # DUMMY TO TRIGGER CACHE make_push_objects(from_date=Date.today().format(), to_date=Date.now().format(), branch="autoland") outatime = Till(seconds=Duration(MAX_RUNTIME).total_seconds()) outatime.then(lambda: Log.alert("Out of time, exit early")) Schedulers(config).process(outatime) except Exception as e: Log.warning("Problem with etl! Shutting down.", cause=e) finally: Log.stop()