def request(method, url, zip=None, retry=None, **kwargs): """ JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES DEMANDS data IS ONE OF: * A JSON-SERIALIZABLE STRUCTURE, OR * LIST OF JSON-SERIALIZABLE STRUCTURES, OR * None Parameters * zip - ZIP THE REQUEST BODY, IF BIG ENOUGH * json - JSON-SERIALIZABLE STRUCTURE * retry - {"times": x, "sleep": y} STRUCTURE THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT** IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH INCLUDES url AND headers """ global _warning_sent global request_count if not default_headers and not _warning_sent: _warning_sent = True Log.warning( text_type( "The pyLibrary.env.http module was meant to add extra " + "default headers to all requests, specifically the 'Referer' " + "header with a URL to the project. Use the `pyLibrary.debug.constants.set()` " + "function to set `pyLibrary.env.http.default_headers`")) if isinstance(url, list): # TRY MANY URLS failures = [] for remaining, u in jx.countdown(url): try: response = request(method, u, zip=zip, retry=retry, **kwargs) if Math.round(response.status_code, decimal=-2) not in [400, 500]: return response if not remaining: return response except Exception as e: e = Except.wrap(e) failures.append(e) Log.error(u"Tried {{num}} urls", num=len(url), cause=failures) if 'session' in kwargs: session = kwargs['session'] del kwargs['session'] sess = Null else: sess = session = sessions.Session() session.headers.update(default_headers) with closing(sess): if zip is None: zip = ZIP_REQUEST if PY2 and isinstance(url, unicode): # httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE url = url.encode('ascii') _to_ascii_dict(kwargs) timeout = kwargs['timeout'] = coalesce(kwargs.get('timeout'), default_timeout) if retry == None: retry = Data(times=1, sleep=0) elif isinstance(retry, Number): retry = Data(times=retry, sleep=1) else: retry = wrap(retry) if isinstance(retry.sleep, Duration): retry.sleep = retry.sleep.seconds set_default(retry, {"times": 1, "sleep": 0}) if 'json' in kwargs: kwargs['data'] = value2json(kwargs['json']).encode('utf8') del kwargs['json'] try: headers = kwargs['headers'] = unwrap( coalesce(kwargs.get('headers'), {})) set_default(headers, {'Accept-Encoding': 'compress, gzip'}) if zip and len(coalesce(kwargs.get('data'))) > 1000: compressed = convert.bytes2zip(kwargs['data']) headers['content-encoding'] = 'gzip' kwargs['data'] = compressed _to_ascii_dict(headers) else: _to_ascii_dict(headers) except Exception as e: Log.error(u"Request setup failure on {{url}}", url=url, cause=e) errors = [] for r in range(retry.times): if r: Till(seconds=retry.sleep).wait() try: if DEBUG: Log.note(u"http {{method|upper}} to {{url}}", method=method, url=text_type(url)) request_count += 1 return session.request(method=method, url=url, **kwargs) except Exception as e: errors.append(Except.wrap(e)) if " Read timed out." in errors[0]: Log.error( u"Tried {{times}} times: Timeout failure (timeout was {{timeout}}", timeout=timeout, times=retry.times, cause=errors[0]) else: Log.error(u"Tried {{times}} times: Request failure of {{url}}", url=url, times=retry.times, cause=errors[0])
def _process_command_item(self, command_item): query, result, signal, trace, transaction = command_item with Timer("SQL Timing", debug=DEBUG): if transaction is None: # THIS IS A TRANSACTIONLESS QUERY, DELAY IT IF THERE IS A CURRENT TRANSACTION if self.transaction_stack: with self.locker: if self.too_long is None: self.too_long = Till(seconds=TOO_LONG_TO_HOLD_TRANSACTION) self.too_long.on_go(self.show_transactions_blocked_warning) self.delayed_queries.append(command_item) return elif self.transaction_stack and self.transaction_stack[-1] not in [transaction, transaction.parent]: # THIS TRANSACTION IS NOT THE CURRENT TRANSACTION, DELAY IT with self.locker: if self.too_long is None: self.too_long = Till(seconds=TOO_LONG_TO_HOLD_TRANSACTION) self.too_long.on_go(self.show_transactions_blocked_warning) self.delayed_transactions.append(command_item) return else: # ENSURE THE CURRENT TRANSACTION IS UP TO DATE FOR THIS query if not self.transaction_stack: # sqlite3 ALLOWS ONLY ONE TRANSACTION AT A TIME DEBUG and Log.note(FORMAT_COMMAND, command=BEGIN) self.db.execute(BEGIN) self.transaction_stack.append(transaction) elif transaction != self.transaction_stack[-1]: self.transaction_stack.append(transaction) elif transaction.exception: result.exception = Except( type=ERROR, template="Not allowed to continue using a transaction that failed", cause=transaction.exception, trace=trace ) signal.release() return try: transaction.do_all() except Exception as e: # DEAL WITH ERRORS IN QUEUED COMMANDS # WE WILL UNWRAP THE OUTER EXCEPTION TO GET THE CAUSE e = Except.wrap(e) err = Except( type=ERROR, template="Bad call to Sqlite3 while "+FORMAT_COMMAND, params={"command": e.params.current.command}, cause=e.cause, trace=e.params.current.trace ) transaction.exception = result.exception = err if query in [COMMIT, ROLLBACK]: self._close_transaction(CommandItem(ROLLBACK, result, signal, trace, transaction)) signal.release() return try: # DEAL WITH END-OF-TRANSACTION MESSAGES if query in [COMMIT, ROLLBACK]: self._close_transaction(command_item) return # EXECUTE QUERY self.last_command_item = command_item DEBUG and Log.note(FORMAT_COMMAND, command=query) curr = self.db.execute(query) result.meta.format = "table" result.header = [d[0] for d in curr.description] if curr.description else None result.data = curr.fetchall() if DEBUG and result.data: text = convert.table2csv(list(result.data)) Log.note("Result:\n{{data|limit(100)|indent}}", data=text) except Exception as e: e = Except.wrap(e) err = Except( type=ERROR, template="Bad call to Sqlite while " + FORMAT_COMMAND, params={"command": query}, trace=trace, cause=e ) result.exception = err if transaction: transaction.exception = err finally: signal.release()
def csetLog_maintenance(self, please_stop=None): ''' Handles deleting old csetLog entries and timestamping revisions once they pass the length for permanent storage for deletion later. :param please_stop: :return: ''' while not please_stop: try: # Wait until something signals the maintenance cycle # to begin (or end). (self.maintenance_signal | please_stop).wait() if please_stop: break if self.disable_maintenance: continue # Reset signal so we don't request # maintenance infinitely. with self.maintenance_signal.lock: self.maintenance_signal._go = False with self.working_locker: all_data = None with self.conn.transaction() as t: all_data = sorted(t.get( "SELECT revnum, revision, timestamp FROM csetLog"), key=lambda x: int(x[0])) # Restore maximum permanents (if overflowing) new_data = [] modified = False for count, (revnum, revision, timestamp) in enumerate(all_data[::-1]): if count < MINIMUM_PERMANENT_CSETS: if timestamp != -1: modified = True new_data.append((revnum, revision, -1)) else: new_data.append((revnum, revision, timestamp)) elif type(timestamp) != int or timestamp == -1: modified = True new_data.append( (revnum, revision, int(time.time()))) else: new_data.append((revnum, revision, timestamp)) # Delete annotations at revisions with timestamps # that are too old. The csetLog entries will have # their timestamps reset here. new_data1 = [] annrevs_to_del = [] current_time = time.time() for count, (revnum, revision, timestamp) in enumerate(new_data[::-1]): new_timestamp = timestamp if timestamp != -1: if current_time >= timestamp + TIME_TO_KEEP_ANNOTATIONS.seconds: modified = True new_timestamp = current_time annrevs_to_del.append(revision) new_data1.append((revnum, revision, new_timestamp)) if len(annrevs_to_del) > 0: # Delete any latestFileMod and annotation entries # that are too old. Log.note( "Deleting annotations and latestFileMod for revisions for being " "older than {{oldest}}: {{revisions}}", oldest=TIME_TO_KEEP_ANNOTATIONS, revisions=annrevs_to_del) with self.conn.transaction() as t: t.execute( "DELETE FROM latestFileMod WHERE revision IN " + quote_set(annrevs_to_del)) t.execute( "DELETE FROM annotations WHERE revision IN " + quote_set(annrevs_to_del)) # Delete any overflowing entries new_data2 = new_data1 reved_all_data = all_data[::-1] deleted_data = reved_all_data[MAXIMUM_NONPERMANENT_CSETS:] delete_overflowing_revstart = None if len(deleted_data) > 0: _, delete_overflowing_revstart, _ = deleted_data[0] new_data2 = set(all_data) - set(deleted_data) # Update old frontiers if requested, otherwise # they will all get deleted by the csetLog_deleter # worker if UPDATE_VERY_OLD_FRONTIERS: _, max_revision, _ = all_data[-1] for _, revision, _ in deleted_data: with self.conn.transaction() as t: old_files = t.get( "SELECT file FROM latestFileMod WHERE revision=?", (revision, )) if old_files is None or len(old_files) <= 0: continue self.tuid_service.get_tuids_from_files( old_files, max_revision, going_forward=True, ) still_exist = True while still_exist and not please_stop: Till(seconds=TUID_EXISTENCE_WAIT_TIME ).wait() with self.conn.transaction() as t: old_files = t.get( "SELECT file FROM latestFileMod WHERE revision=?", (revision, )) if old_files is None or len( old_files) <= 0: still_exist = False # Update table and schedule a deletion if modified: with self.conn.transaction() as t: t.execute( "INSERT OR REPLACE INTO csetLog (revnum, revision, timestamp) VALUES " + sql_list( quote_set(cset_entry) for cset_entry in new_data2)) if not deleted_data: continue Log.note("Scheduling {{num_csets}} for deletion", num_csets=len(deleted_data)) self.deletions_todo.add(delete_overflowing_revstart) except Exception as e: Log.warning( "Unexpected error occured while maintaining csetLog, continuing to try: ", cause=e) return
def csetLog_deleter(self, please_stop=None): ''' Deletes changesets from the csetLog table and also changesets from the annotation table that have revisions matching the given changesets. Accepts lists of csets from self.deletions_todo. :param please_stop: :return: ''' while not please_stop: try: request = self.deletions_todo.pop(till=please_stop) if please_stop: break # If deletion is disabled, ignore the current # request - it will need to be re-requested. if self.disable_deletion: Till(till=CSET_DELETION_WAIT_TIME).wait() continue with self.working_locker: first_cset = request # Since we are deleting and moving stuff around in the # TUID tables, we need everything to be contained in # one transaction with no interruptions. with self.conn.transaction() as t: revnum = self._get_one_revnum(t, first_cset)[0] csets_to_del = t.get( "SELECT revnum, revision FROM csetLog WHERE revnum <= ?", (revnum, )) csets_to_del = [cset for _, cset in csets_to_del] existing_frontiers = t.query( "SELECT revision FROM latestFileMod WHERE revision IN " + quote_set(csets_to_del)).data existing_frontiers = [ existing_frontiers[i][0] for i, _ in enumerate(existing_frontiers) ] Log.note( "Deleting all annotations and changeset log entries with revisions in the list: {{csets}}", csets=csets_to_del) if len(existing_frontiers) > 0: # This handles files which no longer exist anymore in # the main branch. Log.note( "Deleting existing frontiers for revisions: {{revisions}}", revisions=existing_frontiers) t.execute( "DELETE FROM latestFileMod WHERE revision IN " + quote_set(existing_frontiers)) Log.note("Deleting annotations...") t.execute( "DELETE FROM annotations WHERE revision IN " + quote_set(csets_to_del)) Log.note("Deleting {{num_entries}} csetLog entries...", num_entries=len(csets_to_del)) t.execute("DELETE FROM csetLog WHERE revision IN " + quote_set(csets_to_del)) # Recalculate the revnums self.recompute_table_revnums() except Exception as e: Log.warning( "Unexpected error occured while deleting from csetLog:", cause=e) Till(seconds=CSET_DELETION_WAIT_TIME).wait() return
def _db_worker(self, please_stop): while not please_stop: try: with self._db_transaction(): result = self._query( SQL_SELECT + all_columns + SQL_FROM + db_table_name + SQL_WHERE + "last_updated>" + quote_value(self.last_load) + SQL_ORDERBY + sql_list( map(quote_column, ["es_index", "name", "es_column"]))) with self.locker: for r in result.data: c = row_to_column(result.header, r) self._add(c) if c.last_updated > self.last_load: self.last_load = c.last_updated updates = self.todo.pop_all() DEBUG and updates and Log.note("{{num}} columns to push to db", num=len(updates)) for action, column in updates: while not please_stop: try: with self._db_transaction(): DEBUG and Log.note( "{{action}} db for {{table}}.{{column}}", action=action, table=column.es_index, column=column.es_column) if action is EXECUTE: self.db.execute(column) elif action is UPDATE: self.db.execute( "UPDATE" + db_table_name + "SET" + sql_list([ "count=" + quote_value( column.count), "cardinality=" + quote_value(column.cardinality), "multi=" + quote_value(column.multi), "partitions=" + quote_value( value2json(column.partitions)), "last_updated=" + quote_value(column.last_updated) ]) + SQL_WHERE + SQL_AND.join([ "es_index = " + quote_value(column.es_index), "es_column = " + quote_value(column.es_column), "last_updated < " + quote_value(column.last_updated) ])) elif action is DELETE: self.db.execute( "DELETE FROM" + db_table_name + SQL_WHERE + SQL_AND.join([ "es_index = " + quote_value(column.es_index), "es_column = " + quote_value(column.es_column) ])) else: self._db_insert_column(column) break except Exception as e: e = Except.wrap(e) if "database is locked" in e: Log.note("metadata database is locked") Till(seconds=1).wait() break else: Log.warning("problem updataing database", cause=e) except Exception as e: Log.warning("problem updataing database", cause=e) (Till(seconds=10) | please_stop).wait()
def _normalize_revision(self, r, found_revision, push, get_diff, get_moves): new_names = set(r.keys()) - KNOWN_TAGS if new_names and not r.tags: Log.warning( "hg is returning new property names {{names|quote}} for {{changeset}} from {{url}}", names=new_names, changeset=r.node, url=found_revision.branch.url) changeset = Changeset( id=r.node, id12=r.node[0:12], author=r.user, description=strings.limit(coalesce(r.description, r.desc), 2000), date=parse_hg_date(r.date), files=r.files, backedoutby=r.backedoutby if r.backedoutby else None, bug=self._extract_bug_id(r.description)) rev = Revision(branch=found_revision.branch, index=r.rev, changeset=changeset, parents=unwraplist(list(set(r.parents))), children=unwraplist(list(set(r.children))), push=push, phase=r.phase, bookmarks=unwraplist(r.bookmarks), landingsystem=r.landingsystem, etl={ "timestamp": Date.now().unix, "machine": machine_metadata }) r.pushuser = None r.pushdate = None r.pushid = None r.node = None r.user = None r.desc = None r.description = None r.date = None r.files = None r.backedoutby = None r.parents = None r.children = None r.bookmarks = None r.landingsystem = None set_default(rev, r) # ADD THE DIFF if get_diff: rev.changeset.diff = self._get_json_diff_from_hg(rev) if get_moves: rev.changeset.moves = self._get_moves_from_hg(rev) try: _id = coalesce(rev.changeset.id12, "") + "-" + rev.branch.name + "-" + coalesce( rev.branch.locale, DEFAULT_LOCALE) with self.es_locker: self.es.add({"id": _id, "value": rev}) except Exception as e: e = Except.wrap(e) Log.warning("Did not save to ES, waiting {{duration}} seconds", duration=WAIT_AFTER_NODE_FAILURE, cause=e) Till(seconds=WAIT_AFTER_NODE_FAILURE).wait() if "FORBIDDEN/12/index read-only" in e: pass # KNOWN FAILURE MODE return rev
def _get_from_elasticsearch(self, revision, locale=None, get_diff=False, get_moves=True): rev = revision.changeset.id if self.es.cluster.version.startswith("1.7."): query = { "query": { "filtered": { "query": { "match_all": {} }, "filter": { "and": [{ "term": { "changeset.id12": rev[0:12] } }, { "term": { "branch.name": revision.branch.name } }, { "term": { "branch.locale": coalesce(locale, revision.branch.locale, DEFAULT_LOCALE) } }, { "range": { "etl.timestamp": { "gt": MIN_ETL_AGE } } }] } } }, "size": 20 } else: query = { "query": { "bool": { "must": [{ "term": { "changeset.id12": rev[0:12] } }, { "term": { "branch.name": revision.branch.name } }, { "term": { "branch.locale": coalesce(locale, revision.branch.locale, DEFAULT_LOCALE) } }, { "range": { "etl.timestamp": { "gt": MIN_ETL_AGE } } }] } }, "size": 20 } for attempt in range(3): try: with self.es_locker: docs = self.es.search(query).hits.hits if len(docs) == 0: return None best = docs[0]._source if len(docs) > 1: for d in docs: if d._id.endswith(d._source.branch.locale): best = d._source Log.warning("expecting no more than one document") return best except Exception as e: e = Except.wrap(e) if "EsRejectedExecutionException[rejected execution (queue capacity" in e: (Till(seconds=Random.int(30))).wait() continue else: Log.warning("Bad ES call, waiting for {{num}} seconds", num=WAIT_AFTER_NODE_FAILURE, cause=e) Till(seconds=WAIT_AFTER_NODE_FAILURE).wait() continue Log.warning("ES did not deliver, fall back to HG") return None
def get_revision(self, revision, locale=None, get_diff=False, get_moves=True): """ EXPECTING INCOMPLETE revision OBJECT RETURNS revision """ rev = revision.changeset.id if not rev: return Null elif rev == "None": return Null elif revision.branch.name == None: return Null locale = coalesce(locale, revision.branch.locale, DEFAULT_LOCALE) output = self._get_from_elasticsearch(revision, locale=locale, get_diff=get_diff) if output: if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED output.changeset.diff = None if not get_moves: output.changeset.moves = None DEBUG and Log.note( "Got hg ({{branch}}, {{locale}}, {{revision}}) from ES", branch=output.branch.name, locale=locale, revision=output.changeset.id) if output.push.date >= Date.now() - MAX_TODO_AGE: self.todo.add((output.branch, listwrap(output.parents))) self.todo.add((output.branch, listwrap(output.children))) if output.push.date: return output # RATE LIMIT CALLS TO HG (CACHE MISSES) next_cache_miss = self.last_cache_miss + ( Random.float(WAIT_AFTER_CACHE_MISS * 2) * SECOND) self.last_cache_miss = Date.now() if next_cache_miss > self.last_cache_miss: Log.note("delaying next hg call for {{seconds|round(decimal=1)}}", seconds=next_cache_miss - self.last_cache_miss) Till(till=next_cache_miss.unix).wait() found_revision = copy(revision) if isinstance(found_revision.branch, (text_type, binary_type)): lower_name = found_revision.branch.lower() else: lower_name = found_revision.branch.name.lower() if not lower_name: Log.error("Defective revision? {{rev|json}}", rev=found_revision.branch) b = found_revision.branch = self.branches[(lower_name, locale)] if not b: b = found_revision.branch = self.branches[(lower_name, DEFAULT_LOCALE)] if not b: Log.warning("can not find branch ({{branch}}, {{locale}})", branch=lower_name, locale=locale) return Null if Date.now() - Date(b.etl.timestamp) > _OLD_BRANCH: self.branches = _hg_branches.get_branches(kwargs=self.settings) push = self._get_push(found_revision.branch, found_revision.changeset.id) url1 = found_revision.branch.url.rstrip( "/") + "/json-info?node=" + found_revision.changeset.id[0:12] url2 = found_revision.branch.url.rstrip( "/") + "/json-rev/" + found_revision.changeset.id[0:12] with Explanation("get revision from {{url}}", url=url1, debug=DEBUG): raw_rev2 = Null try: raw_rev1 = self._get_raw_json_info(url1, found_revision.branch) raw_rev2 = self._get_raw_json_rev(url2, found_revision.branch) except Exception as e: if "Hg denies it exists" in e: raw_rev1 = Data(node=revision.changeset.id) else: raise e output = self._normalize_revision(set_default(raw_rev1, raw_rev2), found_revision, push, get_diff, get_moves) if output.push.date >= Date.now() - MAX_TODO_AGE: self.todo.add((output.branch, listwrap(output.parents))) self.todo.add((output.branch, listwrap(output.children))) if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED output.changeset.diff = None if not get_moves: output.changeset.moves = None return output
class Sqlite(DB): """ Allows multi-threaded access Loads extension functions (like SQRT) """ @override def __init__(self, filename=None, db=None, get_trace=None, upgrade=True, load_functions=False, kwargs=None): """ :param filename: FILE TO USE FOR DATABASE :param db: AN EXISTING sqlite3 DB YOU WOULD LIKE TO USE (INSTEAD OF USING filename) :param get_trace: GET THE STACK TRACE AND THREAD FOR EVERY DB COMMAND (GOOD FOR DEBUGGING) :param upgrade: REPLACE PYTHON sqlite3 DLL WITH MORE RECENT ONE, WITH MORE FUNCTIONS (NOT WORKING) :param load_functions: LOAD EXTENDED MATH FUNCTIONS (MAY REQUIRE upgrade) :param kwargs: """ global _upgraded global _sqlite3 self.settings = kwargs if not _upgraded: if upgrade: _upgrade() _upgraded = True import sqlite3 as _sqlite3 _ = _sqlite3 self.filename = File(filename).abspath if filename else None if known_databases.get(self.filename): Log.error( "Not allowed to create more than one Sqlite instance for {{file}}", file=self.filename) # SETUP DATABASE DEBUG and Log.note("Sqlite version {{version}}", version=_sqlite3.sqlite_version) try: if db == None: self.db = _sqlite3.connect(database=coalesce( self.filename, ":memory:"), check_same_thread=False, isolation_level=None) else: self.db = db except Exception as e: Log.error("could not open file {{filename}}", filename=self.filename, cause=e) load_functions and self._load_functions() self.locker = Lock() self.available_transactions = [ ] # LIST OF ALL THE TRANSACTIONS BEING MANAGED self.queue = Queue( "sql commands" ) # HOLD (command, result, signal, stacktrace) TUPLES self.get_trace = coalesce(get_trace, TRACE) self.upgrade = upgrade self.closed = False # WORKER VARIABLES self.transaction_stack = [ ] # THE TRANSACTION OBJECT WE HAVE PARTIALLY RUN self.last_command_item = None # USE THIS TO HELP BLAME current_transaction FOR HANGING ON TOO LONG self.too_long = None self.delayed_queries = [] self.delayed_transactions = [] self.worker = Thread.run("sqlite db thread", self._worker) DEBUG and Log.note( "Sqlite version {{version}}", version=self.query("select sqlite_version()").data[0][0]) def _enhancements(self): def regex(pattern, value): return 1 if re.match(pattern + "$", value) else 0 con = self.db.create_function("regex", 2, regex) class Percentile(object): def __init__(self, percentile): self.percentile = percentile self.acc = [] def step(self, value): self.acc.append(value) def finalize(self): return percentile(self.acc, self.percentile) con.create_aggregate("percentile", 2, Percentile) def transaction(self): thread = Thread.current() parent = None with self.locker: for t in self.available_transactions: if t.thread is thread: parent = t output = Transaction(self, parent=parent) self.available_transactions.append(output) return output def query(self, command): """ WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED :param command: COMMAND FOR SQLITE :return: list OF RESULTS """ if self.closed: Log.error("database is closed") signal = _allocate_lock() signal.acquire() result = Data() trace = extract_stack(1) if self.get_trace else None if self.get_trace: current_thread = Thread.current() with self.locker: for t in self.available_transactions: if t.thread is current_thread: Log.error(DOUBLE_TRANSACTION_ERROR) self.queue.add(CommandItem(command, result, signal, trace, None)) signal.acquire() if result.exception: Log.error("Problem with Sqlite call", cause=result.exception) return result def close(self): """ OPTIONAL COMMIT-AND-CLOSE IF THIS IS NOT DONE, THEN THE THREAD THAT SPAWNED THIS INSTANCE :return: """ self.closed = True signal = _allocate_lock() signal.acquire() self.queue.add(CommandItem(COMMIT, None, signal, None, None)) signal.acquire() self.worker.please_stop.go() return def __enter__(self): pass def __exit__(self, exc_type, exc_val, exc_tb): self.close() def _load_functions(self): global _load_extension_warning_sent library_loc = File.new_instance(sys.modules[__name__].__file__, "../..") full_path = File.new_instance( library_loc, "vendor/sqlite/libsqlitefunctions.so").abspath try: trace = extract_stack(0)[0] if self.upgrade: if os.name == 'nt': file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions.so") else: file = File.new_instance( trace["file"], "../../vendor/sqlite/libsqlitefunctions") full_path = file.abspath self.db.enable_load_extension(True) self.db.execute(SQL_SELECT + "load_extension" + sql_iso(quote_value(full_path))) except Exception as e: if not _load_extension_warning_sent: _load_extension_warning_sent = True Log.warning( "Could not load {{file}}, doing without. (no SQRT for you!)", file=full_path, cause=e) def create_new_functions(self): def regexp(pattern, item): reg = re.compile(pattern) return reg.search(item) is not None self.db.create_function("REGEXP", 2, regexp) def show_transactions_blocked_warning(self): blocker = self.last_command_item blocked = (self.delayed_queries + self.delayed_transactions)[0] Log.warning( "Query on thread {{blocked_thread|json}} at\n" "{{blocked_trace|indent}}" "is blocked by {{blocker_thread|json}} at\n" "{{blocker_trace|indent}}" "this message brought to you by....", blocker_trace=format_trace(blocker.trace), blocked_trace=format_trace(blocked.trace), blocker_thread=blocker.transaction.thread.name if blocker.transaction is not None else None, blocked_thread=blocked.transaction.thread.name if blocked.transaction is not None else None) def _close_transaction(self, command_item): query, result, signal, trace, transaction = command_item transaction.end_of_life = True with self.locker: self.available_transactions.remove(transaction) assert transaction not in self.available_transactions old_length = len(self.transaction_stack) old_trans = self.transaction_stack[-1] del self.transaction_stack[-1] assert old_length - 1 == len(self.transaction_stack) assert old_trans assert old_trans not in self.transaction_stack if not self.transaction_stack: # NESTED TRANSACTIONS NOT ALLOWED IN sqlite3 DEBUG and Log.note(FORMAT_COMMAND, command=query) self.db.execute(query) has_been_too_long = False with self.locker: if self.too_long is not None: self.too_long, too_long = None, self.too_long # WE ARE CHEATING HERE: WE REACH INTO THE Signal MEMBERS AND REMOVE WHAT WE ADDED TO THE INTERNAL job_queue with too_long.lock: has_been_too_long = bool(too_long) too_long.job_queue = None # PUT delayed BACK ON THE QUEUE, IN THE ORDER FOUND, BUT WITH QUERIES FIRST if self.delayed_transactions: for c in reversed(self.delayed_transactions): self.queue.push(c) del self.delayed_transactions[:] if self.delayed_queries: for c in reversed(self.delayed_queries): self.queue.push(c) del self.delayed_queries[:] if has_been_too_long: Log.note("Transaction blockage cleared") def _worker(self, please_stop): try: # MAIN EXECUTION LOOP while not please_stop: command_item = self.queue.pop(till=please_stop) if command_item is None: break try: self._process_command_item(command_item) except Exception as e: Log.warning("worker can not execute command", cause=e) except Exception as e: e = Except.wrap(e) if not please_stop: Log.warning("Problem with sql", cause=e) finally: self.closed = True DEBUG and Log.note("Database is closed") self.db.close() def _process_command_item(self, command_item): query, result, signal, trace, transaction = command_item with Timer("SQL Timing", silent=not DEBUG): if transaction is None: # THIS IS A TRANSACTIONLESS QUERY, DELAY IT IF THERE IS A CURRENT TRANSACTION if self.transaction_stack: with self.locker: if self.too_long is None: self.too_long = Till( seconds=TOO_LONG_TO_HOLD_TRANSACTION) self.too_long.then( self.show_transactions_blocked_warning) self.delayed_queries.append(command_item) return elif self.transaction_stack and self.transaction_stack[-1] not in [ transaction, transaction.parent ]: # THIS TRANSACTION IS NOT THE CURRENT TRANSACTION, DELAY IT with self.locker: if self.too_long is None: self.too_long = Till( seconds=TOO_LONG_TO_HOLD_TRANSACTION) self.too_long.then( self.show_transactions_blocked_warning) self.delayed_transactions.append(command_item) return else: # ENSURE THE CURRENT TRANSACTION IS UP TO DATE FOR THIS query if not self.transaction_stack: # sqlite3 ALLOWS ONLY ONE TRANSACTION AT A TIME DEBUG and Log.note(FORMAT_COMMAND, command=BEGIN) self.db.execute(BEGIN) self.transaction_stack.append(transaction) elif transaction is not self.transaction_stack[-1]: self.transaction_stack.append(transaction) elif transaction.exception and query is not ROLLBACK: result.exception = Except( context=ERROR, template= "Not allowed to continue using a transaction that failed", cause=transaction.exception, trace=trace) signal.release() return try: transaction.do_all() except Exception as e: # DEAL WITH ERRORS IN QUEUED COMMANDS # WE WILL UNWRAP THE OUTER EXCEPTION TO GET THE CAUSE err = Except(context=ERROR, template="Bad call to Sqlite3 while " + FORMAT_COMMAND, params={"command": e.params.current.command}, cause=e.cause, trace=e.params.current.trace) transaction.exception = result.exception = err if query in [COMMIT, ROLLBACK]: self._close_transaction( CommandItem(ROLLBACK, result, signal, trace, transaction)) signal.release() return try: # DEAL WITH END-OF-TRANSACTION MESSAGES if query in [COMMIT, ROLLBACK]: self._close_transaction(command_item) return # EXECUTE QUERY self.last_command_item = command_item DEBUG and Log.note(FORMAT_COMMAND, command=query) curr = self.db.execute(query) result.meta.format = "table" result.header = [d[0] for d in curr.description ] if curr.description else None result.data = curr.fetchall() if DEBUG and result.data: text = convert.table2csv(list(result.data)) Log.note("Result:\n{{data|limit(100)|indent}}", data=text) except Exception as e: e = Except.wrap(e) err = Except(context=ERROR, template="Bad call to Sqlite while " + FORMAT_COMMAND, params={"command": query}, trace=trace, cause=e) result.exception = err if transaction: transaction.exception = err finally: signal.release()