def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None): self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD) self.rate = coalesce(rate, HG_REQUEST_PER_SECOND) self.cache_locker = Lock() self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR self.no_cache = {} # VERY SHORT TERM CACHE self.workers = [] self.todo = Queue(APP_NAME+" todo") self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds)) self.url = URL(source.url) self.db = Sqlite(database) self.inbound_rate = RateLogger("Inbound") self.outbound_rate = RateLogger("hg.mo") if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data: with self.db.transaction() as t: t.execute( "CREATE TABLE cache (" " path TEXT PRIMARY KEY, " " headers TEXT, " " response TEXT, " " timestamp REAL " ")" ) self.threads = [ Thread.run(APP_NAME+" worker" + text_type(i), self._worker) for i in range(CONCURRENCY) ] self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter) self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def __init__(self, name, db=None, uid=UID, kwargs=None): """ :param name: NAME FOR THIS TABLE :param db: THE DB TO USE :param uid: THE UNIQUE INDEX FOR THIS TABLE :return: HANDLE FOR TABLE IN db """ global _config if isinstance(db, Sqlite): self.db = db else: self.db = db = Sqlite(db) if not _config: # REGISTER sqlite AS THE DEFAULT CONTAINER TYPE from jx_base.container import config as _config if not _config.default: _config.default = {"type": "sqlite", "settings": {"db": db}} ns = Namespace(db=db) self.facts = ns.create_or_replace_facts(fact_name=name) self._next_guid = generateGuid() self._next_uid = 1 self._make_digits_table() self.uid_accessor = jx.get(uid)
def __init__(self, name, db=None, uid=UID, kwargs=None): """ :param name: NAME FOR THIS TABLE :param db: THE DB TO USE :param uid: THE UNIQUE INDEX FOR THIS TABLE :return: HANDLE FOR TABLE IN db """ global _config if db: self.db = db else: self.db = db = Sqlite() if not _config: # REGISTER sqlite AS THE DEFAULT CONTAINER TYPE from jx_base.container import config as _config if not _config.default: _config.default = {"type": "sqlite", "settings": {"db": db}} self.sf = Snowflake(fact=name, uid=uid, db=db) self._next_guid = generateGuid() self._next_uid = 1 self._make_digits_table() self.uid_accessor = jx.get(self.sf.uid)
def __init__(self, db, kwargs): if is_data(db): self.db = Sqlite(db) elif isinstance(db, Sqlite): self.db = db else: Log.error("Bad db parameter") if not self.db.about(PERMISSION_TABLE): self.setup() self.next_id = id_generator(self.db)
def test_bad_exists_properties(self): test = { "data": [{ "~e~": 1 }, { "~e~": 1 }], "query": { "from": TEST_TABLE, "select": [{ "name": "count", "aggregate": "count" }], }, "expecting_list": { "meta": { "format": "value" }, "data": { "count": 2 } } } subtest = wrap(test) cont = self.utils.fill_container(subtest, typed=False) db = Sqlite(filename="metadata.localhost.sqlite") try: with db.transaction() as t: t.execute( "insert into " + quote_column("meta.columns") + "(name, es_type, jx_type, nested_path, es_column, es_index, last_updated) VALUES " + quote_set([ ".", "object", "exists", '["."]', ".", cont.alias, Date.now() ])) except Exception as e: pass try: with db.transaction() as t: t.execute( "insert into " + quote_column("meta.columns") + "(name, es_type, jx_type, nested_path, es_column, es_index, last_updated) VALUES " + quote_set([ "~e~", "long", "exists", '["."]', "~e~", cont.alias, Date.now() ])) except Exception as e: pass self.utils.send_queries(subtest)
def __init__(self, endpoint, push_queue=None, timeout=30, db=None, kwargs=None): self.enabled = True self.num_bad_requests = 0 self.endpoint = endpoint self.timeout = timeout self.push_queue = aws.Queue(push_queue) if push_queue else None self.config = kwargs self.db = Sqlite(filename=coalesce(db.filename, "tuid_client.sqlite"), kwargs=db) if not self.db.query("SELECT name FROM sqlite_master WHERE type='table';").data: with self.db.transaction() as transaction: self._setup(transaction)
def test_transactionqueries(): db = Sqlite() db.query("CREATE TABLE my_table (value TEXT)") with db.transaction() as t: t.execute("INSERT INTO my_table (value) VALUES ('a')") try: result1 = db.query("SELECT * FROM my_table") assert False except Exception as e: assert DOUBLE_TRANSACTION_ERROR in e result2 = t.query("SELECT * FROM my_table") assert result2.data[0][0] == 'a'
def test_nested_transaction2(): db = Sqlite() db.query("CREATE TABLE my_table (value TEXT)") with db.transaction() as t: with db.transaction() as t2: t2.execute("INSERT INTO my_table VALUES ('b')") result = t2.query("SELECT * FROM my_table") assert len(result.data) == 1 assert result.data[0][0] == 'b' t.execute("INSERT INTO my_table VALUES ('a')") _teardown(db, {})
def _setup(): threads = Data() signals = Data() db = Sqlite() db.query("CREATE TABLE my_table (value TEXT)") for name in ["a", "b"]: signals[name] = [{ "begin": Signal(), "done": Signal() } for _ in range(4)] threads[name] = Thread.run(name, _work, name, db, signals[name]) return db, threads, signals
def __init__(self, flask_app, auth0, permissions, session_manager, device=None): if not auth0.domain: Log.error("expecting auth0 configuration") self.auth0 = auth0 self.permissions = permissions self.session_manager = session_manager # ATTACH ENDPOINTS TO FLASK APP endpoints = auth0.endpoints if not endpoints.login or not endpoints.logout or not endpoints.keep_alive: Log.error("Expecting paths for login, logout and keep_alive") add_flask_rule(flask_app, endpoints.login, self.login) add_flask_rule(flask_app, endpoints.logout, self.logout) add_flask_rule(flask_app, endpoints.keep_alive, self.keep_alive) if device: self.device = device db = self.device.db = Sqlite(device.db) if not db.about("device"): with db.transaction() as t: t.execute( sql_create( "device", { "state": "TEXT PRIMARY KEY", "session_id": "TEXT" }, )) if device.auth0.redirect_uri != text_type( URL(device.home, path=device.endpoints.callback)): Log.error( "expecting home+endpoints.callback == auth0.redirect_uri") add_flask_rule(flask_app, device.endpoints.register, self.device_register) add_flask_rule(flask_app, device.endpoints.status, self.device_status) add_flask_rule(flask_app, device.endpoints.login, self.device_login) add_flask_rule(flask_app, device.endpoints.callback, self.device_callback)
def __init__(self, flask_app, db, cookie, table="sessions"): global SINGLTON if SINGLTON: Log.error("Can only handle one session manager at a time") SINGLTON = self if is_data(db): self.db = Sqlite(db) else: self.db = db self.table = table self.cookie = cookie self.cookie.max_lifetime = parse(self.cookie.max_lifetime) self.cookie.inactive_lifetime = parse(self.cookie.inactive_lifetime) if not self.db.about(self.table): self.setup() Thread.run("session monitor", self.monitor)
def __init__(self, db=None): global _config if isinstance(db, Sqlite): self.db = db else: self.db = db = Sqlite(db) self.db.create_new_functions() # creating new functions: regexp if not _config: # REGISTER sqlite AS THE DEFAULT CONTAINER TYPE from jx_base.container import config as _config if not _config.default: _config.default = {"type": "sqlite", "settings": {"db": db}} self.setup() self.ns = Namespace(db=db) self.about = QueryTable("meta.about", self) self.next_uid = ( self._gen_ids().__next__ ) # A DELIGHTFUL SOURCE OF UNIQUE INTEGERS
def __init__(self, name, db=None, uid=GUID, exists=False, kwargs=None): """ :param name: NAME FOR THIS TABLE :param db: THE DB TO USE :param uid: THE UNIQUE INDEX FOR THIS TABLE :return: HANDLE FOR TABLE IN db """ global _config Container.__init__(self, frum=None) if db: self.db = db else: self.db = db = Sqlite() if not _config: from pyLibrary.queries.containers import config as _config if not _config.default: _config.default = {"type": "sqlite", "settings": {"db": db}} self.name = name self.uid = listwrap(uid) self._next_uid = 1 self._make_digits_table() self.uid_accessor = jx.get(self.uid) self.nested_tables = OrderedDict( ) # MAP FROM NESTED PATH TO Table OBJECT, PARENTS PROCEED CHILDREN self.nested_tables["."] = self self.columns = Index( keys=[join_field(["names", self.name])] ) # MAP FROM DOCUMENT ABS PROPERTY NAME TO THE SET OF SQL COLUMNS IT REPRESENTS (ONE FOR EACH REALIZED DATATYPE) if not exists: for u in self.uid: if u == GUID: pass else: c = Column(names={name: u}, type="string", es_column=typed_column(u, "string"), es_index=name) self.add_column_to_schema(self.nested_tables, c) command = ("CREATE TABLE " + quote_table(name) + "(" + (",".join([quoted_UID + " INTEGER"] + [ _quote_column(c) + " " + sql_types[c.type] for u, cs in self.columns.items() for c in cs ])) + ", PRIMARY KEY (" + (", ".join([quoted_UID] + [ _quote_column(c) for u in self.uid for c in self.columns[u] ])) + "))") self.db.execute(command) else: # LOAD THE COLUMNS command = "PRAGMA table_info(" + quote_table(name) + ")" details = self.db.query(command) for r in details: cname = untyped_column(r[1]) ctype = r[2].lower() column = Column(names={name: cname}, type=ctype, nested_path=['.'], es_column=typed_column(cname, ctype), es_index=name) self.add_column_to_schema(self.columns, column)
def __init__(self, config): self.db = Sqlite(config)
def __init__(self, name, db=None, uid=UID_PREFIX + "id", exists=False): """ :param name: NAME FOR THIS TABLE :param db: THE DB TO USE :param uid: THE UNIQUE INDEX FOR THIS TABLE :return: HANDLE FOR TABLE IN db """ Container.__init__(self, frum=None) if db: self.db = db else: self.db = db = Sqlite() self.name = name self.uid = listwrap(uid) self.columns = {} for u in uid: if not self.columns.get(u, None): cs = self.columns[u] = set() if u.startswith(UID_PREFIX): cs.add( Column(name=u, table=name, type="integer", es_column=typed_column(u, "integer"), es_index=name)) else: cs.add( Column(name=u, table=name, type="text", es_column=typed_column(u, "text"), es_index=name)) self.uid_accessor = jx.get(self.uid) self.nested_tables = {} # MAP FROM TABLE NAME TO Table OBJECT if exists: # LOAD THE COLUMNS command = "PRAGMA table_info(" + quote_table(name) + ")" details = self.db.query(command) self.columns = {} for r in details: cname = untyped_column(r[1]) ctype = r[2].lower() column = Column(name=cname, table=name, type=ctype, es_column=typed_column(cname, ctype), es_index=name) cs = self.columns.get(name, Null) if not cs: cs = self.columns[name] = set() cs.add(column) else: command = "CREATE TABLE " + quote_table(name) + "(" + \ (",".join(_quote_column(c) + " " + c.type for u, cs in self.columns.items() for c in cs)) + \ ", PRIMARY KEY (" + \ (", ".join(_quote_column(c) for u in self.uid for c in self.columns[u])) + \ "))" self.db.execute(command)
@verify_user def private_scoped(user): """A valid access token and an appropriate scope are required to access this route """ if requires_scope(config.auth0.scope): response = ( "Hello from a private endpoint! You need to be authenticated and have a scope of " + config.auth0.scope + " to see this." ) return jsonify(message=response) Log.error("You don't have access to {{scope}}", scope=config.auth0.scope, code=403) add_flask_rule(APP, "/api/public", public) add_flask_rule(APP, "/api/private", private) add_flask_rule(APP, "/api/private-scoped", private_scoped) config = startup.read_settings() constants.set(config.constants) Log.start(config.debug) session_manager = setup_flask_session(APP, config.session) perm = Permissions(Sqlite(config.permissions.store)) auth = Authenticator(APP, config.auth0, perm, session_manager) Log.note("start servers") setup_flask_ssl(APP, config.flask) APP.run(**config.flask)