def __init__(self, rate=None, amortization_period=None, source=None, database=None, kwargs=None):
self.amortization_period = coalesce(amortization_period, AMORTIZATION_PERIOD)
self.rate = coalesce(rate, HG_REQUEST_PER_SECOND)
self.cache_locker = Lock()
self.cache = {} # MAP FROM url TO (ready, headers, response, timestamp) PAIR
self.no_cache = {} # VERY SHORT TERM CACHE
self.workers = []
self.todo = Queue(APP_NAME+" todo")
self.requests = Queue(APP_NAME + " requests", max=int(self.rate * self.amortization_period.seconds))
self.url = URL(source.url)
self.db = Sqlite(database)
self.inbound_rate = RateLogger("Inbound")
self.outbound_rate = RateLogger("hg.mo")
if not self.db.query("SELECT name FROM sqlite_master WHERE type='table'").data:
with self.db.transaction() as t:
t.execute(
"CREATE TABLE cache ("
" path TEXT PRIMARY KEY, "
" headers TEXT, "
" response TEXT, "
" timestamp REAL "
")"
)
self.threads = [
Thread.run(APP_NAME+" worker" + text_type(i), self._worker)
for i in range(CONCURRENCY)
]
self.limiter = Thread.run(APP_NAME+" limiter", self._rate_limiter)
self.cleaner = Thread.run(APP_NAME+" cleaner", self._cache_cleaner)
def __init__(self, name, db=None, uid=UID, kwargs=None):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
global _config
if isinstance(db, Sqlite):
self.db = db
else:
self.db = db = Sqlite(db)
if not _config:
# REGISTER sqlite AS THE DEFAULT CONTAINER TYPE
from jx_base.container import config as _config
if not _config.default:
_config.default = {"type": "sqlite", "settings": {"db": db}}
ns = Namespace(db=db)
self.facts = ns.create_or_replace_facts(fact_name=name)
self._next_guid = generateGuid()
self._next_uid = 1
self._make_digits_table()
self.uid_accessor = jx.get(uid)
def __init__(self, name, db=None, uid=UID, kwargs=None):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
global _config
if db:
self.db = db
else:
self.db = db = Sqlite()
if not _config:
# REGISTER sqlite AS THE DEFAULT CONTAINER TYPE
from jx_base.container import config as _config
if not _config.default:
_config.default = {"type": "sqlite", "settings": {"db": db}}
self.sf = Snowflake(fact=name, uid=uid, db=db)
self._next_guid = generateGuid()
self._next_uid = 1
self._make_digits_table()
self.uid_accessor = jx.get(self.sf.uid)
def __init__(self, db, kwargs):
if is_data(db):
self.db = Sqlite(db)
elif isinstance(db, Sqlite):
self.db = db
else:
Log.error("Bad db parameter")
if not self.db.about(PERMISSION_TABLE):
self.setup()
self.next_id = id_generator(self.db)
def test_bad_exists_properties(self):
test = {
"data": [{
"~e~": 1
}, {
"~e~": 1
}],
"query": {
"from": TEST_TABLE,
"select": [{
"name": "count",
"aggregate": "count"
}],
},
"expecting_list": {
"meta": {
"format": "value"
},
"data": {
"count": 2
}
}
}
subtest = wrap(test)
cont = self.utils.fill_container(subtest, typed=False)
db = Sqlite(filename="metadata.localhost.sqlite")
try:
with db.transaction() as t:
t.execute(
"insert into " + quote_column("meta.columns") +
"(name, es_type, jx_type, nested_path, es_column, es_index, last_updated) VALUES "
+ quote_set([
".", "object", "exists", '["."]', ".", cont.alias,
Date.now()
]))
except Exception as e:
pass
try:
with db.transaction() as t:
t.execute(
"insert into " + quote_column("meta.columns") +
"(name, es_type, jx_type, nested_path, es_column, es_index, last_updated) VALUES "
+ quote_set([
"~e~", "long", "exists", '["."]', "~e~", cont.alias,
Date.now()
]))
except Exception as e:
pass
self.utils.send_queries(subtest)
def __init__(self, endpoint, push_queue=None, timeout=30, db=None, kwargs=None):
self.enabled = True
self.num_bad_requests = 0
self.endpoint = endpoint
self.timeout = timeout
self.push_queue = aws.Queue(push_queue) if push_queue else None
self.config = kwargs
self.db = Sqlite(filename=coalesce(db.filename, "tuid_client.sqlite"), kwargs=db)
if not self.db.query("SELECT name FROM sqlite_master WHERE type='table';").data:
with self.db.transaction() as transaction:
self._setup(transaction)
def test_transactionqueries():
db = Sqlite()
db.query("CREATE TABLE my_table (value TEXT)")
with db.transaction() as t:
t.execute("INSERT INTO my_table (value) VALUES ('a')")
try:
result1 = db.query("SELECT * FROM my_table")
assert False
except Exception as e:
assert DOUBLE_TRANSACTION_ERROR in e
result2 = t.query("SELECT * FROM my_table")
assert result2.data[0][0] == 'a'
def test_nested_transaction2():
db = Sqlite()
db.query("CREATE TABLE my_table (value TEXT)")
with db.transaction() as t:
with db.transaction() as t2:
t2.execute("INSERT INTO my_table VALUES ('b')")
result = t2.query("SELECT * FROM my_table")
assert len(result.data) == 1
assert result.data[0][0] == 'b'
t.execute("INSERT INTO my_table VALUES ('a')")
_teardown(db, {})
def _setup():
threads = Data()
signals = Data()
db = Sqlite()
db.query("CREATE TABLE my_table (value TEXT)")
for name in ["a", "b"]:
signals[name] = [{
"begin": Signal(),
"done": Signal()
} for _ in range(4)]
threads[name] = Thread.run(name, _work, name, db, signals[name])
return db, threads, signals
def __init__(self,
flask_app,
auth0,
permissions,
session_manager,
device=None):
if not auth0.domain:
Log.error("expecting auth0 configuration")
self.auth0 = auth0
self.permissions = permissions
self.session_manager = session_manager
# ATTACH ENDPOINTS TO FLASK APP
endpoints = auth0.endpoints
if not endpoints.login or not endpoints.logout or not endpoints.keep_alive:
Log.error("Expecting paths for login, logout and keep_alive")
add_flask_rule(flask_app, endpoints.login, self.login)
add_flask_rule(flask_app, endpoints.logout, self.logout)
add_flask_rule(flask_app, endpoints.keep_alive, self.keep_alive)
if device:
self.device = device
db = self.device.db = Sqlite(device.db)
if not db.about("device"):
with db.transaction() as t:
t.execute(
sql_create(
"device",
{
"state": "TEXT PRIMARY KEY",
"session_id": "TEXT"
},
))
if device.auth0.redirect_uri != text_type(
URL(device.home, path=device.endpoints.callback)):
Log.error(
"expecting home+endpoints.callback == auth0.redirect_uri")
add_flask_rule(flask_app, device.endpoints.register,
self.device_register)
add_flask_rule(flask_app, device.endpoints.status,
self.device_status)
add_flask_rule(flask_app, device.endpoints.login,
self.device_login)
add_flask_rule(flask_app, device.endpoints.callback,
self.device_callback)
def __init__(self, flask_app, db, cookie, table="sessions"):
global SINGLTON
if SINGLTON:
Log.error("Can only handle one session manager at a time")
SINGLTON = self
if is_data(db):
self.db = Sqlite(db)
else:
self.db = db
self.table = table
self.cookie = cookie
self.cookie.max_lifetime = parse(self.cookie.max_lifetime)
self.cookie.inactive_lifetime = parse(self.cookie.inactive_lifetime)
if not self.db.about(self.table):
self.setup()
Thread.run("session monitor", self.monitor)
def __init__(self, db=None):
global _config
if isinstance(db, Sqlite):
self.db = db
else:
self.db = db = Sqlite(db)
self.db.create_new_functions() # creating new functions: regexp
if not _config:
# REGISTER sqlite AS THE DEFAULT CONTAINER TYPE
from jx_base.container import config as _config
if not _config.default:
_config.default = {"type": "sqlite", "settings": {"db": db}}
self.setup()
self.ns = Namespace(db=db)
self.about = QueryTable("meta.about", self)
self.next_uid = (
self._gen_ids().__next__
) # A DELIGHTFUL SOURCE OF UNIQUE INTEGERS
def __init__(self, name, db=None, uid=GUID, exists=False, kwargs=None):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
global _config
Container.__init__(self, frum=None)
if db:
self.db = db
else:
self.db = db = Sqlite()
if not _config:
from pyLibrary.queries.containers import config as _config
if not _config.default:
_config.default = {"type": "sqlite", "settings": {"db": db}}
self.name = name
self.uid = listwrap(uid)
self._next_uid = 1
self._make_digits_table()
self.uid_accessor = jx.get(self.uid)
self.nested_tables = OrderedDict(
) # MAP FROM NESTED PATH TO Table OBJECT, PARENTS PROCEED CHILDREN
self.nested_tables["."] = self
self.columns = Index(
keys=[join_field(["names", self.name])]
) # MAP FROM DOCUMENT ABS PROPERTY NAME TO THE SET OF SQL COLUMNS IT REPRESENTS (ONE FOR EACH REALIZED DATATYPE)
if not exists:
for u in self.uid:
if u == GUID:
pass
else:
c = Column(names={name: u},
type="string",
es_column=typed_column(u, "string"),
es_index=name)
self.add_column_to_schema(self.nested_tables, c)
command = ("CREATE TABLE " + quote_table(name) + "(" +
(",".join([quoted_UID + " INTEGER"] + [
_quote_column(c) + " " + sql_types[c.type]
for u, cs in self.columns.items() for c in cs
])) + ", PRIMARY KEY (" + (", ".join([quoted_UID] + [
_quote_column(c) for u in self.uid
for c in self.columns[u]
])) + "))")
self.db.execute(command)
else:
# LOAD THE COLUMNS
command = "PRAGMA table_info(" + quote_table(name) + ")"
details = self.db.query(command)
for r in details:
cname = untyped_column(r[1])
ctype = r[2].lower()
column = Column(names={name: cname},
type=ctype,
nested_path=['.'],
es_column=typed_column(cname, ctype),
es_index=name)
self.add_column_to_schema(self.columns, column)
def __init__(self, config):
self.db = Sqlite(config)
def __init__(self, name, db=None, uid=UID_PREFIX + "id", exists=False):
"""
:param name: NAME FOR THIS TABLE
:param db: THE DB TO USE
:param uid: THE UNIQUE INDEX FOR THIS TABLE
:return: HANDLE FOR TABLE IN db
"""
Container.__init__(self, frum=None)
if db:
self.db = db
else:
self.db = db = Sqlite()
self.name = name
self.uid = listwrap(uid)
self.columns = {}
for u in uid:
if not self.columns.get(u, None):
cs = self.columns[u] = set()
if u.startswith(UID_PREFIX):
cs.add(
Column(name=u,
table=name,
type="integer",
es_column=typed_column(u, "integer"),
es_index=name))
else:
cs.add(
Column(name=u,
table=name,
type="text",
es_column=typed_column(u, "text"),
es_index=name))
self.uid_accessor = jx.get(self.uid)
self.nested_tables = {} # MAP FROM TABLE NAME TO Table OBJECT
if exists:
# LOAD THE COLUMNS
command = "PRAGMA table_info(" + quote_table(name) + ")"
details = self.db.query(command)
self.columns = {}
for r in details:
cname = untyped_column(r[1])
ctype = r[2].lower()
column = Column(name=cname,
table=name,
type=ctype,
es_column=typed_column(cname, ctype),
es_index=name)
cs = self.columns.get(name, Null)
if not cs:
cs = self.columns[name] = set()
cs.add(column)
else:
command = "CREATE TABLE " + quote_table(name) + "(" + \
(",".join(_quote_column(c) + " " + c.type for u, cs in self.columns.items() for c in cs)) + \
", PRIMARY KEY (" + \
(", ".join(_quote_column(c) for u in self.uid for c in self.columns[u])) + \
"))"
self.db.execute(command)
@verify_user
def private_scoped(user):
"""A valid access token and an appropriate scope are required to access this route
"""
if requires_scope(config.auth0.scope):
response = (
"Hello from a private endpoint! You need to be authenticated and have a scope of "
+ config.auth0.scope
+ " to see this."
)
return jsonify(message=response)
Log.error("You don't have access to {{scope}}", scope=config.auth0.scope, code=403)
add_flask_rule(APP, "/api/public", public)
add_flask_rule(APP, "/api/private", private)
add_flask_rule(APP, "/api/private-scoped", private_scoped)
config = startup.read_settings()
constants.set(config.constants)
Log.start(config.debug)
session_manager = setup_flask_session(APP, config.session)
perm = Permissions(Sqlite(config.permissions.store))
auth = Authenticator(APP, config.auth0, perm, session_manager)
Log.note("start servers")
setup_flask_ssl(APP, config.flask)
APP.run(**config.flask)