def device_login(self, path=None):
"""
REDIRECT BROWSER TO AUTH0 LOGIN
"""
state = request.args.get("state")
self.session_manager.setup_session(session)
session.code_verifier = bytes2base64URL(Random.bytes(32))
code_challenge = bytes2base64URL(
sha256(session.code_verifier.encode("utf8")))
query = Data(
client_id=self.device.auth0.client_id,
redirect_uri=self.device.auth0.redirect_uri,
state=state,
nonce=bytes2base64URL(Random.bytes(32)),
code_challenge=code_challenge,
response_type="code",
code_challenge_method="S256",
response_mode="query",
audience=self.device.auth0.audience,
scope=self.device.auth0.scope,
)
url = str(
URL("https://" + self.device.auth0.domain + "/authorize",
query=query))
Log.note("Forward browser to {{url}}", url=url)
return redirect(url, code=302)
def output(*args, **kwargs):
response = func(*args, **kwargs)
headers = response.headers
# WATCH OUT FOR THE RUBE GOLDBERG LOGIC!
# https://fetch.spec.whatwg.org/#cors-protocol-and-credentials
origin = URL(flask.request.headers.get("Origin"))
if origin.host:
allow_origin = str(origin)
# allow_origin = origin.scheme + "://" + origin.host
else:
allow_origin = "*"
_setdefault(headers, "Access-Control-Allow-Origin", allow_origin)
_setdefault(headers, "Access-Control-Allow-Credentials", "true")
_setdefault(
headers,
"Access-Control-Allow-Headers",
flask.request.headers.get("Access-Control-Request-Headers"),
)
_setdefault(
headers,
"Access-Control-Allow-Methods", # PLURAL "Methods"
flask.request.headers.get(
"Access-Control-Request-Method"), # SINGULAR "Method"
# "GET, PUT, POST, DELETE, PATCH, OPTIONS"
)
_setdefault(headers, "Content-Type", mimetype.JSON)
_setdefault(
headers,
"Strict-Transport-Security",
"max-age=31536000; includeSubDomains; preload",
)
return response
def _open(self):
""" DO NOT USE THIS UNLESS YOU close() FIRST"""
if self.settings.host.startswith("mysql://"):
# DECODE THE URI: mysql://username:password@host:optional_port/database_name
up = strings.between(self.settings.host, "mysql://", "@")
if ":" in up:
self.settings.username, self.settings.password = unquote(
up).split(":")
else:
self.settings.username = up
url = strings.between(self.settings.host, "@", None)
hp, self.settings.schema = url.split("/", 1)
if ":" in hp:
self.settings.host, self.settings.port = hp.split(":")
self.settings.port = int(self.settings.port)
else:
self.settings.host = hp
# SSL PEM
if self.settings.host in ("localhost", "mysql", '127.0.0.1'):
ssl_context = None
else:
if self.settings.ssl and not self.settings.ssl.pem:
Log.error("Expecting 'pem' property in ssl")
# ssl_context = ssl.create_default_context(**get_ssl_pem_file(self.settings.ssl.pem))
filename = File(".pem") / URL(self.settings.ssl.pem).host
filename.write_bytes(http.get(self.settings.ssl.pem).content)
ssl_context = {"ca": filename.abspath}
try:
self.db = connect(
host=self.settings.host,
port=self.settings.port,
user=coalesce(self.settings.username, self.settings.user),
passwd=coalesce(self.settings.password, self.settings.passwd),
db=coalesce(self.settings.schema, self.settings.db),
read_timeout=coalesce(self.settings.read_timeout,
(EXECUTE_TIMEOUT / 1000) -
10 if EXECUTE_TIMEOUT else None, 5 * 60),
charset=u"utf8",
use_unicode=True,
ssl=ssl_context,
cursorclass=cursors.SSCursor)
except Exception as e:
if self.settings.host.find("://") == -1:
Log.error(u"Failure to connect to {{host}}:{{port}}",
host=self.settings.host,
port=self.settings.port,
cause=e)
else:
Log.error(
u"Failure to connect. PROTOCOL PREFIX IS PROBABLY BAD", e)
self.cursor = None
self.partial_rollback = False
self.transaction_level = 0
self.backlog = [
] # accumulate the write commands so they are sent at once
if self.readonly:
self.begin()
def device_register(self, path=None):
"""
EXPECTING A SIGNED REGISTRATION REQUEST
RETURN JSON WITH url FOR LOGIN
"""
now = Date.now()
expires = now + parse(self.device.register.session['max-age'])
request_body = request.get_data()
signed = json2value(request_body.decode("utf8"))
command = json2value(base642bytes(signed.data).decode("utf8"))
session.public_key = command.public_key
rsa_crypto.verify(signed, session.public_key)
self.session_manager.create_session(session)
session.expires = expires.unix
session.state = bytes2base64URL(crypto.bytes(32))
with self.device.db.transaction() as t:
t.execute(
sql_insert(
self.device.table,
{
"state": session.state,
"session_id": session.session_id
},
))
body = value2json(
Data(
session_id=session.session_id,
interval="5second",
expires=session.expires,
url=URL(
self.device.home,
path=self.device.endpoints.login,
query={"state": session.state},
),
))
response = Response(body,
headers={"Content-Type": mimetype.JSON},
status=200)
response.set_cookie(self.device.register.session.name,
session.session_id,
path=self.device.login.session.path,
domain=self.device.login.session.domain,
expires=expires.format(RFC1123),
secure=self.device.login.session.secure,
httponly=self.device.login.session.httponly)
return response
def __init__(self,
flask_app,
auth0,
permissions,
session_manager,
device=None):
if not auth0.domain:
Log.error("expecting auth0 configuration")
self.auth0 = auth0
self.permissions = permissions
self.session_manager = session_manager
# ATTACH ENDPOINTS TO FLASK APP
endpoints = auth0.endpoints
if not endpoints.login or not endpoints.logout or not endpoints.keep_alive:
Log.error("Expecting paths for login, logout and keep_alive")
add_flask_rule(flask_app, endpoints.login, self.login)
add_flask_rule(flask_app, endpoints.logout, self.logout)
add_flask_rule(flask_app, endpoints.keep_alive, self.keep_alive)
if device:
self.device = device
db = self.device.db = Sqlite(device.db)
if not db.about("device"):
with db.transaction() as t:
t.execute(
sql_create(
"device",
{
"state": "TEXT PRIMARY KEY",
"session_id": "TEXT"
},
))
if device.auth0.redirect_uri != text_type(
URL(device.home, path=device.endpoints.callback)):
Log.error(
"expecting home+endpoints.callback == auth0.redirect_uri")
add_flask_rule(flask_app, device.endpoints.register,
self.device_register)
add_flask_rule(flask_app, device.endpoints.status,
self.device_status)
add_flask_rule(flask_app, device.endpoints.login,
self.device_login)
add_flask_rule(flask_app, device.endpoints.callback,
self.device_callback)
def inner(changeset_id):
# ALWAYS TRY ES FIRST
moves = _get_changeset_from_es(self.moves,
changeset_id).changeset.moves
if moves:
return moves
url = URL(revision.branch.url) / "raw-rev" / changeset_id
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
# THE ENCODING DOES NOT MATTER BECAUSE WE ONLY USE THE '+', '-' PREFIXES IN THE DIFF
moves = http.get(url).content.decode("latin1")
return diff_to_moves(text(moves))
except Exception as e:
Log.warning("could not get unified diff from {{url}}",
url=url,
cause=e)
def __init__(self,
host,
index,
sql_file='metadata.sqlite',
alias=None,
name=None,
port=9200,
kwargs=None):
if hasattr(self, "settings"):
return
self.too_old = TOO_OLD
self.settings = kwargs
self.default_name = coalesce(name, alias, index)
self.es_cluster = elasticsearch.Cluster(kwargs=kwargs)
self.index_does_not_exist = set()
self.todo = Queue("refresh metadata", max=100000, unique=True)
self.index_to_alias = {}
self.es_metadata = Null
self.metadata_last_updated = Date.now() - OLD_METADATA
self.meta = Data()
self.meta.columns = ColumnList(URL(self.es_cluster.settings.host).host)
self.alias_to_query_paths = {
"meta.columns": [ROOT_PATH],
"meta.tables": [ROOT_PATH]
}
self.alias_last_updated = {
"meta.columns": Date.now(),
"meta.tables": Date.now()
}
table_columns = metadata_tables()
self.meta.tables = ListContainer("meta.tables", [],
jx_base.Schema(".", table_columns))
self.meta.columns.extend(table_columns)
# TODO: fix monitor so it does not bring down ES
if ENABLE_META_SCAN:
self.worker = Thread.run("refresh metadata", self.monitor)
else:
self.worker = Thread.run("not refresh metadata", self.not_monitor)
return
def device_register(self, path=None):
"""
EXPECTING A SIGNED REGISTRATION REQUEST
RETURN JSON WITH url FOR LOGIN
"""
now = Date.now().unix
request_body = request.get_data().strip()
signed = json2value(request_body.decode("utf8"))
command = json2value(base642bytes(signed.data).decode("utf8"))
session.public_key = command.public_key
rsa_crypto.verify(signed, session.public_key)
self.session_manager.setup_session(session)
session.expires = now + parse("10minute").seconds
session.state = bytes2base64URL(Random.bytes(32))
with self.device.db.transaction() as t:
t.execute(
sql_insert(
self.device.table,
{
"state": session.state,
"session_id": session.session_id
},
))
response = value2json(
Data(
session_id=session.session_id,
interval="5second",
expiry=session.expires,
url=URL(
self.device.home,
path=self.device.endpoints.login,
query={"state": session.state},
),
))
return Response(response,
headers={"Content-Type": "application/json"},
status=200)
def device_login(self, path=None):
"""
REDIRECT BROWSER TO AUTH0 LOGIN
"""
now = Date.now()
expires = now + parse(self.device.login.session['max-age'])
state = request.args.get("state")
self.session_manager.create_session(session)
session.expires = expires.unix
session.code_verifier = bytes2base64URL(crypto.bytes(32))
code_challenge = bytes2base64URL(
sha256(session.code_verifier.encode("utf8")))
query = Data(
client_id=self.device.auth0.client_id,
redirect_uri=self.device.auth0.redirect_uri,
state=state,
nonce=bytes2base64URL(crypto.bytes(32)),
code_challenge=code_challenge,
response_type="code",
code_challenge_method="S256",
response_mode="query",
audience=self.device.auth0.audience,
scope=self.device.auth0.scope,
)
url = str(
URL("https://" + self.device.auth0.domain + "/authorize",
query=query))
Log.note("Forward browser to {{url}}", url=url)
response = redirect(url, code=302)
response.set_cookie(self.device.login.session.name,
session.session_id,
path=self.device.login.session.path,
domain=self.device.login.session.domain,
expires=expires.format(RFC1123),
secure=self.device.login.session.secure,
httponly=self.device.login.session.httponly)
return response
def inner(changeset_id):
# ALWAYS TRY ES FIRST
json_diff = _get_changeset_from_es(self.repo,
changeset_id).changeset.diff
if json_diff:
return json_diff
url = URL(revision.branch.url) / "raw-rev" / changeset_id
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
response = http.get(url)
try:
diff = response.content.decode("utf8")
except Exception as e:
diff = response.content.decode("latin1")
# File("tests/resources/big.patch").write_bytes(response.content)
json_diff = diff_to_json(diff)
num_changes = _count(c for f in json_diff for c in f.changes)
if json_diff:
if (IGNORE_MERGE_DIFFS
and revision.changeset.description.startswith(
"merge ")):
return None # IGNORE THE MERGE CHANGESETS
elif num_changes < MAX_DIFF_SIZE:
return json_diff
else:
Log.warning(
"Revision at {{url}} has a diff with {{num}} changes, ignored",
url=url,
num=num_changes,
)
for file in json_diff:
file.changes = None
return json_diff
except Exception as e:
Log.warning("could not get unified diff from {{url}}",
url=url,
cause=e)
def device_callback(self, path=None):
# HANDLE BROWESR RETURN FROM AUTH0 LOGIN
error = request.args.get("error")
if error:
Log.error("You did it wrong")
code = request.args.get("code")
state = request.args.get("state")
referer = request.headers.get("Referer")
result = self.device.db.query(
sql_query({
"from": "device",
"select": "session_id",
"where": {
"eq": {
"state": state
}
},
}))
if not result.data:
Log.error("expecting valid state")
device_session_id = result.data[0][0]
# GO BACK TO AUTH0 TO GET TOKENS
token_request = {
"client_id": self.device.auth0.client_id,
"redirect_uri": self.device.auth0.redirect_uri,
"code_verifier": session.code_verifier,
"code": code,
"grant_type": "authorization_code",
}
DEBUG and Log.note("Send token request to Auth0:\n {{request}}",
request=token_request)
auth_response = requests.request(
"POST",
str(URL("https://" + self.device.auth0.domain,
path="oauth/token")),
headers={
"Accept": "application/json",
"Content-Type": "application/json",
# "Referer": str(URL(self.device.auth0.redirect_uri, query={"code": code, "state": state})),
},
data=value2json(token_request),
)
try:
auth_result = wrap(auth_response.json())
except Exception as e:
Log.error("not json {{value}}",
value=auth_response.content,
cause=e)
# VERIFY TOKENS, ADD USER TO DEVICE'S SESSION
user_details = self.verify_opaque_token(auth_result.access_token)
self.session_manager.update_session(
device_session_id,
{"user": self.permissions.get_or_create_user(user_details)},
)
# REMOVE DEVICE SETUP STATE
with self.device.db.transaction() as t:
t.execute(SQL_DELETE + SQL_FROM + quote_column(self.device.table) +
SQL_WHERE + sql_eq(state=state))
Log.note("login complete")
return Response("Login complete. You may close this page", status=200)
def get_ssl_pem_file(url):
filename = File(".pem") / URL(url).host
filename.write_bytes(http.get(url).content)
return {"cafile": filename.abspath}
def inner(changeset_id):
if self.repo.cluster.version.startswith("1.7."):
query = {
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [
{
"prefix": {
"changeset.id": changeset_id
}
},
{
"range": {
"etl.timestamp": {
"gt": MIN_ETL_AGE
}
}
},
]
},
}
},
"size": 1,
}
else:
query = {
"query": {
"bool": {
"must": [
{
"prefix": {
"changeset.id": changeset_id
}
},
{
"range": {
"etl.timestamp": {
"gt": MIN_ETL_AGE
}
}
},
]
}
},
"size": 1,
}
try:
# ALWAYS TRY ES FIRST
with self.repo_locker:
response = self.repo.search(query)
json_diff = response.hits.hits[0]._source.changeset.diff
if json_diff:
return json_diff
except Exception as e:
pass
url = URL(revision.branch.url) / "raw-rev" / changeset_id
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
response = http.get(url)
try:
diff = response.content.decode("utf8")
except Exception as e:
diff = response.content.decode("latin1")
json_diff = diff_to_json(diff)
num_changes = _count(c for f in json_diff for c in f.changes)
if json_diff:
if revision.changeset.description.startswith("merge "):
return None # IGNORE THE MERGE CHANGESETS
elif num_changes < MAX_DIFF_SIZE:
return json_diff
else:
Log.warning(
"Revision at {{url}} has a diff with {{num}} changes, ignored",
url=url,
num=num_changes,
)
for file in json_diff:
file.changes = None
return json_diff
except Exception as e:
Log.warning("could not get unified diff from {{url}}",
url=url,
cause=e)
def setup(app, config):
oauth = OAuth(app)
domain = URL(config.domain)
domain.scheme = "https"
auth0 = oauth.register(
"auth0",
client_id=config.client.id,
client_secret=config.client.secret,
api_base_url=domain,
access_token_url=domain + "oauth/token",
authorize_url=domain + "authorize",
client_kwargs={"scope": "openid profile"},
)
def requires_auth(f):
@decorate(f)
def decorated(*args, **kwargs):
if PROFILE_KEY not in session:
return redirect("/login")
return f(*args, **kwargs)
return decorated
@register_thread
def login():
output = auth0.authorize_redirect(
redirect_uri=config.callback,
audience=config.audience
)
return output
@register_thread
def logout():
session.clear()
return_url = url_for("home", _external=True)
params = {
"returnTo": return_url,
"client_id": config.client.id,
}
return redirect(auth0.api_base_url + "/v2/logout?" + urlencode(params))
@register_thread
def callback():
try:
auth0.authorize_access_token()
resp = auth0.get("userinfo")
userinfo = resp.json()
session[JWT_PAYLOAD] = userinfo
session[PROFILE_KEY] = {
"user_id": userinfo["sub"],
"name": userinfo["name"],
"picture": userinfo["picture"],
}
return redirect("/dashboard")
except Exception as e:
Log.warning("problem with callback {{url}}", url=request, cause=e)
raise e
return requires_auth, login, logout, callback
def _get_source_code_from_hg(self, revision, file_path):
response = http.get(
URL(revision.branch.url) / "raw-file" / revision.changeset.id /
file_path)
return response.content.decode("utf8", "replace")
def login(self, please_stop=None):
"""
WILL REGISTER THIS DEVICE, AND SHOW A QR-CODE TO LOGIN
WILL POLL THE SERVICE ENDPOINT UNTIL LOGIN IS COMPLETED, OR FAILED
:param please_stop: SIGNAL TO STOP EARLY
:return: SESSION THAT CAN BE USED TO SEND AUTHENTICATED REQUESTS
"""
# SEND PUBLIC KEY
now = Date.now().unix
self.session = requests.Session()
signed = rsa_crypto.sign(
Data(public_key=self.public_key, timestamp=now), self.private_key)
DEBUG and Log.note("register (unsigned)\n{{request|json}}",
request=rsa_crypto.verify(signed, self.public_key))
DEBUG and Log.note("register (signed)\n{{request|json}}",
request=signed)
try:
response = self.session.request(
"POST",
str(URL(self.config.service) / self.config.endpoints.register),
data=value2json(signed))
except Exception as e:
raise Log.error("problem registering device", cause=e)
device = wrap(response.json())
DEBUG and Log.note("response:\n{{response}}", response=device)
device.interval = parse(device.interval).seconds
expires = Till(till=parse(device.expiry).unix)
cookie = self.session.cookies.get(self.config.cookie.name)
if not cookie:
Log.error("expecting a session cookie")
# SHOW URL AS QR CODE
image = text2QRCode(device.url)
sys.stdout.write("\n\nLogin using thie URL:\n")
sys.stdout.write(device.url + CR)
sys.stdout.write(image)
while not please_stop and not expires:
Log.note("waiting for login...")
try:
now = Date.now()
signed = rsa_crypto.sign(Data(timestamp=now, session=cookie),
self.private_key)
url = URL(self.config.service) / self.config.endpoints.status
DEBUG and Log.note("ping (unsigned) {{url}}\n{{request|json}}",
url=url,
request=rsa_crypto.verify(
signed, self.public_key))
response = self.session.request("POST",
url,
data=value2json(signed))
ping = wrap(response.json())
DEBUG and Log.note("response\n{{response|json}}",
response=ping)
if ping.status == "verified":
return self.session
if not ping.try_again:
Log.note("Failed to login {{reason}}", reason=ping.status)
return
except Exception as e:
Log.warning(
"problem calling {{url}}",
url=URL(self.config.service) /
self.config.endpoints.status,
cause=e,
)
(Till(seconds=device.interval) | please_stop | expires).wait()
return self.session
def inner(changeset_id):
if self.moves.cluster.version.startswith("1.7."):
query = {
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"and": [
{
"prefix": {
"changeset.id": changeset_id
}
},
{
"range": {
"etl.timestamp": {
"gt": MIN_ETL_AGE
}
}
},
]
},
}
},
"size": 1,
}
else:
query = {
"query": {
"bool": {
"must": [
{
"prefix": {
"changeset.id": changeset_id
}
},
{
"range": {
"etl.timestamp": {
"gt": MIN_ETL_AGE
}
}
},
]
}
},
"size": 1,
}
try:
# ALWAYS TRY ES FIRST
with self.moves_locker:
response = self.moves.search(query)
moves = response.hits.hits[0]._source.changeset.moves
if moves:
return moves
except Exception as e:
pass
url = URL(revision.branch.url) / "raw-rev" / changeset_id
DEBUG and Log.note("get unified diff from {{url}}", url=url)
try:
moves = http.get(url).content.decode(
"latin1"
) # THE ENCODING DOES NOT MATTER BECAUSE WE ONLY USE THE '+', '-' PREFIXES IN THE DIFF
return diff_to_moves(text(moves))
except Exception as e:
Log.warning("could not get unified diff from {{url}}",
url=url,
cause=e)
from jx_elasticsearch.es52.agg_op import aggop_to_es_queries
from mo_dots import listwrap, unwrap, Null, to_data, coalesce
from mo_files import TempFile, URL, mimetype
from mo_future import first
from mo_json import value2json
from mo_logs import Log, Except
from mo_math import randoms
from mo_testing.fuzzytestcase import assertAlmostEqual
from mo_threads import Thread
from mo_times import Timer, Date
from pyLibrary.aws.s3 import Connection
DEBUG = False
MAX_CHUNK_SIZE = 5000
MAX_PARTITIONS = 200
URL_PREFIX = URL(
"https://active-data-query-results.s3-us-west-2.amazonaws.com")
S3_CONFIG = Null
def is_bulk_agg(esq, query):
# ONLY ACCEPTING ONE DIMENSION AT THIS TIME
if not S3_CONFIG:
return False
if query.destination not in {"s3", "url"}:
return False
if query.format not in {"list", "table"}:
return False
if len(listwrap(query.groupby)) != 1:
return False
gb = first(_normalize_group(first(listwrap(query.groupby)), 0,
def _get_from_hg(self,
revision,
locale=None,
get_diff=False,
get_moves=True):
# RATE LIMIT CALLS TO HG (CACHE MISSES)
next_cache_miss = self.last_cache_miss + (
Random.float(WAIT_AFTER_CACHE_MISS * 2) * SECOND)
self.last_cache_miss = Date.now()
if next_cache_miss > self.last_cache_miss:
Log.note(
"delaying next hg call for {{seconds|round(decimal=1)}} seconds",
seconds=next_cache_miss - self.last_cache_miss,
)
Till(till=next_cache_miss.unix).wait()
# CLEAN UP BRANCH NAME
found_revision = copy(revision)
if isinstance(found_revision.branch, (text, binary_type)):
lower_name = found_revision.branch.lower()
else:
lower_name = found_revision.branch.name.lower()
if not lower_name:
Log.error("Defective revision? {{rev|json}}",
rev=found_revision.branch)
b = found_revision.branch = self.branches[(lower_name, locale)]
if not b:
b = found_revision.branch = self.branches[(lower_name,
DEFAULT_LOCALE)]
if not b:
Log.warning(
"can not find branch ({{branch}}, {{locale}})",
branch=lower_name,
locale=locale,
)
return Null
# REFRESH BRANCHES, IF TOO OLD
if Date.now() - Date(b.etl.timestamp) > _hg_branches.OLD_BRANCH:
self.branches = _hg_branches.get_branches(kwargs=self.settings)
# FIND THE PUSH
push = self._get_push(found_revision.branch,
found_revision.changeset.id)
id12 = found_revision.changeset.id[0:12]
base_url = URL(found_revision.branch.url)
with Explanation("get revision from {{url}}",
url=base_url,
debug=DEBUG):
raw_rev2 = Null
automation_details = Null
try:
raw_rev1 = self._get_raw_json_info((base_url / "json-info") +
{"node": id12})
raw_rev2 = self._get_raw_json_rev(base_url / "json-rev" / id12)
automation_details = self._get_raw_json_rev(
base_url / "json-automationrelevance" / id12)
except Exception as e:
if "Hg denies it exists" in e:
raw_rev1 = Data(node=revision.changeset.id)
else:
raise e
raw_rev3_changeset = first(r for r in automation_details.changesets
if r.node[:12] == id12)
if last(automation_details.changesets) != raw_rev3_changeset:
Log.note("interesting")
output = self._normalize_revision(
set_default(raw_rev1, raw_rev2, raw_rev3_changeset),
found_revision,
push,
get_diff,
get_moves,
)
if output.push.date >= Date.now() - MAX_TODO_AGE:
self.todo.extend([
(output.branch, listwrap(output.parents), None),
(output.branch, listwrap(output.children), None),
(
output.branch,
listwrap(output.backsoutnodes),
output.push.date,
),
])
if not get_diff: # DIFF IS BIG, DO NOT KEEP IT IF NOT NEEDED
output.changeset.diff = None
if not get_moves:
output.changeset.moves = None
return output