def setUpClass(cls):
while True:
try:
es = test_jx.global_settings.elasticsearch
http.get_json(URL(es.host, port=es.port))
break
except Exception as e:
e = Except.wrap(e)
if "No connection could be made because the target machine actively refused it" in e or "Connection refused" in e:
Log.alert("Problem connecting")
else:
Log.error("Server raised exception", e)
# REMOVE OLD INDEXES
cluster = elasticsearch.Cluster(test_jx.global_settings.elasticsearch)
aliases = cluster.get_aliases()
for a in aliases:
try:
if a.index.startswith("testing_"):
create_time = Date(
a.index[-15:], "%Y%m%d_%H%M%S"
) # EXAMPLE testing_0ef53e45b320160118_180420
if create_time < Date.now() - 10 * MINUTE:
cluster.delete_index(a.index)
except Exception as e:
Log.warning("Problem removing {{index|quote}}",
index=a.index,
cause=e)
def verify_jwt_token(self, token):
jwks = http.get_json("https://" + self.auth0.domain +
"/.well-known/jwks.json")
unverified_header = jwt.get_unverified_header(token)
algorithm = unverified_header["alg"]
if algorithm != "RS256":
Log.error("Expecting a RS256 signed JWT Access Token")
key_id = unverified_header["kid"]
key = unwrap(first(key for key in jwks["keys"]
if key["kid"] == key_id))
if not key:
Log.error("could not find {{key}}", key=key_id)
try:
return jwt.decode(
token,
key,
algorithms=algorithm,
audience=self.auth0.api.identifier,
issuer="https://" + self.auth0.domain + "/",
)
except jwt.ExpiredSignatureError as e:
Log.error("Token has expired", code=403, cause=e)
except jwt.JWTClaimsError as e:
Log.error(
"Incorrect claims, please check the audience and issuer",
code=403,
cause=e,
)
except Exception as e:
Log.error("Problem parsing", cause=e)
def new_instance(
host,
index,
type=None,
name=None,
port=9200,
read_only=True,
timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests)
wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency)
typed=None,
kwargs=None):
try:
known = known_hosts.get((host, port))
if known:
return known(kwargs=kwargs)
url = URL(host)
url.port = port
status = http.get_json(url, stream=False)
version = status.version.number
if version.startswith(("5.", "6.")):
from jx_elasticsearch.es52 import ES52
type2container.setdefault("elasticsearch", ES52)
known_hosts[(host, port)] = ES52
output = ES52(kwargs=kwargs)
return output
else:
Log.error("No jx interpreter for Elasticsearch {{version}}",
version=version)
except Exception as e:
Log.error("Can not make an interpreter for Elasticsearch", cause=e)
def get_content():
content = http.get_json(result.url)
self.assertEqual(content.header, ["a"])
self.assertEqual(content.meta.format, "table")
sorted_content = jx.sort(content.data, 0)
sorted_expected = [(row.a, ) for row in expected]
self.assertEqual(sorted_content, sorted_expected)
def verify_opaque_token(self, token):
# Opaque Access Token
url = "https://" + self.auth0.domain + "/userinfo"
response = http.get_json(url,
headers={"Authorization": "Bearer " + token})
DEBUG and Log.note("content: {{body|json}}", body=response)
return response
def _get_clog(self, clog_url):
try:
Log.note("Searching through changelog {{url}}", url=clog_url)
clog_obj = http.get_json(clog_url, retry=RETRY)
return clog_obj
except Exception as e:
Log.error(
"Unexpected error getting changset-log for {{url}}: {{error}}",
url=clog_url,
error=e)
def setUpClass(self):
while True:
try:
es = test_jx.global_settings.backend_es
http.get_json(URL(es.host, port=es.port))
break
except Exception as e:
e = Except.wrap(e)
if "No connection could be made because the target machine actively refused it" in e or "Connection refused" in e:
Log.alert("Problem connecting")
Till(seconds=WAIT_AFTER_PROBLEM).wait()
else:
Log.error("Server raised exception", e)
# REMOVE OLD INDEXES
cluster = elasticsearch.Cluster(test_jx.global_settings.backend_es)
aliases = cluster.get_aliases()
for a in aliases:
try:
if a.index.startswith("testing_"):
cluster.delete_index(a.index)
except Exception as e:
Log.warning("Problem removing {{index|quote}}", index=a.index, cause=e)
def _get_and_retry(self, url):
try:
data = http.get_json(**set_default({"url": url}, self.hg))
if data.error.startswith("unknown revision"):
Log.error(UNKNOWN_PUSH,
revision=strings.between(data.error, "'", "'"))
if is_text(data) and data.startswith("unknown revision"):
Log.error(UNKNOWN_PUSH,
revision=strings.between(data, "'", "'"))
# branch.url = _trim(url) # RECORD THIS SUCCESS IN THE BRANCH
return data
except Exception as e:
path = url.split("/")
if path[3] == "l10n-central":
# FROM https://hg.mozilla.org/l10n-central/tr/json-pushes?full=1&changeset=a6eeb28458fd
# TO https://hg.mozilla.org/mozilla-central/json-pushes?full=1&changeset=a6eeb28458fd
path = path[0:3] + ["mozilla-central"] + path[5:]
return self._get_and_retry("/".join(path))
elif len(path) > 5 and path[5] == "mozilla-aurora":
# FROM https://hg.mozilla.org/releases/l10n/mozilla-aurora/pt-PT/json-pushes?full=1&changeset=b44a8c68fc60
# TO https://hg.mozilla.org/releases/mozilla-aurora/json-pushes?full=1&changeset=b44a8c68fc60
path = path[0:4] + ["mozilla-aurora"] + path[7:]
return self._get_and_retry("/".join(path))
elif len(path) > 5 and path[5] == "mozilla-beta":
# FROM https://hg.mozilla.org/releases/l10n/mozilla-beta/lt/json-pushes?full=1&changeset=03fbf7556c94
# TO https://hg.mozilla.org/releases/mozilla-beta/json-pushes?full=1&changeset=b44a8c68fc60
path = path[0:4] + ["mozilla-beta"] + path[7:]
return self._get_and_retry("/".join(path))
elif len(path) > 7 and path[5] == "mozilla-release":
# FROM https://hg.mozilla.org/releases/l10n/mozilla-release/en-GB/json-pushes?full=1&changeset=57f513ab03308adc7aa02cc2ea8d73fe56ae644b
# TO https://hg.mozilla.org/releases/mozilla-release/json-pushes?full=1&changeset=57f513ab03308adc7aa02cc2ea8d73fe56ae644b
path = path[0:4] + ["mozilla-release"] + path[7:]
return self._get_and_retry("/".join(path))
elif len(path) > 5 and path[4] == "autoland":
# FROM https://hg.mozilla.org/build/autoland/json-pushes?full=1&changeset=3ccccf8e5036179a3178437cabc154b5e04b333d
# TO https://hg.mozilla.org/integration/autoland/json-pushes?full=1&changeset=3ccccf8e5036179a3178437cabc154b5e04b333d
path = path[0:3] + ["try"] + path[5:]
return self._get_and_retry("/".join(path))
raise e
def get_content():
content = http.get_json(result.url)
self.assertEqual(content.meta.format, "list")
sorted_content = jx.sort(content.data, "a")
sorted_expected = jx.sort(expected, "a")
self.assertEqual(sorted_content, sorted_expected)
def backend_check():
http.get_json(config.elasticsearch.host + ":" +
text(config.elasticsearch.port))
