def setUpClass(self): while True: try: es = test_jx.global_settings.backend_es http.get_json(URL(es.host, port=es.port)) break except Exception as e: e = Except.wrap(e) if "No connection could be made because the target machine actively refused it" in e or "Connection refused" in e: Log.alert("Problem connecting") else: Log.error("Server raised exception", e) # REMOVE OLD INDEXES cluster = elasticsearch.Cluster(test_jx.global_settings.backend_es) aliases = cluster.get_aliases() for a in aliases: try: if a.index.startswith("testing_"): create_time = Date( a.index[-15:], "%Y%m%d_%H%M%S" ) # EXAMPLE testing_0ef53e45b320160118_180420 if create_time < Date.now() - 10 * MINUTE: cluster.delete_index(a.index) except Exception as e: Log.warning("Problem removing {{index|quote}}", index=a.index, cause=e)
def download_perfherder(desc, repo, id, dummy, framework): sig_result = http.get_json( "https://treeherder.mozilla.org/api/project/" + repo + "/performance/signatures/?format=json&framework=" + str(framework) + "&id=" + str(id) ) signature = first(sig_result.keys()) data_result = http.get_json( "https://treeherder.mozilla.org/api/project/" + repo + "/performance/data/?signatures=" + signature ) Log.note( "{{result|json}}", result={ "name": desc, "data": jx.run({ "from": ListContainer("data", data_result[signature]), "sort": "push_timestamp", "select": "value" }).data }, )
def __init__(self, hg, rate_limit, use_cache=True, cache=None, settings=None): self.settings = settings self.failure_classification = {c.id: c.name for c in http.get_json(FAILURE_CLASSIFICATION_URL)} self.repo = {c.id: c.name for c in http.get_json(REPO_URL)} self.hg = hg self.cache = elasticsearch.Cluster(cache).get_or_create_index(cache) self.locker = Lock() self.pending = {} self.rate_locker = Lock() self.request_times = [0] * rate_limit self.request_pointer = 0
def verify_jwt_token(self, token): jwks = http.get_json("https://" + self.auth0.domain + "/.well-known/jwks.json") unverified_header = jwt.get_unverified_header(token) algorithm = unverified_header["alg"] if algorithm != "RS256": Log.error("Expecting a RS256 signed JWT Access Token") key_id = unverified_header["kid"] key = unwrap(first(key for key in jwks["keys"] if key["kid"] == key_id)) if not key: Log.error("could not find {{key}}", key=key_id) try: return jwt.decode( token, key, algorithms=algorithm, audience=self.auth0.api.identifier, issuer="https://" + self.auth0.domain + "/", ) except jwt.ExpiredSignatureError as e: Log.error("Token has expired", code=403, cause=e) except jwt.JWTClaimsError as e: Log.error( "Incorrect claims, please check the audience and issuer", code=403, cause=e, ) except Exception as e: Log.error("Problem parsing", cause=e)
def new_instance( host, index, type=None, name=None, port=9200, read_only=True, timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests) wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency) typed=None, kwargs=None): try: known = known_hosts.get((host, port)) if known: return known(kwargs=kwargs) url = URL(host) url.port = port status = http.get_json(url, stream=False) version = status.version.number if version.startswith(("5.", "6.")): from jx_elasticsearch.es52 import ES52 type2container.setdefault("elasticsearch", ES52) known_hosts[(host, port)] = ES52 output = ES52(kwargs=kwargs) return output else: Log.error("No jx interpreter for Elasticsearch {{version}}", version=version) except Exception as e: Log.error("Can not make an interpreter for Elasticsearch", cause=e)
def verify_opaque_token(self, token): # Opaque Access Token url = "https://" + self.auth0.domain + "/userinfo" response = http.get_json(url, headers={"Authorization": "Bearer " + token}) DEBUG and Log.note("content: {{body|json}}", body=response) return response
def _setup_grcov(self): sudo("apt-get install -y gcc") response = http.get_json("https://api.github.com/repos/marco-c/grcov/releases/latest") with cd("~/ActiveData-ETL"): for asset in response.assets: if self.settings.grcov.platform in asset.browser_download_url: run("wget "+asset.browser_download_url) run(expand_template("tar xf grcov-{{platform}}.tar.bz2", self.settings.grcov)) run(expand_template("rm grcov-{{platform}}.tar.bz2", self.settings.grcov))
def _get_clog(self, clog_url): try: Log.note("Searching through changelog {{url}}", url=clog_url) clog_obj = http.get_json(clog_url, retry=RETRY) return clog_obj except Exception as e: Log.error( "Unexpected error getting changset-log for {{url}}: {{error}}", url=clog_url, error=e)
def _rate_limited_get_json(self, *args, **kwargs): now = Date.now().unix with self.rate_locker: if self.request_times[self.request_pointer] >= now - 1: Log.note("Rate limiting") Thread.sleep(seconds=self.request_times[self.request_pointer] - now + 1) self.request_times[self.request_pointer] = now self.request_pointer += 1 self.request_pointer %= len(self.request_times) return http.get_json(*args, **kwargs)
def __init__(self, hg, rate_limit, use_cache=True, cache=None, settings=None): self.settings = settings self.failure_classification = { c.id: c.name for c in http.get_json(FAILURE_CLASSIFICATION_URL) } self.repo = {c.id: c.name for c in http.get_json(REPO_URL)} self.hg = hg self.cache = elasticsearch.Cluster(cache).get_or_create_index(cache) self.locker = Lock() self.pending = {} self.rate_locker = Lock() self.request_times = [0] * rate_limit self.request_pointer = 0
def _get_clog(self, clog_url): try: Log.note("Searching through changelog {{url}}", url=clog_url) clog_obj = http.get_json(clog_url, retry=RETRY) return clog_obj except Exception as e: Log.error( "Unexpected error getting changset-log for {{url}}: {{error}}", url=clog_url, error=e )
def _rate_limited_get_json(self, *args, **kwargs): now = Date.now().unix with self.rate_locker: if self.request_times[self.request_pointer] >= now - 1: Log.note("Rate limiting") Thread.sleep(seconds=self.request_times[self.request_pointer] - now + 1) self.request_times[self.request_pointer] = now self.request_pointer += 1 self.request_pointer %= len(self.request_times) return http.get_json(*args, **kwargs)
def get_job_classification(self, branch, revision): results = http.get_json(expand_template(RESULT_SET_URL, {"branch": branch, "revision": revision[0:12:]})) for r in results.results: jobs = http.get_json(expand_template(JOBS_URL, {"branch": branch, "result_set_id": r.id})) for j in jobs: notes = http.get_json(expand_template(NOTES_URL, {"branch": branch, "job_id": j.id})) for n in notes: if not n.note: continue Log.note( "{{note|json}}", note={ "job_id": j.id, "result_set_id": r.id, "branch": branch, "revision": r.revision, "failure_classification_id": j.failure_classification_id, "result": j.result, "note_timestamp": n.timestamp, "note": n.note } )
def _setup_grcov(self): sudo("apt-get install -y gcc") response = http.get_json( "https://api.github.com/repos/marco-c/grcov/releases/latest") with cd("~/ActiveData-ETL"): for asset in response.assets: if self.settings.grcov.platform in asset.browser_download_url: run("wget " + asset.browser_download_url) run( expand_template("tar xf grcov-{{platform}}.tar.bz2", self.settings.grcov)) run( expand_template("rm grcov-{{platform}}.tar.bz2", self.settings.grcov))
def new_instance( host, index, type=None, name=None, port=9200, read_only=True, timeout=None, # NUMBER OF SECONDS TO WAIT FOR RESPONSE, OR SECONDS TO WAIT FOR DOWNLOAD (PASSED TO requests) wait_for_active_shards=1, # ES WRITE CONSISTENCY (https://www.elastic.co/guide/en/elasticsearch/reference/1.7/docs-index_.html#index-consistency) typed=None, kwargs=None ): try: known = known_hosts.get((host, port)) if known: return known(kwargs=kwargs) url = URL(host) url.port = port status = http.get_json(url, stream=False) version = status.version.number if version.startswith("1."): from jx_elasticsearch.es14 import ES14 type2container.setdefault("elasticsearch", ES14) known_hosts[(host, port)] = ES14 output = ES14(kwargs=kwargs) return output elif version.startswith(("5.", "6.")): from jx_elasticsearch.es52 import ES52 type2container.setdefault("elasticsearch", ES52) known_hosts[(host, port)] = ES52 output = ES52(kwargs=kwargs) return output else: Log.error("No jx interpreter for Elasticsearch {{version}}", version=version) except Exception as e: Log.error("Can not make an interpreter for Elasticsearch", cause=e)
def queue_consumer(client, pull_queue, please_stop=None, kwargs=None): queue = aws.Queue(pull_queue) client = TuidClient(client) try_revs = {} test_try_revs = True #while len(queue) > 0: # request = queue.pop(till=please_stop) # if request: # Log.note("Popping request from {{time}}", time=request.meta.request_time) # queue.commit() while not please_stop: request = queue.pop(till=please_stop) if please_stop: break if not request: Log.note("Nothing in queue, pausing for 5 seconds...") (please_stop | Till(seconds=5)).wait() continue Log.note("Found something in queue") repo = 'mozilla-central' and_op = request.where['and'] revision = None files = None for a in and_op: if a.eq.revision: revision = a.eq.revision elif a['in'].path: files = a['in'].path elif a.eq.path: files = [a.eq.path] if len(files) <= 0: Log.warning("No files in the given request: {{request}}", request=request) continue if revision[:12] in try_revs and not test_try_revs: Log.warning( "Revision {{cset}} does not exist in the {{branch}} branch", cset=revision[:12], branch='mozilla-central') queue.commit() continue clog_url = HG_URL / 'mozilla-central' / 'json-log' / revision[:12] clog_obj = http.get_json(clog_url, retry=RETRY) if isinstance(clog_obj, (text_type, str)): Log.warning( "Revision {{cset}} does not exist in the {{branch}} branch", cset=revision[:12], branch='mozilla-central') try_revs[revision[:12]] = True if not test_try_revs: queue.commit() continue else: json_rev_url = 'https://hg.mozilla.org/try/json-rev/' + revision[: 12] clog_obj = http.get_json(json_rev_url, retry=RETRY) if 'phase' not in clog_obj: Log.warning( "Revision {{cset}} does not exist in the try branch", cset=revision[:12], branch='mozilla-central') queue.commit() continue if clog_obj['phase'] == 'draft': repo = 'try' else: Log.note("Revision {{cset}} exists on mozilla-central.", cset=revision[:12]) request.branch = repo with Timer("Make TUID request from {{timestamp|date}}", {"timestamp": request.meta.request_time}): client.enabled = True # ENSURE THE REQUEST IS MADE result = http.post_json("http://localhost:5000/tuid", json=request, timeout=10000) if not client.enabled: Log.note("pausing consumer for {{num}}sec", num=PAUSE_ON_FAILURE) Till(seconds=PAUSE_ON_FAILURE).wait() if result is None or len(result.data) != len(files): Log.warning("expecting response for every file requested") queue.commit()
def backend_check(): http.get_json(config.elasticsearch.host + ":" + text_type(config.elasticsearch.port))
# Get the service to delete entries service = TUIDService(conn=sql.Sql("resources/tuid_app.db"), kwargs=config.tuid) # Get a list of 1000 files from stressfiles with open('resources/stressfiles.json', 'r') as f: files = json.load(f) # Get rev_count revisions from changelogs csets = [] final_rev = '' while len(csets) < rev_count: # Get a changelog clog_url = HG_URL /'mozilla-central' / 'json-log' / final_rev try: Log.note("Searching through changelog {{url}}", url=clog_url) clog_obj = http.get_json(clog_url, retry=RETRY) except Exception as e: Log.error("Unexpected error getting changset-log for {{url}}", url=clog_url, error=e) cset = '' for clog_cset in clog_obj['changesets']: cset = clog_cset['node'][:12] if len(csets) < rev_count: csets.append(cset) final_rev = cset # Oldest is now first csets.reverse() # Make the initial insertion (always slow)
service = TUIDService(conn=sql.Sql("resources/tuid_app.db"), kwargs=config.tuid) # Get a list of 1000 files from stressfiles with open('resources/stressfiles.json', 'r') as f: files = json.load(f) # Get rev_count revisions from changelogs csets = [] final_rev = '' while len(csets) < rev_count: # Get a changelog clog_url = HG_URL / 'mozilla-central' / 'json-log' / final_rev try: Log.note("Searching through changelog {{url}}", url=clog_url) clog_obj = http.get_json(clog_url, retry=RETRY) except Exception as e: Log.error("Unexpected error getting changset-log for {{url}}", url=clog_url, error=e) cset = '' for clog_cset in clog_obj['changesets']: cset = clog_cset['node'][:12] if len(csets) < rev_count: csets.append(cset) final_rev = cset # Oldest is now first csets.reverse()