def createJsonArtifact(queue, taskid, runid, name, data, expires): """Creates a Taskcluster artifact for the given taskid and runid, and uploads the artifact contents to location provided.""" data = json.dumps(data) resp = queue.createArtifact(taskid, runid, name, { "storageType": "s3", "contentType": "application/json", "expires": expires, }) log.debug("task %s: run %s: createArtifact returned %s", taskid, runid, resp) if resp.get("storageType") != "s3": raise ValueError("Can't upload artifact for task %s with runid %s because storageType is wrong" % (taskid, runid)) assert resp["storageType"] == "s3" put_url = resp["putUrl"] log.debug("task %s: run %s: Uploading to %s", taskid, runid, put_url) for _ in retrier(): try: resp = requests.put(put_url, data=data, headers={ "Content-Type": "application/json", "Content-Length": len(data), }) log.debug("task %s: run %s: Got %s %s", taskid, runid, resp, resp.headers) return except Exception: log.debug("task %s: run %s: Error submitting to s3", taskid, runid, exc_info=True) continue else: log.error("task %s: run %s: couldn't upload artifact to s3", taskid, runid) raise IOError("couldn't upload artifact to s3")
def retry(self, action: Callable): attempt = 1 for sleep_time in retrier(attempts=self.attempts, sleeptime=self.sleep_time, max_sleeptime=self.max_sleep_time, sleepscale=self.sleep_scale, jitter=self.jitter): try: return action() except Exception as e: for exception_to_check in self.retry_exceptions: if isinstance(e, exception_to_check): if attempt == self.attempts: _logger.warning( f"Reached maximum number of attempts ({self.attempts}), raising exception" ) raise _logger.info( f"Attempt number {attempt} out of {self.attempts} failed, " f"previous sleep time was {sleep_time} seconds. Exception: {e.__class__.__name__}('{str(e)}')" ) break else: raise attempt += 1
def test_retrier_maxsleep(self): with mock.patch("time.sleep") as sleep: # Test that max sleep time works for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=30, sleepscale=2, jitter=0): pass expected = [mock.call(x) for x in (10, 20, 30, 30)] self.assertEquals(sleep.call_args_list, expected)
def test_retrier_yields(self): """Ensure that retrier yields the sleep time""" for real_sleeptime in retrier(attempts=3, sleeptime=1, sleepscale=1, jitter=0): self.assertEqual(real_sleeptime, 1)
def wait_until_healthy(compose_project: str = "", timeout: int = 60): """ wait_until_healthy polls all running containers health check endpoints until they return a non-error status code. :param compose_project: the docker-compose project ID, if empty it checks all running containers. :param timeout: timeout in seconds. """ client = docker.from_env() kwargs = {} if compose_project != "": kwargs["filters"] = {"label": f"com.docker.compose.project={compose_project}"} path_map = { "mender-api-gateway": "/ping", "mender-auditlogs": "/api/internal/v1/auditlogs/health", "mender-deviceconnect": "/api/internal/v1/deviceconnect/health", "mender-deviceconfig": "/api/internal/v1/deviceconfig/health", "mender-device-auth": "/api/internal/v1/devauth/health", "mender-deployments": "/api/internal/v1/deployments/health", "mender-inventory": "/api/internal/v1/inventory/health", "mender-tenantadm": "/api/internal/v1/tenantadm/health", "mender-useradm": "/api/internal/v1/useradm/health", "mender-workflows": "/api/v1/health", "minio": "/minio/health/live", } containers = client.containers.list(all=True, **kwargs) for container in containers: container_ip = None for _, net in container.attrs["NetworkSettings"]["Networks"].items(): container_ip = net["IPAddress"] break if container_ip is None or container_ip == "": continue service = container.labels.get( "com.docker.compose.service", container.name ).split("-enterprise")[0] if service.startswith("mender-workflows-server"): service = "mender-workflows" path = path_map.get(service) if path is None: continue port = 8080 if service != "minio" else 9000 for _ in redo.retrier(attempts=timeout, sleeptime=1): try: rsp = requests.request("GET", f"http://{container_ip}:{port}{path}") except requests.exceptions.ConnectionError: # A ConnectionError is expected if the service is not running yet continue if rsp.status_code < 300: break else: raise TimeoutError( f"Timed out waiting for service '{service}' to become healthy" )
def is_reachable(url, *, session, headers=None, verify=True): """ Send a HEAD request with short timeout, return True if ressource has 2xx status code, False instead. """ if headers is None: headers = {} if "User-Agent" not in headers: headers["User-Agent"] = DEFAULT_USER_AGENT try: for attempt, _ in enumerate(redo.retrier(attempts=HTTP_MAX_ATTEMPTS, sleeptime=1.5, max_sleeptime=3, sleepscale=1.25, jitter=1), 1): try: response = session.head(url, headers=headers, timeout=HTTP_SHORT_TIMEOUT_S, verify=verify) break except (socket.timeout, requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e: logging.getLogger().warning("Querying '%s' failed (attempt %u/%u): %s %s" % (url, attempt, HTTP_MAX_ATTEMPTS, e.__class__.__qualname__, e)) if attempt == HTTP_MAX_ATTEMPTS: raise response.raise_for_status() except Exception: return False return True
def start(self): profile = Profile('BBTZ5TEST', self.id, self.userdata) cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -ZoteroDebugText -datadir profile > {shlex.quote(profile.path + ".log")} 2>&1' print(f'Starting {self.id}: {cmd}') self.proc = subprocess.Popen(cmd, shell=True) print(f'{self.id} started: {self.proc.pid}') ready = False with benchmark(f'starting {self.id}'): for _ in redo.retrier(attempts=30, sleeptime=1): print('connecting...') try: ready = self.execute(""" if (!Zotero.BetterBibTeX) { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded') return false; } if (!Zotero.BetterBibTeX.ready) { if (typeof Zotero.BetterBibTeX.ready === 'boolean') { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error') } else { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized') } return false; } Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...') await Zotero.BetterBibTeX.ready; Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!'); return true; """) if ready: break except (urllib.error.HTTPError, urllib.error.URLError): pass assert ready, f'{self.id} did not start'
def test_device_audit_log_events(self, event_type: str, tenant_users: Tenant): """Test if device events are loggged with correct fields.""" user = tenant_users.users[0] device = make_pending_device( self.devauthd, self.devauthm, user.token, tenant_token=tenant_users.tenant_token, ) if event_type == "decommission": response = self.devauthm.with_auth(user.token).call( "DELETE", deviceauth.URL_DEVICE, path_params={"id": device.id}) assert response.status_code == 204 elif event_type == "reject": for auth_set in device.authsets: change_authset_status(self.devauthm, device.id, auth_set.id, "rejected", user.token) for _ in redo.retrier(attempts=3, sleeptime=1): res = self.alogs.with_auth(user.token).call( "GET", auditlogs.URL_LOGS + "?object_type=device&object_id=" + device.id) assert res.status_code == 200 if len(res.json()) == 1: break else: assert False, f"max GET /logs retries hit, logs returned: {res.json()}" expected = event_device(user, device, event_type=event_type) check_log(res.json()[0], expected)
def __init__(self, task_id, artifact_name): for _ in redo.retrier(attempts=retry + 1, sleeptime=60): cot = cache._download_manager.session.get( get_artifact_url(task_id, 'public/chain-of-trust.json')) if cot.status_code >= 500: continue cot.raise_for_status() break else: cot.raise_for_status() digest = algorithm = None data = json.loads(cot.text) for algorithm, digest in (data.get('artifacts', {}).get(artifact_name, {}).items()): pass name = os.path.basename(artifact_name) artifact_url = get_artifact_url( task_id, artifact_name, use_proxy=not artifact_name.startswith('public/')) super(ArtifactRecord, self).__init__(artifact_url, name, None, digest, algorithm, unpack=True)
def createJsonArtifact(queue, taskid, runid, name, data, expires): """Creates a Taskcluster artifact for the given taskid and runid, and uploads the artifact contents to location provided.""" data = json.dumps(data) resp = queue.createArtifact(taskid, runid, name, { "storageType": "s3", "contentType": "application/json", "expires": expires, }) log.debug("Got %s", resp) if resp.get("storageType") != "s3": raise ValueError("Can't upload artifact for task %s with runid %s because storageType is wrong" % (taskid, runid)) assert resp["storageType"] == "s3" put_url = resp["putUrl"] log.debug("Uploading to %s", put_url) for _ in retrier(): try: resp = requests.put(put_url, data=data, headers={ "Content-Type": "application/json", "Content-Length": len(data), }) log.debug("Got %s %s", resp, resp.headers) return except Exception: log.debug("Error submitting to s3", exc_info=True) continue else: log.error("couldn't upload artifact to s3") raise IOError("couldn't upload artifact to s3")
def talent_pool_candidate_api(access_token, talent_pool_id, data='', action='GET', expected_count=0): headers = {'Authorization': 'Bearer %s' % access_token} if action == 'GET': response = None for _ in retrier(attempts=33, sleeptime=3): response = requests.get( url=CandidatePoolApiUrl.TALENT_POOL_CANDIDATE % talent_pool_id, headers=headers) if not response.ok or response.json().get( 'talent_pool_candidates').get( 'total_found') >= expected_count: break return response.json(), response.status_code elif action == 'DELETE': headers['content-type'] = 'application/json' response = requests.delete( url=CandidatePoolApiUrl.TALENT_POOL_CANDIDATE % talent_pool_id, data=json.dumps(data), headers=headers) return response.json(), response.status_code elif action == 'POST': headers['content-type'] = 'application/json' response = requests.post( url=CandidatePoolApiUrl.TALENT_POOL_CANDIDATE % talent_pool_id, headers=headers, data=json.dumps(data)) return response.json(), response.status_code else: raise Exception('No valid action is provided')
def test_configuration(self, standard_setup_one_client): """Tests the deployment and reporting of the configuration The tests set the configuration of a device and verifies the new configuration is reported back to the back-end. """ # accept the device devauth.accept_devices(1) # list of devices devices = list( set([ device["id"] for device in devauth.get_devices_status("accepted") ])) assert 1 == len(devices) auth = authentication.Authentication() wait_for_connect(auth, devices[0]) # set and verify the device's configuration # retry to skip possible race conditions between update poll and update trigger for _ in redo.retrier(attempts=3, sleeptime=1): set_and_verify_config({"key": "value"}, devices[0], auth.get_auth_token()) forced = was_update_forced(standard_setup_one_client.device) if forced: return assert False, "the update check was never triggered"
def test_retrier_sleep_no_jitter(self): """Make sure retrier sleep is behaving""" with mock.patch("time.sleep") as sleep: # Test that normal sleep scaling works without a jitter for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=None): pass expected = [mock.call(x) for x in (10, 20, 40, 80)] self.assertEquals(sleep.call_args_list, expected)
def main(): if len(sys.argv) != 2: print('Usage: uploadsymbols.py <zip file>', file=sys.stderr) return 1 if not os.path.isfile(sys.argv[1]): print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]), file=sys.stderr) return 1 symbols_zip = sys.argv[1] if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs: print( 'Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!', file=sys.stderr) return 1 token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE'] if not os.path.isfile(token_file): print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'. format(token_file), file=sys.stderr) return 1 auth_token = open(token_file, 'r').read().strip() print('Uploading symbol file "{0}" to "{1}"...'.format(sys.argv[1], url)) for _ in redo.retrier(): try: r = requests.post(url, files={'symbols.zip': open(sys.argv[1], 'rb')}, headers={'Auth-Token': auth_token}, allow_redirects=False, timeout=120) break except requests.exceptions.RequestException as e: print('Error: {0}'.format(e)) else: print('Maximum retries hit, giving up!') return 1 if r.status_code >= 200 and r.status_code < 300: print('Uploaded successfully!') return 0 if r.status_code < 400: print('Error: bad auth token? ({0}: {1})'.format( r.status_code, r.reason), file=sys.stderr) else: print('Error: got HTTP response {0}: {1}'.format( r.status_code, r.reason), file=sys.stderr) print('Response body:\n{sep}\n{body}\n{sep}\n'.format(sep='=' * 20, body=r.text)) return 1
def check_file_exists(url): for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): try: resp = requests.head(url, allow_redirects=True) return resp.status_code == requests.codes.ok except requests.exceptions.RequestException as e: log.error('Error: {0}'.format(e)) log.info('Retrying...') return False
def start(self): self.needs_restart = False profile = self.create_profile() shutil.rmtree(os.path.join(profile.path, self.client, 'better-bibtex'), ignore_errors=True) if self.client == 'zotero': datadir_profile = '-datadir profile' else: utils.print('\n\n** WORKAROUNDS FOR JURIS-M IN PLACE -- SEE https://github.com/Juris-M/zotero/issues/34 **\n\n') datadir_profile = '' cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -jsconsole -ZoteroDebugText {datadir_profile} {self.redir} {shlex.quote(profile.path + ".log")} 2>&1' utils.print(f'Starting {self.client}: {cmd}') self.proc = subprocess.Popen(cmd, shell=True) utils.print(f'{self.client} started: {self.proc.pid}') ready = False self.config.stash() self.config.timeout = 2 with benchmark(f'starting {self.client}') as bm: posted = False for _ in redo.retrier(attempts=120,sleeptime=1): utils.print('connecting... (%.2fs)' % (bm.elapsed,)) try: ready = self.execute(""" if (!Zotero.BetterBibTeX) { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded') return false; } if (!Zotero.BetterBibTeX.ready) { if (typeof Zotero.BetterBibTeX.ready === 'boolean') { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error') } else { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized') } return false; } Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...') await Zotero.BetterBibTeX.ready; if (testing && !Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!') Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!'); return true; """, testing = self.testing) if ready: break except (urllib.error.HTTPError, urllib.error.URLError,socket.timeout): pass if bm.elapsed > 2000 and not posted: posted = post_log() assert ready, f'{self.client} did not start' self.config.pop() if self.import_at_start: self.execute(f'return await Zotero.BetterBibTeX.TestSupport.importFile({json.dumps(self.import_at_start)})') self.import_at_start = None
def main(): if len(sys.argv) != 2: print('Usage: uploadsymbols.py <zip file>', file=sys.stderr) return 1 if not os.path.isfile(sys.argv[1]): print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]), file=sys.stderr) return 1 symbols_zip = sys.argv[1] if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs: print('Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!', file=sys.stderr) return 1 token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE'] if not os.path.isfile(token_file): print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file), file=sys.stderr) return 1 auth_token = open(token_file, 'r').read().strip() print('Uploading symbol file "{0}" to "{1}"...'.format(sys.argv[1], url)) for _ in redo.retrier(): try: r = requests.post( url, files={'symbols.zip': open(sys.argv[1], 'rb')}, headers={'Auth-Token': auth_token}, allow_redirects=False, timeout=120) break except requests.exceptions.RequestException as e: print('Error: {0}'.format(e)) else: print('Maximum retries hit, giving up!') return 1 if r.status_code >= 200 and r.status_code < 300: print('Uploaded successfully!') return 0 if r.status_code < 400: print('Error: bad auth token? ({0}: {1})'.format(r.status_code, r.reason), file=sys.stderr) else: print('Error: got HTTP response {0}: {1}'.format(r.status_code, r.reason), file=sys.stderr) print('Response body:\n{sep}\n{body}\n{sep}\n'.format( sep='=' * 20, body=r.text )) return 1
def downloadReleaseBuilds(stageServer, productName, brandName, version, buildNumber, platform, candidatesDir=None, signed=False, usePymake=False): if candidatesDir is None: candidatesDir = makeCandidatesDir(productName, version, buildNumber, protocol='http', server=stageServer) files = makeReleaseRepackUrls(productName, brandName, version, platform, signed=signed) env = {} for fileName, remoteFile in files.iteritems(): url = '/'.join( [p.strip('/') for p in [candidatesDir, urllib.quote(remoteFile)]]) log.info("Downloading %s to %s", url, fileName) for _ in retrier(): with open(fileName, "wb") as f: try: r = requests.get(url, stream=True, timeout=15) r.raise_for_status() for chunk in r.iter_content(chunk_size=5 * 1024**2): f.write(chunk) r.close() break except (requests.HTTPError, requests.ConnectionError, requests.Timeout): log.exception("Caught exception downloading") if fileName.endswith('exe'): if usePymake: env['WIN32_INSTALLER_IN'] = msys2windows( path.join(os.getcwd(), fileName)) else: env['WIN32_INSTALLER_IN'] = windows2msys( path.join(os.getcwd(), fileName)) else: if platform.startswith('win') and not usePymake: env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName)) else: env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName)) return env
def start(self): self.needs_restart = False profile = self.create_profile() cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -jsconsole -ZoteroDebugText -datadir profile {self.redir} {shlex.quote(profile.path + ".log")} 2>&1' utils.print(f'Starting {self.client}: {cmd}') self.proc = subprocess.Popen(cmd, shell=True) utils.print(f'{self.client} started: {self.proc.pid}') ready = False self.config.stash() self.config.timeout = 2 with benchmark(f'starting {self.client}') as bm: posted = None for _ in redo.retrier(attempts=120, sleeptime=1): utils.print('connecting... (%.2fs)' % (bm.elapsed, )) try: ready = self.execute(""" if (!Zotero.BetterBibTeX) { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded') return false; } if (!Zotero.BetterBibTeX.ready) { if (typeof Zotero.BetterBibTeX.ready === 'boolean') { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error') } else { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized') } return false; } Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...') await Zotero.BetterBibTeX.ready; if (testing && !Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!') Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!'); return true; """, testing=self.testing) if ready: break except (urllib.error.HTTPError, urllib.error.URLError, socket.timeout): pass if bm.elapsed > 2000 and not posted: posted = PostLog() if posted: utils.print( 'connected, but log posted, waiting for upload to finish...' ) posted.join() assert ready, f'{self.client} did not start' self.config.pop()
def test_retrier_jitter(self): with mock.patch("time.sleep") as sleep: # Test that jitter works with mock.patch("random.uniform") as uniform: uniform.return_value = 3 for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=3): uniform.return_value *= -1 expected = [mock.call(x) for x in (7, 23, 37, 83)] self.assertEquals(sleep.call_args_list, expected) self.assertEquals(uniform.call_args, mock.call(-48, 48))
def ping_with_retries(self, attempts): retrier_kwargs = { 'attempts': attempts, 'sleeptime': 0.3, 'jitter': 0 } for _ in redo.retrier(**retrier_kwargs): if self.ping(): return True return False
def test_retrier_jitter(self): with mock.patch("time.sleep") as sleep: # Test that jitter works with mock.patch("random.randint") as randint: randint.return_value = 3 for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=3): randint.return_value *= -1 expected = [mock.call(x) for x in (7, 23, 37, 83)] self.assertEquals(sleep.call_args_list, expected) self.assertEquals(randint.call_args, mock.call(-48, 48))
def main(): if len(sys.argv) != 2: print('Usage: uploadsymbols.py <zip file>', file=sys.stderr) return 1 if not os.path.isfile(sys.argv[1]): print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]), file=sys.stderr) return 1 symbols_zip = sys.argv[1] if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs: print( 'Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!', file=sys.stderr) return 1 token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE'] if not os.path.isfile(token_file): print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'. format(token_file), file=sys.stderr) return 1 auth_token = open(token_file, 'r').read().strip() print('Uploading symbol file "{0}" to "{1}"'.format(sys.argv[1], url)) for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): print('Attempt %d of %d...' % (i, MAX_RETRIES)) try: r = requests.post(url, files={'symbols.zip': open(sys.argv[1], 'rb')}, headers={'Auth-Token': auth_token}, allow_redirects=False, timeout=120) # 500 is likely to be a transient failure. # Break out for success or other error codes. if r.status_code < 500: break print_error(r) except requests.exceptions.RequestException as e: print('Error: {0}'.format(e)) print('Retrying...') else: print('Maximum retries hit, giving up!') return 1 if r.status_code >= 200 and r.status_code < 300: print('Uploaded successfully!') return 0 print_error(r) return 1
def upload_symbols(zip_file, token_file): print("Uploading symbols file '{0}' to '{1}'".format( zip_file, DEFAULT_SYMBOL_URL), file=sys.stdout) zip_name = os.path.basename(zip_file) # XXX: fetch the symbol upload token from local file, taskgraph handles # already that communication with Taskcluster to get the credentials for # communicating with the server auth_token = '' with open(token_file, 'r') as f: auth_token = f.read().strip() if len(auth_token) == 0: print("Failed to get the symbol token.", file=sys.stderr) if auth_token == 'faketoken': print("'faketoken` detected, not pushing anything", file=sys.stdout) sys.exit(0) for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): print("Attempt %d of %d..." % (i, MAX_RETRIES)) try: if zip_file.startswith("http"): zip_arg = {"data": {"url", zip_file}} else: zip_arg = {"files": {zip_name: open(zip_file, 'rb')}} r = requests.post( DEFAULT_SYMBOL_URL, headers={"Auth-Token": auth_token}, allow_redirects=False, # Allow a longer read timeout because uploading by URL means the server # has to fetch the entire zip file, which can take a while. The load balancer # in front of symbols.mozilla.org has a 300 second timeout, so we'll use that. timeout=(10, 300), **zip_arg) # 500 is likely to be a transient failure. # Break out for success or other error codes. if r.status_code < 500: break print("Error: {0}".format(r), file=sys.stderr) except requests.exceptions.RequestException as e: print("Error: {0}".format(e), file=sys.stderr) print("Retrying...", file=sys.stdout) else: print("Maximum retries hit, giving up!", file=sys.stderr) return False if r.status_code >= 200 and r.status_code < 300: print("Uploaded successfully", file=sys.stdout) return True print("Upload symbols failed: {0}".format(r), file=sys.stderr) return False
def _prepare_device( self, azure_user: User, httpserver: HTTPServer, httpserver_ssl_context: ssl.SSLContext, ) -> Device: """Create accepted device in Mender and make sure it has been successfully added in Azure IoT Hub.""" if self.azure_iot_hub_mock: httpserver.expect_oneshot_request( re.compile("^/devices"), method="PUT", query_string="api-version=2021-04-12", ).respond_with_json(self._prepare_iot_hub_upsert_device_response()) httpserver.expect_oneshot_request( re.compile("^/devices"), method="GET", query_string="api-version=2021-04-12", ).respond_with_data(status=200) httpserver.expect_oneshot_request( re.compile("^/devices"), method="PUT", query_string="api-version=2021-04-12", ).respond_with_data(status=200) httpserver.expect_oneshot_request( re.compile("^/twins"), method="PATCH", query_string="api-version=2021-04-12", ).respond_with_data(status=200) tenant_token = getattr(getattr(azure_user, "tenant", {}), "tenant_token", "") dev = make_accepted_device( self.api_devauth_devices, self.api_devauth_mgmt, azure_user.token, tenant_token=tenant_token, test_type="azure", ) self.devices.append(dev.id) for _ in retrier(attempts=5, sleeptime=1): if self.azure_iot_hub_mock: httpserver.expect_oneshot_request( re.compile("^/twins"), method="GET", query_string="api-version=2021-04-12", ).respond_with_json( self._prepare_iot_hub_upsert_device_response()) rsp = self.api_azure.with_auth(azure_user.token).call( "GET", iot.URL_DEVICE(dev.id)) if rsp.status_code == 200: break return dev
def retry_aws_request(callable, *args, **kwargs): """Calls callable(*args, **kwargs), and sleeps/retries on RequestLimitExceeded errors""" for _ in retrier(): try: return callable(*args, **kwargs) except BotoServerError, e: if e.code == 'RequestLimitExceeded': # Try again log.debug("Got RequestLimitExceeded; retrying", exc_info=True) continue # Otherwise re-raise raise
def get_and_assert_zero(url, key, token, sleep_time=SLEEP_TIME): """ This function gets list of objects from given url and asserts that length of objects under a given key is zero. It keeps on retrying this process until it founds some records or sleep_time is over :param string url: URL of requested resource :param string key: key in response that has resource list :param string token: user access token :param int sleep_time: maximum time to wait """ attempts = sleep_time / SLEEP_INTERVAL for _ in retrier(attempts=attempts, sleeptime=SLEEP_INTERVAL, sleepscale=1): assert len(send_request('get', url, token).json()[key]) == 0
def Upload_Symbols(zip_file): print("Uploading symbols file '{0}' to '{1}'".format( zip_file, DEFAULT_SYMBOL_URL), file=sys.stdout) zip_name = os.path.basename(zip_file) # Fetch the symbol server token from Taskcluster secrets. secrets_url = "http://taskcluster/secrets/v1/secret/{}".format( "project/application-services/symbols-token") res = requests.get(secrets_url) res.raise_for_status() secret = res.json() auth_token = secret["secret"]["token"] if len(auth_token) == 0: print("Failed to get the symbol token.", file=sys.stderr) for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): print("Attempt %d of %d..." % (i, MAX_RETRIES)) try: if zip_file.startswith("http"): zip_arg = {"data": {"url", zip_file}} else: zip_arg = {"files": {zip_name: open(zip_file, 'rb')}} r = requests.post( DEFAULT_SYMBOL_URL, headers={"Auth-Token": auth_token}, allow_redirects=False, # Allow a longer read timeout because uploading by URL means the server # has to fetch the entire zip file, which can take a while. The load balancer # in front of symbols.mozilla.org has a 300 second timeout, so we'll use that. timeout=(10, 300), **zip_arg) # 500 is likely to be a transient failure. # Break out for success or other error codes. if r.status_code < 500: break print("Error: {0}".format(r), file=sys.stderr) except requests.exceptions.RequestException as e: print("Error: {0}".format(e), file=sys.stderr) print("Retrying...", file=sys.stdout) else: print("Maximum retries hit, giving up!", file=sys.stderr) return False if r.status_code >= 200 and r.status_code < 300: print("Uploaded successfully", file=sys.stdout) return True print("Upload symbols failed: {0}".format(r), file=sys.stderr) return False
def reboot(self, timeout=60, reconnect_attempts=5): command_schedule_time = 10 self.sudo_exec("shutdown -r -t {}".format(command_schedule_time)) self.close() time.sleep(command_schedule_time) retries = 0 for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts): try: self.connect() return except _reconnect_exceptions: retries += 1 logger.info("Trying to connect to %s after reboot issued for the %d time.", self._host, retries) raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host)
def get_auditlog_time(oid: str): # get exact time for filter testing found = None for _ in redo.retrier(attempts=3, sleeptime=1): resp = self.alogs.with_auth(tenant_users.users[0].token).call( "GET", auditlogs.URL_LOGS) found = [ audit_log for audit_log in resp.json() if audit_log["object"]["id"] == oid ] if len(found) == 1: return found[0]["time"] else: assert False, "max GET /logs retries hit"
def _retry_on_http_errors(url, auth, verify, params, errors): for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10): try: req = requests.get(url, auth=auth, verify=verify, params=params) req.raise_for_status() return req except requests.HTTPError as e: if e.response.status_code in errors: log.exception("Got HTTP %s trying to reach %s", e.response.status_code, url) else: raise else: raise
def get_auditlog_time(oid): # get exact time for filter testing found = None for _ in redo.retrier(attempts=3, sleeptime=1): alogs = ApiClient(auditlogs.URL_MGMT) resp = alogs.with_auth(tenant_users.users[0].token).call( "GET", auditlogs.URL_LOGS ) resp = resp.json() found = [e for e in resp if e["object"]["id"] == oid] if len(found) == 1: return found[0]["time"] else: assert False, "max GET /logs retries hit"
def main(): if len(sys.argv) != 2: print('Usage: uploadsymbols.py <zip file>', file=sys.stderr) return 1 if not os.path.isfile(sys.argv[1]): print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]), file=sys.stderr) return 1 symbols_zip = sys.argv[1] if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs: print('Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!', file=sys.stderr) return 1 token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE'] if not os.path.isfile(token_file): print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file), file=sys.stderr) return 1 auth_token = open(token_file, 'r').read().strip() print('Uploading symbol file "{0}" to "{1}"'.format(sys.argv[1], url)) for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): print('Attempt %d of %d...' % (i, MAX_RETRIES)) try: r = requests.post( url, files={'symbols.zip': open(sys.argv[1], 'rb')}, headers={'Auth-Token': auth_token}, allow_redirects=False, timeout=120) # 500 is likely to be a transient failure. # Break out for success or other error codes. if r.status_code < 500: break print_error(r) except requests.exceptions.RequestException as e: print('Error: {0}'.format(e)) print('Retrying...') else: print('Maximum retries hit, giving up!') return 1 if r.status_code >= 200 and r.status_code < 300: print('Uploaded successfully!') return 0 print_error(r) return 1
def test_retrier_jitter(self): with mock.patch("time.sleep") as sleep: # Test that jitter works with mock.patch("random.randint") as randint: randint.return_value = 3 for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=3): randint.return_value *= -1 expected = [mock.call(x) for x in (7, 17, 31, 65)] self.assertEquals(sleep.call_args_list, expected) self.assertEquals(randint.call_args, mock.call(-3, 3))
def test_get_params(self, tenant_users): """ Mix up some audiltog events, check GET with various params """ # N various events from both users events = [] for i in range(10): uidx = i % 2 user = tenant_users.users[uidx] evt = None oid = None if i % 3 == 0: uuidv4 = str(uuid.uuid4()) uid = make_user(user.token, uuidv4 + "@acme.com", "secretsecret") evt = evt_user_create(user, uid, uuidv4 + "@acme.com") oid = uid else: d = make_deployment(user.token) evt = evt_deployment_create(user, d) oid = d["id"] time.sleep(0.5) # get exact time for filter testing found = None for _ in redo.retrier(attempts=3, sleeptime=1): alogs = ApiClient(auditlogs.URL_MGMT) resp = alogs.with_auth(tenant_users.users[0].token).call( "GET", auditlogs.URL_LOGS) resp = resp.json() found = [e for e in resp if e["object"]["id"] == oid] if len(found) == 1: break else: assert False, "max GET /logs retries hit" evt["time"] = found[0]["time"] events.append(evt) # default sorting is desc by time events.reverse() self._test_args_paging(tenant_users, events) self._test_args_actor(tenant_users, events) self._test_args_before_after(tenant_users, events) self._test_args_object(tenant_users, events)
def get_mender_gateway(self): """Returns IP address of mender-api-gateway service Has internal retry - upon setup 'up', the gateway will not be available for a while. """ for _ in redo.retrier(attempts=10, sleeptime=1): gateway = self.get_ip_of_service("mender-api-gateway") if len(gateway) != 1: continue else: return gateway[0] else: assert ( False ), "expected one instance of api-gateway running, but found: {} instance(s)".format( len(gateway))
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs): """Pulls changes from hg repo and places it in `dest`. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. If `mirrors` is set, will try and pull from the mirrors first before `repo`.""" if mirrors: for mirror in mirrors: try: return pull(mirror, dest, update_dest=update_dest, **kwargs) except: log.exception("Problem pulling from mirror %s", mirror) continue else: log.info("Pulling from mirrors failed; falling back to %s", repo) # Convert repo to an absolute path if it's a local repository repo = _make_absolute(repo) cmd = ['pull'] # Don't pass -r to "hg pull", except when it's a valid HG revision. # Pulling using tag names is dangerous: it uses the local .hgtags, so if # the tag has moved on the remote side you won't pull the new revision the # remote tag refers to. pull_kwargs = kwargs.copy() if 'revision' in pull_kwargs and \ not is_hg_cset(pull_kwargs['revision']): del pull_kwargs['revision'] cmd.extend(common_args(**pull_kwargs)) cmd.append(repo) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS): try: get_hg_output(cmd=cmd, cwd=dest, include_stderr=True) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! continue raise
def revision_to_revision_hash(th_api_root, branch, revision): url = "{th_api_root}/project/{branch}/resultset".format( th_api_root=th_api_root, branch=branch ) # Use short revision for treeherder API revision = revision[:12] params = {"revision": revision} for _ in redo.retrier(sleeptime=5, max_sleeptime=30): params_str = "&".join("=".join([k, str(v)]) for k, v in params.iteritems()) try: log.debug("Connecting to %s?%s", url, params_str) r = requests.get(url, params=params) return r.json()["results"][0]["revision_hash"] except: log.exception("Failed to connect to %s?%s", url, params_str) else: raise RuntimeError("Cannot fetch revision hash for {} {}".format( branch, revision))
def _retry_on_http_errors(url, verify, params, errors): if params: params_str = "&".join("=".join([k, str(v)]) for k, v in params.iteritems()) else: params_str = '' logger.info("Connecting to %s?%s", url, params_str) for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10): try: req = requests.get(url, verify=verify, params=params, timeout=4) req.raise_for_status() return req except requests.HTTPError as e: if e.response.status_code in errors: logger.exception("Got HTTP %s trying to reach %s", e.response.status_code, url) else: raise else: raise
def downloadReleaseBuilds(stageServer, productName, brandName, version, buildNumber, platform, candidatesDir=None, signed=False, usePymake=False): if candidatesDir is None: candidatesDir = makeCandidatesDir(productName, version, buildNumber, protocol='http', server=stageServer) files = makeReleaseRepackUrls(productName, brandName, version, platform, signed=signed) env = {} for fileName, remoteFile in files.iteritems(): url = '/'.join([p.strip('/') for p in [candidatesDir, urllib.quote(remoteFile)]]) log.info("Downloading %s to %s", url, fileName) for _ in retrier(): with open(fileName, "wb") as f: try: r = requests.get(url, stream=True, timeout=15) r.raise_for_status() for chunk in r.iter_content(chunk_size=5*1024**2): f.write(chunk) r.close() break except (requests.HTTPError, requests.ConnectionError, requests.Timeout): log.exception("Caught exception downloading") if fileName.endswith('exe'): if usePymake: env['WIN32_INSTALLER_IN'] = msys2windows(path.join(os.getcwd(), fileName)) else: env['WIN32_INSTALLER_IN'] = windows2msys(path.join(os.getcwd(), fileName)) else: if platform.startswith('win') and not usePymake: env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName)) else: env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName)) return env
def shutdown(self): if self.proc is None: return # graceful shutdown try: self.execute(""" const appStartup = Components.classes['@mozilla.org/toolkit/app-startup;1'].getService(Components.interfaces.nsIAppStartup); appStartup.quit(Components.interfaces.nsIAppStartup.eAttemptQuit); """) except: pass stopped = False for _ in redo.retrier(attempts=5,sleeptime=1): stopped = not running(self.proc.pid) if stopped: break zotero = psutil.Process(self.proc.pid) for proc in zotero.children(recursive=True): proc.kill() zotero.kill()
def start(self): profile = Profile('BBTZ5TEST', self.id, self.userdata) cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -ZoteroDebugText -datadir profile > {shlex.quote(profile.path + ".log")} 2>&1' print(f'Starting {self.id}: {cmd}') self.proc = subprocess.Popen(cmd, shell=True) print(f'{self.id} started: {self.proc.pid}') ready = False with benchmark(f'starting {self.id}'): for _ in redo.retrier(attempts=30,sleeptime=1): print('connecting...') try: ready = self.execute(""" if (!Zotero.BetterBibTeX) { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded') return false; } if (!Zotero.BetterBibTeX.ready) { if (typeof Zotero.BetterBibTeX.ready === 'boolean') { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error') } else { Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized') } return false; } Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...') await Zotero.BetterBibTeX.ready; if (!Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!') Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!'); return true; """) if ready: break except (urllib.error.HTTPError, urllib.error.URLError): pass assert ready, f'{self.id} did not start'
def main(): config = MozbuildObject.from_environment() config._activate_virtualenv() import redo import requests logging.basicConfig() parser = argparse.ArgumentParser( description='Upload symbols in ZIP using token from Taskcluster secrets service.') parser.add_argument('zip', help='Symbols zip file - URL or path to local file') args = parser.parse_args() if not args.zip.startswith('http') and not os.path.isfile(args.zip): log.error('Error: zip file "{0}" does not exist!'.format(args.zip)) return 1 secret_name = os.environ.get('SYMBOL_SECRET') if secret_name is not None: auth_token = get_taskcluster_secret(secret_name) elif 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' in os.environ: token_file = os.environ['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE'] if not os.path.isfile(token_file): log.error('SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file)) return 1 auth_token = open(token_file, 'r').read().strip() else: log.error('You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE ' 'environment variables!') return 1 # Allow overwriting of the upload url with an environmental variable if 'SOCORRO_SYMBOL_UPLOAD_URL' in os.environ: url = os.environ['SOCORRO_SYMBOL_UPLOAD_URL'] elif os.environ.get('MOZ_SCM_LEVEL', '1') == '1': # Use the Tecken staging server for try uploads for now. # This will eventually be changed in bug 1138617. url = 'https://symbols.stage.mozaws.net/upload/' else: url = DEFAULT_URL log.info('Uploading symbol file "{0}" to "{1}"'.format(args.zip, url)) for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): log.info('Attempt %d of %d...' % (i, MAX_RETRIES)) try: if args.zip.startswith('http'): zip_arg = {'data': {'url': args.zip}} else: zip_arg = {'files': {'symbols.zip': open(args.zip, 'rb')}} r = requests.post( url, headers={'Auth-Token': auth_token}, allow_redirects=False, # Allow a longer read timeout because uploading by URL means the server # has to fetch the entire zip file, which can take a while. The load balancer # in front of symbols.mozilla.org has a 300 second timeout, so we'll use that. timeout=(10, 300), **zip_arg) # 500 is likely to be a transient failure. # Break out for success or other error codes. if r.status_code < 500: break print_error(r) except requests.exceptions.RequestException as e: log.error('Error: {0}'.format(e)) log.info('Retrying...') else: log.warn('Maximum retries hit, giving up!') return 1 if r.status_code >= 200 and r.status_code < 300: log.info('Uploaded successfully!') return 0 print_error(r) return 1
def test_jitter_bounds(self): self.assertRaises(Exception, retrier(sleeptime=1, jitter=2))
def test_retrier(self): """Make sure retrier behaves properly""" n = 0 for _ in retrier(attempts=5, sleeptime=0, jitter=0): n += 1 self.assertEqual(n, 5)
def http_request( self, method, path, params={}, headers={}, json=None, etag=None, endpoint=None, retry=False, ): _headers = self._http_headers['default'].copy() _headers.update(self._http_headers[method]) _headers.update(headers) headers = _headers token = self.get_bearer_token() if self.logged_in: headers.update({ 'Authorization': 'Bearer %s' % token, }) if etag: headers.update({ 'If-Match': etag, }) if endpoint: url = endpoint + '/' + path else: url = self.endpoint + '/api' + path # Setting the parameter at all (even False) turns on admin mode if self.admin: params.update({'admin': self.admin}) if params: self.logger.debug( "params={}".format(params) ) if json: self.logger.debug( "json={}".format(json) ) if retry: retry_attempts = HTTP_RETRY_LIMIT else: retry_attempts = 1 for _ in retrier( attempts=retry_attempts, sleeptime=RETRY_BACKOFF_INTERVAL, ): response = self.session.request( method, url, params=params, headers=headers, json=json, ) if response.status_code < 500: break else: raise PanoptesAPIException( 'Received HTTP status code {} from API'.format( response.status_code ) ) return response
def clone(repo, dest, branch=None, revision=None, update_dest=True, clone_by_rev=False, mirrors=None, bundles=None): """Clones hg repo and places it at `dest`, replacing whatever else is there. The working copy will be empty. If `revision` is set, only the specified revision and its ancestors will be cloned. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. If `mirrors` is set, will try and clone from the mirrors before cloning from `repo`. If `bundles` is set, will try and download the bundle first and unbundle it. If successful, will pull in new revisions from mirrors or the master repo. If unbundling fails, will fall back to doing a regular clone from mirrors or the master repo. Regardless of how the repository ends up being cloned, the 'default' path will point to `repo`. """ if os.path.exists(dest): remove_path(dest) if bundles: log.info("Attempting to initialize clone with bundles") for bundle in bundles: if os.path.exists(dest): remove_path(dest) init(dest) log.info("Trying to use bundle %s", bundle) try: if not unbundle(bundle, dest): remove_path(dest) continue adjust_paths(dest, default=repo) # Now pull / update return pull(repo, dest, update_dest=update_dest, mirrors=mirrors, revision=revision, branch=branch) except Exception: remove_path(dest) log.exception("Problem unbundling/pulling from %s", bundle) continue else: log.info("Using bundles failed; falling back to clone") if mirrors: log.info("Attempting to clone from mirrors") for mirror in mirrors: log.info("Cloning from %s", mirror) try: retval = clone(mirror, dest, branch, revision, update_dest=update_dest, clone_by_rev=clone_by_rev) adjust_paths(dest, default=repo) return retval except: log.exception("Problem cloning from mirror %s", mirror) continue else: log.info("Pulling from mirrors failed; falling back to %s", repo) # We may have a partial repo here; mercurial() copes with that # We need to make sure our paths are correct though if os.path.exists(os.path.join(dest, '.hg')): adjust_paths(dest, default=repo) return mercurial(repo, dest, branch, revision, autoPurge=True, update_dest=update_dest, clone_by_rev=clone_by_rev) cmd = ['clone'] if not update_dest: cmd.append('-U') if clone_by_rev: if revision: cmd.extend(['-r', revision]) elif branch: # hg >= 1.6 supports -b branch for cloning ver = hg_ver() if ver >= (1, 6, 0): cmd.extend(['-b', branch]) cmd.extend([repo, dest]) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS): try: get_hg_output(cmd=cmd, include_stderr=True) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! # Make sure the dest is clean if os.path.exists(dest): log.debug("deleting %s", dest) remove_path(dest) continue raise