def createJsonArtifact(queue, taskid, runid, name, data, expires):
"""Creates a Taskcluster artifact for the given taskid and runid, and
uploads the artifact contents to location provided."""
data = json.dumps(data)
resp = queue.createArtifact(taskid, runid, name, {
"storageType": "s3",
"contentType": "application/json",
"expires": expires,
})
log.debug("task %s: run %s: createArtifact returned %s", taskid, runid, resp)
if resp.get("storageType") != "s3":
raise ValueError("Can't upload artifact for task %s with runid %s because storageType is wrong" % (taskid, runid))
assert resp["storageType"] == "s3"
put_url = resp["putUrl"]
log.debug("task %s: run %s: Uploading to %s", taskid, runid, put_url)
for _ in retrier():
try:
resp = requests.put(put_url, data=data, headers={
"Content-Type": "application/json",
"Content-Length": len(data),
})
log.debug("task %s: run %s: Got %s %s", taskid, runid, resp, resp.headers)
return
except Exception:
log.debug("task %s: run %s: Error submitting to s3", taskid, runid, exc_info=True)
continue
else:
log.error("task %s: run %s: couldn't upload artifact to s3", taskid, runid)
raise IOError("couldn't upload artifact to s3")
def retry(self, action: Callable):
attempt = 1
for sleep_time in retrier(attempts=self.attempts,
sleeptime=self.sleep_time,
max_sleeptime=self.max_sleep_time,
sleepscale=self.sleep_scale,
jitter=self.jitter):
try:
return action()
except Exception as e:
for exception_to_check in self.retry_exceptions:
if isinstance(e, exception_to_check):
if attempt == self.attempts:
_logger.warning(
f"Reached maximum number of attempts ({self.attempts}), raising exception"
)
raise
_logger.info(
f"Attempt number {attempt} out of {self.attempts} failed, "
f"previous sleep time was {sleep_time} seconds. Exception: {e.__class__.__name__}('{str(e)}')"
)
break
else:
raise
attempt += 1
def test_retrier_maxsleep(self):
with mock.patch("time.sleep") as sleep:
# Test that max sleep time works
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=30, sleepscale=2, jitter=0):
pass
expected = [mock.call(x) for x in (10, 20, 30, 30)]
self.assertEquals(sleep.call_args_list, expected)
def test_retrier_yields(self):
"""Ensure that retrier yields the sleep time"""
for real_sleeptime in retrier(attempts=3,
sleeptime=1,
sleepscale=1,
jitter=0):
self.assertEqual(real_sleeptime, 1)
def wait_until_healthy(compose_project: str = "", timeout: int = 60):
"""
wait_until_healthy polls all running containers health check
endpoints until they return a non-error status code.
:param compose_project: the docker-compose project ID, if empty it
checks all running containers.
:param timeout: timeout in seconds.
"""
client = docker.from_env()
kwargs = {}
if compose_project != "":
kwargs["filters"] = {"label": f"com.docker.compose.project={compose_project}"}
path_map = {
"mender-api-gateway": "/ping",
"mender-auditlogs": "/api/internal/v1/auditlogs/health",
"mender-deviceconnect": "/api/internal/v1/deviceconnect/health",
"mender-deviceconfig": "/api/internal/v1/deviceconfig/health",
"mender-device-auth": "/api/internal/v1/devauth/health",
"mender-deployments": "/api/internal/v1/deployments/health",
"mender-inventory": "/api/internal/v1/inventory/health",
"mender-tenantadm": "/api/internal/v1/tenantadm/health",
"mender-useradm": "/api/internal/v1/useradm/health",
"mender-workflows": "/api/v1/health",
"minio": "/minio/health/live",
}
containers = client.containers.list(all=True, **kwargs)
for container in containers:
container_ip = None
for _, net in container.attrs["NetworkSettings"]["Networks"].items():
container_ip = net["IPAddress"]
break
if container_ip is None or container_ip == "":
continue
service = container.labels.get(
"com.docker.compose.service", container.name
).split("-enterprise")[0]
if service.startswith("mender-workflows-server"):
service = "mender-workflows"
path = path_map.get(service)
if path is None:
continue
port = 8080 if service != "minio" else 9000
for _ in redo.retrier(attempts=timeout, sleeptime=1):
try:
rsp = requests.request("GET", f"http://{container_ip}:{port}{path}")
except requests.exceptions.ConnectionError:
# A ConnectionError is expected if the service is not running yet
continue
if rsp.status_code < 300:
break
else:
raise TimeoutError(
f"Timed out waiting for service '{service}' to become healthy"
)
def is_reachable(url, *, session, headers=None, verify=True):
""" Send a HEAD request with short timeout, return True if ressource has 2xx status code, False instead. """
if headers is None:
headers = {}
if "User-Agent" not in headers:
headers["User-Agent"] = DEFAULT_USER_AGENT
try:
for attempt, _ in enumerate(redo.retrier(attempts=HTTP_MAX_ATTEMPTS,
sleeptime=1.5,
max_sleeptime=3,
sleepscale=1.25,
jitter=1),
1):
try:
response = session.head(url,
headers=headers,
timeout=HTTP_SHORT_TIMEOUT_S,
verify=verify)
break
except (socket.timeout, requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
logging.getLogger().warning("Querying '%s' failed (attempt %u/%u): %s %s" % (url,
attempt,
HTTP_MAX_ATTEMPTS,
e.__class__.__qualname__,
e))
if attempt == HTTP_MAX_ATTEMPTS:
raise
response.raise_for_status()
except Exception:
return False
return True
def start(self):
profile = Profile('BBTZ5TEST', self.id, self.userdata)
cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -ZoteroDebugText -datadir profile > {shlex.quote(profile.path + ".log")} 2>&1'
print(f'Starting {self.id}: {cmd}')
self.proc = subprocess.Popen(cmd, shell=True)
print(f'{self.id} started: {self.proc.pid}')
ready = False
with benchmark(f'starting {self.id}'):
for _ in redo.retrier(attempts=30, sleeptime=1):
print('connecting...')
try:
ready = self.execute("""
if (!Zotero.BetterBibTeX) {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded')
return false;
}
if (!Zotero.BetterBibTeX.ready) {
if (typeof Zotero.BetterBibTeX.ready === 'boolean') {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error')
} else {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized')
}
return false;
}
Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...')
await Zotero.BetterBibTeX.ready;
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!');
return true;
""")
if ready: break
except (urllib.error.HTTPError, urllib.error.URLError):
pass
assert ready, f'{self.id} did not start'
def test_retrier_maxsleep(self):
with mock.patch("time.sleep") as sleep:
# Test that max sleep time works
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=30, sleepscale=2, jitter=0):
pass
expected = [mock.call(x) for x in (10, 20, 30, 30)]
self.assertEquals(sleep.call_args_list, expected)
def test_device_audit_log_events(self, event_type: str,
tenant_users: Tenant):
"""Test if device events are loggged with correct fields."""
user = tenant_users.users[0]
device = make_pending_device(
self.devauthd,
self.devauthm,
user.token,
tenant_token=tenant_users.tenant_token,
)
if event_type == "decommission":
response = self.devauthm.with_auth(user.token).call(
"DELETE", deviceauth.URL_DEVICE, path_params={"id": device.id})
assert response.status_code == 204
elif event_type == "reject":
for auth_set in device.authsets:
change_authset_status(self.devauthm, device.id, auth_set.id,
"rejected", user.token)
for _ in redo.retrier(attempts=3, sleeptime=1):
res = self.alogs.with_auth(user.token).call(
"GET", auditlogs.URL_LOGS + "?object_type=device&object_id=" +
device.id)
assert res.status_code == 200
if len(res.json()) == 1:
break
else:
assert False, f"max GET /logs retries hit, logs returned: {res.json()}"
expected = event_device(user, device, event_type=event_type)
check_log(res.json()[0], expected)
def __init__(self, task_id, artifact_name):
for _ in redo.retrier(attempts=retry + 1, sleeptime=60):
cot = cache._download_manager.session.get(
get_artifact_url(task_id,
'public/chain-of-trust.json'))
if cot.status_code >= 500:
continue
cot.raise_for_status()
break
else:
cot.raise_for_status()
digest = algorithm = None
data = json.loads(cot.text)
for algorithm, digest in (data.get('artifacts',
{}).get(artifact_name,
{}).items()):
pass
name = os.path.basename(artifact_name)
artifact_url = get_artifact_url(
task_id,
artifact_name,
use_proxy=not artifact_name.startswith('public/'))
super(ArtifactRecord, self).__init__(artifact_url,
name,
None,
digest,
algorithm,
unpack=True)
def createJsonArtifact(queue, taskid, runid, name, data, expires):
"""Creates a Taskcluster artifact for the given taskid and runid, and
uploads the artifact contents to location provided."""
data = json.dumps(data)
resp = queue.createArtifact(taskid, runid, name, {
"storageType": "s3",
"contentType": "application/json",
"expires": expires,
})
log.debug("Got %s", resp)
if resp.get("storageType") != "s3":
raise ValueError("Can't upload artifact for task %s with runid %s because storageType is wrong" % (taskid, runid))
assert resp["storageType"] == "s3"
put_url = resp["putUrl"]
log.debug("Uploading to %s", put_url)
for _ in retrier():
try:
resp = requests.put(put_url, data=data, headers={
"Content-Type": "application/json",
"Content-Length": len(data),
})
log.debug("Got %s %s", resp, resp.headers)
return
except Exception:
log.debug("Error submitting to s3", exc_info=True)
continue
else:
log.error("couldn't upload artifact to s3")
raise IOError("couldn't upload artifact to s3")
def talent_pool_candidate_api(access_token,
talent_pool_id,
data='',
action='GET',
expected_count=0):
headers = {'Authorization': 'Bearer %s' % access_token}
if action == 'GET':
response = None
for _ in retrier(attempts=33, sleeptime=3):
response = requests.get(
url=CandidatePoolApiUrl.TALENT_POOL_CANDIDATE % talent_pool_id,
headers=headers)
if not response.ok or response.json().get(
'talent_pool_candidates').get(
'total_found') >= expected_count:
break
return response.json(), response.status_code
elif action == 'DELETE':
headers['content-type'] = 'application/json'
response = requests.delete(
url=CandidatePoolApiUrl.TALENT_POOL_CANDIDATE % talent_pool_id,
data=json.dumps(data),
headers=headers)
return response.json(), response.status_code
elif action == 'POST':
headers['content-type'] = 'application/json'
response = requests.post(
url=CandidatePoolApiUrl.TALENT_POOL_CANDIDATE % talent_pool_id,
headers=headers,
data=json.dumps(data))
return response.json(), response.status_code
else:
raise Exception('No valid action is provided')
def test_configuration(self, standard_setup_one_client):
"""Tests the deployment and reporting of the configuration
The tests set the configuration of a device and verifies the new
configuration is reported back to the back-end.
"""
# accept the device
devauth.accept_devices(1)
# list of devices
devices = list(
set([
device["id"]
for device in devauth.get_devices_status("accepted")
]))
assert 1 == len(devices)
auth = authentication.Authentication()
wait_for_connect(auth, devices[0])
# set and verify the device's configuration
# retry to skip possible race conditions between update poll and update trigger
for _ in redo.retrier(attempts=3, sleeptime=1):
set_and_verify_config({"key": "value"}, devices[0],
auth.get_auth_token())
forced = was_update_forced(standard_setup_one_client.device)
if forced:
return
assert False, "the update check was never triggered"
def test_retrier_sleep_no_jitter(self):
"""Make sure retrier sleep is behaving"""
with mock.patch("time.sleep") as sleep:
# Test that normal sleep scaling works without a jitter
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=None):
pass
expected = [mock.call(x) for x in (10, 20, 40, 80)]
self.assertEquals(sleep.call_args_list, expected)
def test_retrier_sleep_no_jitter(self):
"""Make sure retrier sleep is behaving"""
with mock.patch("time.sleep") as sleep:
# Test that normal sleep scaling works without a jitter
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=None):
pass
expected = [mock.call(x) for x in (10, 20, 40, 80)]
self.assertEquals(sleep.call_args_list, expected)
def main():
if len(sys.argv) != 2:
print('Usage: uploadsymbols.py <zip file>', file=sys.stderr)
return 1
if not os.path.isfile(sys.argv[1]):
print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]),
file=sys.stderr)
return 1
symbols_zip = sys.argv[1]
if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs:
print(
'Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!',
file=sys.stderr)
return 1
token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.
format(token_file),
file=sys.stderr)
return 1
auth_token = open(token_file, 'r').read().strip()
print('Uploading symbol file "{0}" to "{1}"...'.format(sys.argv[1], url))
for _ in redo.retrier():
try:
r = requests.post(url,
files={'symbols.zip': open(sys.argv[1], 'rb')},
headers={'Auth-Token': auth_token},
allow_redirects=False,
timeout=120)
break
except requests.exceptions.RequestException as e:
print('Error: {0}'.format(e))
else:
print('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
print('Uploaded successfully!')
return 0
if r.status_code < 400:
print('Error: bad auth token? ({0}: {1})'.format(
r.status_code, r.reason),
file=sys.stderr)
else:
print('Error: got HTTP response {0}: {1}'.format(
r.status_code, r.reason),
file=sys.stderr)
print('Response body:\n{sep}\n{body}\n{sep}\n'.format(sep='=' * 20,
body=r.text))
return 1
def check_file_exists(url):
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
try:
resp = requests.head(url, allow_redirects=True)
return resp.status_code == requests.codes.ok
except requests.exceptions.RequestException as e:
log.error('Error: {0}'.format(e))
log.info('Retrying...')
return False
def start(self):
self.needs_restart = False
profile = self.create_profile()
shutil.rmtree(os.path.join(profile.path, self.client, 'better-bibtex'), ignore_errors=True)
if self.client == 'zotero':
datadir_profile = '-datadir profile'
else:
utils.print('\n\n** WORKAROUNDS FOR JURIS-M IN PLACE -- SEE https://github.com/Juris-M/zotero/issues/34 **\n\n')
datadir_profile = ''
cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -jsconsole -ZoteroDebugText {datadir_profile} {self.redir} {shlex.quote(profile.path + ".log")} 2>&1'
utils.print(f'Starting {self.client}: {cmd}')
self.proc = subprocess.Popen(cmd, shell=True)
utils.print(f'{self.client} started: {self.proc.pid}')
ready = False
self.config.stash()
self.config.timeout = 2
with benchmark(f'starting {self.client}') as bm:
posted = False
for _ in redo.retrier(attempts=120,sleeptime=1):
utils.print('connecting... (%.2fs)' % (bm.elapsed,))
try:
ready = self.execute("""
if (!Zotero.BetterBibTeX) {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded')
return false;
}
if (!Zotero.BetterBibTeX.ready) {
if (typeof Zotero.BetterBibTeX.ready === 'boolean') {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error')
} else {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized')
}
return false;
}
Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...')
await Zotero.BetterBibTeX.ready;
if (testing && !Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!')
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!');
return true;
""", testing = self.testing)
if ready: break
except (urllib.error.HTTPError, urllib.error.URLError,socket.timeout):
pass
if bm.elapsed > 2000 and not posted: posted = post_log()
assert ready, f'{self.client} did not start'
self.config.pop()
if self.import_at_start:
self.execute(f'return await Zotero.BetterBibTeX.TestSupport.importFile({json.dumps(self.import_at_start)})')
self.import_at_start = None
def main():
if len(sys.argv) != 2:
print('Usage: uploadsymbols.py <zip file>', file=sys.stderr)
return 1
if not os.path.isfile(sys.argv[1]):
print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]),
file=sys.stderr)
return 1
symbols_zip = sys.argv[1]
if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs:
print('Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!', file=sys.stderr)
return 1
token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file), file=sys.stderr)
return 1
auth_token = open(token_file, 'r').read().strip()
print('Uploading symbol file "{0}" to "{1}"...'.format(sys.argv[1], url))
for _ in redo.retrier():
try:
r = requests.post(
url,
files={'symbols.zip': open(sys.argv[1], 'rb')},
headers={'Auth-Token': auth_token},
allow_redirects=False,
timeout=120)
break
except requests.exceptions.RequestException as e:
print('Error: {0}'.format(e))
else:
print('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
print('Uploaded successfully!')
return 0
if r.status_code < 400:
print('Error: bad auth token? ({0}: {1})'.format(r.status_code,
r.reason),
file=sys.stderr)
else:
print('Error: got HTTP response {0}: {1}'.format(r.status_code,
r.reason),
file=sys.stderr)
print('Response body:\n{sep}\n{body}\n{sep}\n'.format(
sep='=' * 20,
body=r.text
))
return 1
def downloadReleaseBuilds(stageServer,
productName,
brandName,
version,
buildNumber,
platform,
candidatesDir=None,
signed=False,
usePymake=False):
if candidatesDir is None:
candidatesDir = makeCandidatesDir(productName,
version,
buildNumber,
protocol='http',
server=stageServer)
files = makeReleaseRepackUrls(productName,
brandName,
version,
platform,
signed=signed)
env = {}
for fileName, remoteFile in files.iteritems():
url = '/'.join(
[p.strip('/')
for p in [candidatesDir, urllib.quote(remoteFile)]])
log.info("Downloading %s to %s", url, fileName)
for _ in retrier():
with open(fileName, "wb") as f:
try:
r = requests.get(url, stream=True, timeout=15)
r.raise_for_status()
for chunk in r.iter_content(chunk_size=5 * 1024**2):
f.write(chunk)
r.close()
break
except (requests.HTTPError, requests.ConnectionError,
requests.Timeout):
log.exception("Caught exception downloading")
if fileName.endswith('exe'):
if usePymake:
env['WIN32_INSTALLER_IN'] = msys2windows(
path.join(os.getcwd(), fileName))
else:
env['WIN32_INSTALLER_IN'] = windows2msys(
path.join(os.getcwd(), fileName))
else:
if platform.startswith('win') and not usePymake:
env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName))
else:
env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName))
return env
def start(self):
self.needs_restart = False
profile = self.create_profile()
cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -jsconsole -ZoteroDebugText -datadir profile {self.redir} {shlex.quote(profile.path + ".log")} 2>&1'
utils.print(f'Starting {self.client}: {cmd}')
self.proc = subprocess.Popen(cmd, shell=True)
utils.print(f'{self.client} started: {self.proc.pid}')
ready = False
self.config.stash()
self.config.timeout = 2
with benchmark(f'starting {self.client}') as bm:
posted = None
for _ in redo.retrier(attempts=120, sleeptime=1):
utils.print('connecting... (%.2fs)' % (bm.elapsed, ))
try:
ready = self.execute("""
if (!Zotero.BetterBibTeX) {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded')
return false;
}
if (!Zotero.BetterBibTeX.ready) {
if (typeof Zotero.BetterBibTeX.ready === 'boolean') {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error')
} else {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized')
}
return false;
}
Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...')
await Zotero.BetterBibTeX.ready;
if (testing && !Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!')
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!');
return true;
""",
testing=self.testing)
if ready: break
except (urllib.error.HTTPError, urllib.error.URLError,
socket.timeout):
pass
if bm.elapsed > 2000 and not posted: posted = PostLog()
if posted:
utils.print(
'connected, but log posted, waiting for upload to finish...'
)
posted.join()
assert ready, f'{self.client} did not start'
self.config.pop()
def test_retrier_jitter(self):
with mock.patch("time.sleep") as sleep:
# Test that jitter works
with mock.patch("random.uniform") as uniform:
uniform.return_value = 3
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300,
sleepscale=2, jitter=3):
uniform.return_value *= -1
expected = [mock.call(x) for x in (7, 23, 37, 83)]
self.assertEquals(sleep.call_args_list, expected)
self.assertEquals(uniform.call_args, mock.call(-48, 48))
def ping_with_retries(self, attempts):
retrier_kwargs = {
'attempts': attempts,
'sleeptime': 0.3,
'jitter': 0
}
for _ in redo.retrier(**retrier_kwargs):
if self.ping():
return True
return False
def test_retrier_jitter(self):
with mock.patch("time.sleep") as sleep:
# Test that jitter works
with mock.patch("random.randint") as randint:
randint.return_value = 3
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300,
sleepscale=2, jitter=3):
randint.return_value *= -1
expected = [mock.call(x) for x in (7, 23, 37, 83)]
self.assertEquals(sleep.call_args_list, expected)
self.assertEquals(randint.call_args, mock.call(-48, 48))
def main():
if len(sys.argv) != 2:
print('Usage: uploadsymbols.py <zip file>', file=sys.stderr)
return 1
if not os.path.isfile(sys.argv[1]):
print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]),
file=sys.stderr)
return 1
symbols_zip = sys.argv[1]
if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs:
print(
'Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!',
file=sys.stderr)
return 1
token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.
format(token_file),
file=sys.stderr)
return 1
auth_token = open(token_file, 'r').read().strip()
print('Uploading symbol file "{0}" to "{1}"'.format(sys.argv[1], url))
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
print('Attempt %d of %d...' % (i, MAX_RETRIES))
try:
r = requests.post(url,
files={'symbols.zip': open(sys.argv[1], 'rb')},
headers={'Auth-Token': auth_token},
allow_redirects=False,
timeout=120)
# 500 is likely to be a transient failure.
# Break out for success or other error codes.
if r.status_code < 500:
break
print_error(r)
except requests.exceptions.RequestException as e:
print('Error: {0}'.format(e))
print('Retrying...')
else:
print('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
print('Uploaded successfully!')
return 0
print_error(r)
return 1
def upload_symbols(zip_file, token_file):
print("Uploading symbols file '{0}' to '{1}'".format(
zip_file, DEFAULT_SYMBOL_URL),
file=sys.stdout)
zip_name = os.path.basename(zip_file)
# XXX: fetch the symbol upload token from local file, taskgraph handles
# already that communication with Taskcluster to get the credentials for
# communicating with the server
auth_token = ''
with open(token_file, 'r') as f:
auth_token = f.read().strip()
if len(auth_token) == 0:
print("Failed to get the symbol token.", file=sys.stderr)
if auth_token == 'faketoken':
print("'faketoken` detected, not pushing anything", file=sys.stdout)
sys.exit(0)
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
print("Attempt %d of %d..." % (i, MAX_RETRIES))
try:
if zip_file.startswith("http"):
zip_arg = {"data": {"url", zip_file}}
else:
zip_arg = {"files": {zip_name: open(zip_file, 'rb')}}
r = requests.post(
DEFAULT_SYMBOL_URL,
headers={"Auth-Token": auth_token},
allow_redirects=False,
# Allow a longer read timeout because uploading by URL means the server
# has to fetch the entire zip file, which can take a while. The load balancer
# in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
timeout=(10, 300),
**zip_arg)
# 500 is likely to be a transient failure.
# Break out for success or other error codes.
if r.status_code < 500:
break
print("Error: {0}".format(r), file=sys.stderr)
except requests.exceptions.RequestException as e:
print("Error: {0}".format(e), file=sys.stderr)
print("Retrying...", file=sys.stdout)
else:
print("Maximum retries hit, giving up!", file=sys.stderr)
return False
if r.status_code >= 200 and r.status_code < 300:
print("Uploaded successfully", file=sys.stdout)
return True
print("Upload symbols failed: {0}".format(r), file=sys.stderr)
return False
def _prepare_device(
self,
azure_user: User,
httpserver: HTTPServer,
httpserver_ssl_context: ssl.SSLContext,
) -> Device:
"""Create accepted device in Mender and make sure it has been successfully added in Azure IoT Hub."""
if self.azure_iot_hub_mock:
httpserver.expect_oneshot_request(
re.compile("^/devices"),
method="PUT",
query_string="api-version=2021-04-12",
).respond_with_json(self._prepare_iot_hub_upsert_device_response())
httpserver.expect_oneshot_request(
re.compile("^/devices"),
method="GET",
query_string="api-version=2021-04-12",
).respond_with_data(status=200)
httpserver.expect_oneshot_request(
re.compile("^/devices"),
method="PUT",
query_string="api-version=2021-04-12",
).respond_with_data(status=200)
httpserver.expect_oneshot_request(
re.compile("^/twins"),
method="PATCH",
query_string="api-version=2021-04-12",
).respond_with_data(status=200)
tenant_token = getattr(getattr(azure_user, "tenant", {}),
"tenant_token", "")
dev = make_accepted_device(
self.api_devauth_devices,
self.api_devauth_mgmt,
azure_user.token,
tenant_token=tenant_token,
test_type="azure",
)
self.devices.append(dev.id)
for _ in retrier(attempts=5, sleeptime=1):
if self.azure_iot_hub_mock:
httpserver.expect_oneshot_request(
re.compile("^/twins"),
method="GET",
query_string="api-version=2021-04-12",
).respond_with_json(
self._prepare_iot_hub_upsert_device_response())
rsp = self.api_azure.with_auth(azure_user.token).call(
"GET", iot.URL_DEVICE(dev.id))
if rsp.status_code == 200:
break
return dev
def retry_aws_request(callable, *args, **kwargs):
"""Calls callable(*args, **kwargs), and sleeps/retries on
RequestLimitExceeded errors"""
for _ in retrier():
try:
return callable(*args, **kwargs)
except BotoServerError, e:
if e.code == 'RequestLimitExceeded':
# Try again
log.debug("Got RequestLimitExceeded; retrying", exc_info=True)
continue
# Otherwise re-raise
raise
def get_and_assert_zero(url, key, token, sleep_time=SLEEP_TIME):
"""
This function gets list of objects from given url and asserts that length of objects under a given key is zero.
It keeps on retrying this process until it founds some records or sleep_time is over
:param string url: URL of requested resource
:param string key: key in response that has resource list
:param string token: user access token
:param int sleep_time: maximum time to wait
"""
attempts = sleep_time / SLEEP_INTERVAL
for _ in retrier(attempts=attempts, sleeptime=SLEEP_INTERVAL,
sleepscale=1):
assert len(send_request('get', url, token).json()[key]) == 0
def retry_aws_request(callable, *args, **kwargs):
"""Calls callable(*args, **kwargs), and sleeps/retries on
RequestLimitExceeded errors"""
for _ in retrier():
try:
return callable(*args, **kwargs)
except BotoServerError, e:
if e.code == 'RequestLimitExceeded':
# Try again
log.debug("Got RequestLimitExceeded; retrying", exc_info=True)
continue
# Otherwise re-raise
raise
def Upload_Symbols(zip_file):
print("Uploading symbols file '{0}' to '{1}'".format(
zip_file, DEFAULT_SYMBOL_URL),
file=sys.stdout)
zip_name = os.path.basename(zip_file)
# Fetch the symbol server token from Taskcluster secrets.
secrets_url = "http://taskcluster/secrets/v1/secret/{}".format(
"project/application-services/symbols-token")
res = requests.get(secrets_url)
res.raise_for_status()
secret = res.json()
auth_token = secret["secret"]["token"]
if len(auth_token) == 0:
print("Failed to get the symbol token.", file=sys.stderr)
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
print("Attempt %d of %d..." % (i, MAX_RETRIES))
try:
if zip_file.startswith("http"):
zip_arg = {"data": {"url", zip_file}}
else:
zip_arg = {"files": {zip_name: open(zip_file, 'rb')}}
r = requests.post(
DEFAULT_SYMBOL_URL,
headers={"Auth-Token": auth_token},
allow_redirects=False,
# Allow a longer read timeout because uploading by URL means the server
# has to fetch the entire zip file, which can take a while. The load balancer
# in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
timeout=(10, 300),
**zip_arg)
# 500 is likely to be a transient failure.
# Break out for success or other error codes.
if r.status_code < 500:
break
print("Error: {0}".format(r), file=sys.stderr)
except requests.exceptions.RequestException as e:
print("Error: {0}".format(e), file=sys.stderr)
print("Retrying...", file=sys.stdout)
else:
print("Maximum retries hit, giving up!", file=sys.stderr)
return False
if r.status_code >= 200 and r.status_code < 300:
print("Uploaded successfully", file=sys.stdout)
return True
print("Upload symbols failed: {0}".format(r), file=sys.stderr)
return False
def reboot(self, timeout=60, reconnect_attempts=5):
command_schedule_time = 10
self.sudo_exec("shutdown -r -t {}".format(command_schedule_time))
self.close()
time.sleep(command_schedule_time)
retries = 0
for _ in redo.retrier(sleeptime=timeout, attempts=reconnect_attempts):
try:
self.connect()
return
except _reconnect_exceptions:
retries += 1
logger.info("Trying to connect to %s after reboot issued for the %d time.", self._host, retries)
raise RuntimeError("Unable to connect to %s after issuing reboot.", self._host)
def get_auditlog_time(oid: str):
# get exact time for filter testing
found = None
for _ in redo.retrier(attempts=3, sleeptime=1):
resp = self.alogs.with_auth(tenant_users.users[0].token).call(
"GET", auditlogs.URL_LOGS)
found = [
audit_log for audit_log in resp.json()
if audit_log["object"]["id"] == oid
]
if len(found) == 1:
return found[0]["time"]
else:
assert False, "max GET /logs retries hit"
def _retry_on_http_errors(url, auth, verify, params, errors):
for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
try:
req = requests.get(url, auth=auth, verify=verify, params=params)
req.raise_for_status()
return req
except requests.HTTPError as e:
if e.response.status_code in errors:
log.exception("Got HTTP %s trying to reach %s",
e.response.status_code, url)
else:
raise
else:
raise
def get_auditlog_time(oid):
# get exact time for filter testing
found = None
for _ in redo.retrier(attempts=3, sleeptime=1):
alogs = ApiClient(auditlogs.URL_MGMT)
resp = alogs.with_auth(tenant_users.users[0].token).call(
"GET", auditlogs.URL_LOGS
)
resp = resp.json()
found = [e for e in resp if e["object"]["id"] == oid]
if len(found) == 1:
return found[0]["time"]
else:
assert False, "max GET /logs retries hit"
def main():
if len(sys.argv) != 2:
print('Usage: uploadsymbols.py <zip file>', file=sys.stderr)
return 1
if not os.path.isfile(sys.argv[1]):
print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]),
file=sys.stderr)
return 1
symbols_zip = sys.argv[1]
if 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' not in substs:
print('Error: you must set SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE in your mozconfig!', file=sys.stderr)
return 1
token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file), file=sys.stderr)
return 1
auth_token = open(token_file, 'r').read().strip()
print('Uploading symbol file "{0}" to "{1}"'.format(sys.argv[1], url))
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
print('Attempt %d of %d...' % (i, MAX_RETRIES))
try:
r = requests.post(
url,
files={'symbols.zip': open(sys.argv[1], 'rb')},
headers={'Auth-Token': auth_token},
allow_redirects=False,
timeout=120)
# 500 is likely to be a transient failure.
# Break out for success or other error codes.
if r.status_code < 500:
break
print_error(r)
except requests.exceptions.RequestException as e:
print('Error: {0}'.format(e))
print('Retrying...')
else:
print('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
print('Uploaded successfully!')
return 0
print_error(r)
return 1
def test_retrier_jitter(self):
with mock.patch("time.sleep") as sleep:
# Test that jitter works
with mock.patch("random.randint") as randint:
randint.return_value = 3
for _ in retrier(attempts=5,
sleeptime=10,
max_sleeptime=300,
sleepscale=2,
jitter=3):
randint.return_value *= -1
expected = [mock.call(x) for x in (7, 17, 31, 65)]
self.assertEquals(sleep.call_args_list, expected)
self.assertEquals(randint.call_args, mock.call(-3, 3))
def test_get_params(self, tenant_users):
""" Mix up some audiltog events, check GET with various params """
# N various events from both users
events = []
for i in range(10):
uidx = i % 2
user = tenant_users.users[uidx]
evt = None
oid = None
if i % 3 == 0:
uuidv4 = str(uuid.uuid4())
uid = make_user(user.token, uuidv4 + "@acme.com",
"secretsecret")
evt = evt_user_create(user, uid, uuidv4 + "@acme.com")
oid = uid
else:
d = make_deployment(user.token)
evt = evt_deployment_create(user, d)
oid = d["id"]
time.sleep(0.5)
# get exact time for filter testing
found = None
for _ in redo.retrier(attempts=3, sleeptime=1):
alogs = ApiClient(auditlogs.URL_MGMT)
resp = alogs.with_auth(tenant_users.users[0].token).call(
"GET", auditlogs.URL_LOGS)
resp = resp.json()
found = [e for e in resp if e["object"]["id"] == oid]
if len(found) == 1:
break
else:
assert False, "max GET /logs retries hit"
evt["time"] = found[0]["time"]
events.append(evt)
# default sorting is desc by time
events.reverse()
self._test_args_paging(tenant_users, events)
self._test_args_actor(tenant_users, events)
self._test_args_before_after(tenant_users, events)
self._test_args_object(tenant_users, events)
def get_mender_gateway(self):
"""Returns IP address of mender-api-gateway service
Has internal retry - upon setup 'up', the gateway
will not be available for a while.
"""
for _ in redo.retrier(attempts=10, sleeptime=1):
gateway = self.get_ip_of_service("mender-api-gateway")
if len(gateway) != 1:
continue
else:
return gateway[0]
else:
assert (
False
), "expected one instance of api-gateway running, but found: {} instance(s)".format(
len(gateway))
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
"""Pulls changes from hg repo and places it in `dest`.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and pull from the mirrors first before
`repo`."""
if mirrors:
for mirror in mirrors:
try:
return pull(mirror, dest, update_dest=update_dest, **kwargs)
except:
log.exception("Problem pulling from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# Convert repo to an absolute path if it's a local repository
repo = _make_absolute(repo)
cmd = ['pull']
# Don't pass -r to "hg pull", except when it's a valid HG revision.
# Pulling using tag names is dangerous: it uses the local .hgtags, so if
# the tag has moved on the remote side you won't pull the new revision the
# remote tag refers to.
pull_kwargs = kwargs.copy()
if 'revision' in pull_kwargs and \
not is_hg_cset(pull_kwargs['revision']):
del pull_kwargs['revision']
cmd.extend(common_args(**pull_kwargs))
cmd.append(repo)
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS):
try:
get_hg_output(cmd=cmd, cwd=dest, include_stderr=True)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
continue
raise
def revision_to_revision_hash(th_api_root, branch, revision):
url = "{th_api_root}/project/{branch}/resultset".format(
th_api_root=th_api_root, branch=branch
)
# Use short revision for treeherder API
revision = revision[:12]
params = {"revision": revision}
for _ in redo.retrier(sleeptime=5, max_sleeptime=30):
params_str = "&".join("=".join([k, str(v)])
for k, v in params.iteritems())
try:
log.debug("Connecting to %s?%s", url, params_str)
r = requests.get(url, params=params)
return r.json()["results"][0]["revision_hash"]
except:
log.exception("Failed to connect to %s?%s", url, params_str)
else:
raise RuntimeError("Cannot fetch revision hash for {} {}".format(
branch, revision))
def _retry_on_http_errors(url, verify, params, errors):
if params:
params_str = "&".join("=".join([k, str(v)])
for k, v in params.iteritems())
else:
params_str = ''
logger.info("Connecting to %s?%s", url, params_str)
for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
try:
req = requests.get(url, verify=verify, params=params, timeout=4)
req.raise_for_status()
return req
except requests.HTTPError as e:
if e.response.status_code in errors:
logger.exception("Got HTTP %s trying to reach %s",
e.response.status_code, url)
else:
raise
else:
raise
def downloadReleaseBuilds(stageServer, productName, brandName, version,
buildNumber, platform, candidatesDir=None,
signed=False, usePymake=False):
if candidatesDir is None:
candidatesDir = makeCandidatesDir(productName, version, buildNumber,
protocol='http', server=stageServer)
files = makeReleaseRepackUrls(productName, brandName, version, platform,
signed=signed)
env = {}
for fileName, remoteFile in files.iteritems():
url = '/'.join([p.strip('/') for p in [candidatesDir,
urllib.quote(remoteFile)]])
log.info("Downloading %s to %s", url, fileName)
for _ in retrier():
with open(fileName, "wb") as f:
try:
r = requests.get(url, stream=True, timeout=15)
r.raise_for_status()
for chunk in r.iter_content(chunk_size=5*1024**2):
f.write(chunk)
r.close()
break
except (requests.HTTPError, requests.ConnectionError,
requests.Timeout):
log.exception("Caught exception downloading")
if fileName.endswith('exe'):
if usePymake:
env['WIN32_INSTALLER_IN'] = msys2windows(path.join(os.getcwd(),
fileName))
else:
env['WIN32_INSTALLER_IN'] = windows2msys(path.join(os.getcwd(),
fileName))
else:
if platform.startswith('win') and not usePymake:
env['ZIP_IN'] = windows2msys(path.join(os.getcwd(), fileName))
else:
env['ZIP_IN'] = msys2windows(path.join(os.getcwd(), fileName))
return env
def shutdown(self):
if self.proc is None: return
# graceful shutdown
try:
self.execute("""
const appStartup = Components.classes['@mozilla.org/toolkit/app-startup;1'].getService(Components.interfaces.nsIAppStartup);
appStartup.quit(Components.interfaces.nsIAppStartup.eAttemptQuit);
""")
except:
pass
stopped = False
for _ in redo.retrier(attempts=5,sleeptime=1):
stopped = not running(self.proc.pid)
if stopped: break
zotero = psutil.Process(self.proc.pid)
for proc in zotero.children(recursive=True):
proc.kill()
zotero.kill()
def start(self):
profile = Profile('BBTZ5TEST', self.id, self.userdata)
cmd = f'{shlex.quote(profile.binary)} -P {shlex.quote(profile.name)} -ZoteroDebugText -datadir profile > {shlex.quote(profile.path + ".log")} 2>&1'
print(f'Starting {self.id}: {cmd}')
self.proc = subprocess.Popen(cmd, shell=True)
print(f'{self.id} started: {self.proc.pid}')
ready = False
with benchmark(f'starting {self.id}'):
for _ in redo.retrier(attempts=30,sleeptime=1):
print('connecting...')
try:
ready = self.execute("""
if (!Zotero.BetterBibTeX) {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not loaded')
return false;
}
if (!Zotero.BetterBibTeX.ready) {
if (typeof Zotero.BetterBibTeX.ready === 'boolean') {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX initialization error')
} else {
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX not initialized')
}
return false;
}
Zotero.debug('{better-bibtex:debug bridge}: startup: waiting for BetterBibTeX ready...')
await Zotero.BetterBibTeX.ready;
if (!Zotero.Prefs.get('translators.better-bibtex.testing')) throw new Error('translators.better-bibtex.testing not set!')
Zotero.debug('{better-bibtex:debug bridge}: startup: BetterBibTeX ready!');
return true;
""")
if ready: break
except (urllib.error.HTTPError, urllib.error.URLError):
pass
assert ready, f'{self.id} did not start'
def main():
config = MozbuildObject.from_environment()
config._activate_virtualenv()
import redo
import requests
logging.basicConfig()
parser = argparse.ArgumentParser(
description='Upload symbols in ZIP using token from Taskcluster secrets service.')
parser.add_argument('zip',
help='Symbols zip file - URL or path to local file')
args = parser.parse_args()
if not args.zip.startswith('http') and not os.path.isfile(args.zip):
log.error('Error: zip file "{0}" does not exist!'.format(args.zip))
return 1
secret_name = os.environ.get('SYMBOL_SECRET')
if secret_name is not None:
auth_token = get_taskcluster_secret(secret_name)
elif 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' in os.environ:
token_file = os.environ['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
log.error('SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file))
return 1
auth_token = open(token_file, 'r').read().strip()
else:
log.error('You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE '
'environment variables!')
return 1
# Allow overwriting of the upload url with an environmental variable
if 'SOCORRO_SYMBOL_UPLOAD_URL' in os.environ:
url = os.environ['SOCORRO_SYMBOL_UPLOAD_URL']
elif os.environ.get('MOZ_SCM_LEVEL', '1') == '1':
# Use the Tecken staging server for try uploads for now.
# This will eventually be changed in bug 1138617.
url = 'https://symbols.stage.mozaws.net/upload/'
else:
url = DEFAULT_URL
log.info('Uploading symbol file "{0}" to "{1}"'.format(args.zip, url))
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
log.info('Attempt %d of %d...' % (i, MAX_RETRIES))
try:
if args.zip.startswith('http'):
zip_arg = {'data': {'url': args.zip}}
else:
zip_arg = {'files': {'symbols.zip': open(args.zip, 'rb')}}
r = requests.post(
url,
headers={'Auth-Token': auth_token},
allow_redirects=False,
# Allow a longer read timeout because uploading by URL means the server
# has to fetch the entire zip file, which can take a while. The load balancer
# in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
timeout=(10, 300),
**zip_arg)
# 500 is likely to be a transient failure.
# Break out for success or other error codes.
if r.status_code < 500:
break
print_error(r)
except requests.exceptions.RequestException as e:
log.error('Error: {0}'.format(e))
log.info('Retrying...')
else:
log.warn('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
log.info('Uploaded successfully!')
return 0
print_error(r)
return 1
def test_retrier_yields(self):
"""Ensure that retrier yields the sleep time"""
for real_sleeptime in retrier(attempts=3, sleeptime=1, sleepscale=1, jitter=0):
self.assertEqual(real_sleeptime, 1)
def test_jitter_bounds(self):
self.assertRaises(Exception, retrier(sleeptime=1, jitter=2))
def test_retrier(self):
"""Make sure retrier behaves properly"""
n = 0
for _ in retrier(attempts=5, sleeptime=0, jitter=0):
n += 1
self.assertEqual(n, 5)
def http_request(
self,
method,
path,
params={},
headers={},
json=None,
etag=None,
endpoint=None,
retry=False,
):
_headers = self._http_headers['default'].copy()
_headers.update(self._http_headers[method])
_headers.update(headers)
headers = _headers
token = self.get_bearer_token()
if self.logged_in:
headers.update({
'Authorization': 'Bearer %s' % token,
})
if etag:
headers.update({
'If-Match': etag,
})
if endpoint:
url = endpoint + '/' + path
else:
url = self.endpoint + '/api' + path
# Setting the parameter at all (even False) turns on admin mode
if self.admin:
params.update({'admin': self.admin})
if params:
self.logger.debug(
"params={}".format(params)
)
if json:
self.logger.debug(
"json={}".format(json)
)
if retry:
retry_attempts = HTTP_RETRY_LIMIT
else:
retry_attempts = 1
for _ in retrier(
attempts=retry_attempts,
sleeptime=RETRY_BACKOFF_INTERVAL,
):
response = self.session.request(
method,
url,
params=params,
headers=headers,
json=json,
)
if response.status_code < 500:
break
else:
raise PanoptesAPIException(
'Received HTTP status code {} from API'.format(
response.status_code
)
)
return response
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, mirrors=None, bundles=None):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it. If successful, will pull in new revisions from mirrors or
the master repo. If unbundling fails, will fall back to doing a regular
clone from mirrors or the master repo.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
"""
if os.path.exists(dest):
remove_path(dest)
if bundles:
log.info("Attempting to initialize clone with bundles")
for bundle in bundles:
if os.path.exists(dest):
remove_path(dest)
init(dest)
log.info("Trying to use bundle %s", bundle)
try:
if not unbundle(bundle, dest):
remove_path(dest)
continue
adjust_paths(dest, default=repo)
# Now pull / update
return pull(repo, dest, update_dest=update_dest,
mirrors=mirrors, revision=revision, branch=branch)
except Exception:
remove_path(dest)
log.exception("Problem unbundling/pulling from %s", bundle)
continue
else:
log.info("Using bundles failed; falling back to clone")
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, branch, revision,
update_dest=update_dest, clone_by_rev=clone_by_rev)
adjust_paths(dest, default=repo)
return retval
except:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# We may have a partial repo here; mercurial() copes with that
# We need to make sure our paths are correct though
if os.path.exists(os.path.join(dest, '.hg')):
adjust_paths(dest, default=repo)
return mercurial(repo, dest, branch, revision, autoPurge=True,
update_dest=update_dest, clone_by_rev=clone_by_rev)
cmd = ['clone']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS):
try:
get_hg_output(cmd=cmd, include_stderr=True)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
