def get_destination_for_partner_repack_path(context, manifest, full_path, locale):
"""Function to process the final destination path, relative to the root of
the S3 bucket. Depending on whether it's a private or public destination, it
performs several string manipulations.
Input: 'releng/partner/ghost/ghost-var/v1/linux-i686/ro/target.tar.bz2'
Possible ouput(s):
-> ghost/59.0b20-2/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
-> pub/firefox/candidates/59.0b20-candidates/build2/partner-repacks/ghost/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
"""
# make sure we're calling this function from private-partner context
if not is_partner_action(context.action):
raise ScriptWorkerRetryException("Outside of private-partner context!")
# pretty name the `target` part to the actual filename
pretty_full_path = os.path.join(locale, manifest["mapping"][locale][os.path.basename(full_path)])
build_number = context.task["payload"]["build_number"]
version = context.task["payload"]["version"]
if is_partner_private_task(context):
sanity_check_partner_path(locale, {"version": version, "build_number": build_number}, PARTNER_REPACK_PRIVATE_REGEXES)
return pretty_full_path
elif is_partner_public_task(context):
sanity_check_partner_path(locale, {"version": version, "build_number": build_number}, PARTNER_REPACK_PUBLIC_REGEXES)
prefix = PARTNER_REPACK_PUBLIC_PREFIX_TMPL.format(version=version, build_number=build_number)
return os.path.join(prefix, pretty_full_path)
async def claim_task(context, taskId, runId):
"""Attempt to claim a task that we found in the Azure queue.
Args:
context (scriptworker.context.Context): the scriptworker context.
taskId (str): the taskcluster taskId to claim
runId (int): the taskcluster runId to claim
Returns:
dict: claimTask definition, if successful. If unsuccessful, return None.
"""
payload = {
'workerGroup': context.config['worker_group'],
'workerId': context.config['worker_id'],
}
try:
result = await context.queue.claimTask(taskId, runId, payload)
return result
except taskcluster.exceptions.TaskclusterFailure as exc:
log.debug("Got %s" % exc)
if hasattr(exc, 'status_code') and exc.status_code == 409:
# 409 means we found a task that's claimed by another worker
# or cancelled. Let's return None and delete it from Azure
# so we don't get a backlog of bogus tasks in the queue.
log.debug("Got %s" % exc)
return None
else:
raise ScriptWorkerRetryException(str(exc))
async def request(context,
url,
timeout=60,
method='get',
good=(200, ),
retry=tuple(range(500, 512)),
return_type='text',
**kwargs):
"""Async aiohttp request wrapper
"""
session = context.session
with aiohttp.Timeout(timeout):
log.debug("{} {}".format(method.upper(), url))
async with session.request(method, url, **kwargs) as resp:
log.debug("Status {}".format(resp.status))
message = "Bad status {}".format(resp.status)
if resp.status in retry:
raise ScriptWorkerRetryException(message)
if resp.status not in good:
raise ScriptWorkerException(message)
if return_type == 'text':
return await resp.text()
elif return_type == 'json':
return await resp.json()
else:
return resp
async def create_artifact(context,
path,
target_path,
content_type,
content_encoding,
storage_type='s3',
expires=None):
"""Create an artifact and upload it.
This should support s3 and azure out of the box; we'll need some tweaking
if we want to support redirect/error artifacts.
Args:
context (scriptworker.context.Context): the scriptworker context.
path (str): the path of the file to upload.
target_path (str):
content_type (str): Content type (MIME type) of the artifact. Values can be found via
scriptworker.artifacts.guess_content_type_and_encoding()
content_encoding (str): Encoding (per mimetypes' library) of the artifact. None is for no encoding. Values can
be found via scriptworker.artifacts.guess_content_type_and_encoding()
storage_type (str, optional): the taskcluster storage type to use.
Defaults to 's3'
expires (str, optional): datestring of when the artifact expires.
Defaults to None.
Raises:
ScriptWorkerRetryException: on failure.
"""
payload = {
"storageType": storage_type,
"expires": expires or get_expiration_arrow(context).isoformat(),
"contentType": content_type,
}
args = [
get_task_id(context.claim_task),
get_run_id(context.claim_task), target_path, payload
]
tc_response = await context.temp_queue.createArtifact(*args)
skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE]
loggable_url = get_loggable_url(tc_response['putUrl'])
log.info("uploading {path} to {url}...".format(path=path,
url=loggable_url))
with open(path, "rb") as fh:
async with async_timeout.timeout(
context.config['artifact_upload_timeout']):
async with context.session.put(tc_response['putUrl'],
data=fh,
headers=_craft_artifact_put_headers(
content_type, content_encoding),
skip_auto_headers=skip_auto_headers,
compress=False) as resp:
log.info("create_artifact {}: {}".format(path, resp.status))
response_text = await resp.text()
log.info(response_text)
if resp.status not in (200, 204):
raise ScriptWorkerRetryException(
"Bad status {}".format(resp.status), )
async def put(context, url, headers, abs_filename, session=None):
session = session or context.session
with open(abs_filename, "rb") as fh:
async with session.put(url, data=fh, headers=headers, compress=False) as resp:
log.info("put {}: {}".format(abs_filename, resp.status))
response_text = await resp.text()
if response_text:
log.info(response_text)
if resp.status not in (200, 204):
raise ScriptWorkerRetryException("Bad status {}".format(resp.status))
return resp
async def create_artifact(context,
path,
target_path,
storage_type='s3',
expires=None,
content_type=None):
"""Create an artifact and upload it.
This should support s3 and azure out of the box; we'll need some tweaking
if we want to support redirect/error artifacts.
Args:
context (scriptworker.context.Context): the scriptworker context.
path (str): the path of the file to upload.
target_path (str):
storage_type (str, optional): the taskcluster storage type to use.
Defaults to 's3'
expires (str, optional): datestring of when the artifact expires.
Defaults to None.
content_type (str, optional): Specify the content type of the artifact.
If None, use guess_content_type(). Defaults to None.
Raises:
ScriptWorkerRetryException: on failure.
"""
payload = {
"storageType": storage_type,
"expires": expires or get_expiration_arrow(context).isoformat(),
"contentType": content_type or guess_content_type(path),
}
args = [
context.claim_task['status']['taskId'], context.claim_task['runId'],
target_path, payload
]
tc_response = await context.temp_queue.createArtifact(*args)
headers = {
aiohttp.hdrs.CONTENT_TYPE: tc_response['contentType'],
}
skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE]
log.info("uploading {path} to {url}...".format(path=path,
url=tc_response['putUrl']))
with open(path, "rb") as fh:
with aiohttp.Timeout(context.config['artifact_upload_timeout']):
async with context.session.put(tc_response['putUrl'],
data=fh,
headers=headers,
skip_auto_headers=skip_auto_headers,
compress=False) as resp:
log.info(resp.status)
response_text = await resp.text()
log.info(response_text)
if resp.status not in (200, 204):
raise ScriptWorkerRetryException(
"Bad status {}".format(resp.status), )
async def request(context,
url,
timeout=60,
method='get',
good=(200, ),
retry=tuple(range(500, 512)),
return_type='text',
**kwargs):
"""Async aiohttp request wrapper.
Args:
context (scriptworker.context.Context): the scriptworker context.
url (str): the url to request
timeout (int, optional): timeout after this many seconds. Default is 60.
method (str, optional): The request method to use. Default is 'get'.
good (list, optional): the set of good status codes. Default is (200, )
retry (list, optional): the set of status codes that result in a retry.
Default is tuple(range(500, 512)).
return_type (str, optional): The type of value to return. Takes
'json' or 'text'; other values will return the response object.
Default is text.
**kwargs: the kwargs to send to the aiohttp request function.
Returns:
object: the response text() if return_type is 'text'; the response
json() if return_type is 'json'; the aiohttp request response
object otherwise.
Raises:
ScriptWorkerRetryException: if the status code is in the retry list.
ScriptWorkerException: if the status code is not in the retry list or
good list.
"""
session = context.session
loggable_url = get_loggable_url(url)
async with async_timeout.timeout(timeout):
log.debug("{} {}".format(method.upper(), loggable_url))
async with session.request(method, url, **kwargs) as resp:
log.debug("Status {}".format(resp.status))
message = "Bad status {}".format(resp.status)
if resp.status in retry:
raise ScriptWorkerRetryException(message)
if resp.status not in good:
raise ScriptWorkerException(message)
if return_type == 'text':
return await resp.text()
elif return_type == 'json':
return await resp.json()
else:
return resp
def get_destination_for_private_repack_path(context, manifest, full_path,
locale):
"""Function to process the final destination path, relative to the root of
the S3 bucket. Depending on whether it's a private or public destination, it
performs several string manipulations.
Input: 'releng/partner/ghost/ghost-var/v1/linux-i686/ro/target.tar.bz2'
Possible ouput(s):
-> ghost/59.0b20-2/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
-> pub/firefox/candidates/59.0b20-candidates/build2/partner-repacks/ghost/ghost-variant/v1/linux-i686/en-US/firefox-59.0b20.tar.bz2
"""
# make sure we're calling this function from private-partner context
if not is_partner_action(context.action):
raise ScriptWorkerRetryException("Outside of private-partner context!")
# pretty name the `target` part to the actual filename
pretty_full_path = os.path.join(
os.path.dirname(full_path),
manifest['mapping'][locale][os.path.basename(full_path)])
# get rid of leading "releng/partner"
if pretty_full_path.startswith(PARTNER_LEADING_STRING):
pretty_full_path = pretty_full_path[len(PARTNER_LEADING_STRING):]
build_number = context.task['payload']['build_number']
version = context.task['payload']['version']
if is_partner_private_task(context):
elements = pretty_full_path.split('/')
identifier = '{version}-{build_number}'.format(
version=version, build_number=build_number)
# we need need to manually insert the "version-buildno" identifier in
# between `partner` # and `partner-variant` to be consistent
elements.insert(1, identifier)
# TODO: potentially need to remove the `v1` from the path?
path = '/'.join(elements)
# XXX: temp hack until bug 1447673 is solved
if context.bucket == "dep":
prefix = PARTNER_REPACK_PUBLIC_PREFIX_TMPL.format(
version=version, build_number=build_number)
path = os.path.join(prefix, path)
return path
elif is_partner_public_task(context):
prefix = PARTNER_REPACK_PUBLIC_PREFIX_TMPL.format(
version=version, build_number=build_number)
return os.path.join(prefix, pretty_full_path)
async def create_artifact(context,
path,
storage_type='s3',
expires=None,
content_type=None):
"""Create an artifact and upload it. This should support s3 and azure
out of the box; we'll need some tweaking if we want to support
redirect/error artifacts.
"""
temp_queue = get_temp_queue(context)
filename = os.path.basename(path)
payload = {
"storageType": storage_type,
"expires": expires or get_expiration_arrow(context).isoformat(),
"contentType": content_type or guess_content_type(path),
}
target_path = "public/env/{}".format(filename)
args = [
context.claim_task['status']['taskId'], context.claim_task['runId'],
target_path, payload
]
tc_response = await temp_queue.createArtifact(*args)
headers = {
aiohttp.hdrs.CONTENT_TYPE: tc_response['contentType'],
}
skip_auto_headers = [aiohttp.hdrs.CONTENT_TYPE]
log.info("uploading {path} to {url}...".format(path=path,
url=tc_response['putUrl']))
with open(path, "rb") as fh:
with aiohttp.Timeout(context.config['artifact_upload_timeout']):
async with context.session.put(tc_response['putUrl'],
data=fh,
headers=headers,
skip_auto_headers=skip_auto_headers,
compress=False) as resp:
log.info(resp.status)
response_text = await resp.text()
log.info(response_text)
if resp.status not in (200, 204):
raise ScriptWorkerRetryException(
"Bad status {}".format(resp.status), )
async def fail_first(*args, **kwargs):
global retry_count
retry_count["fail_first"] += 1
if retry_count["fail_first"] < 2:
raise ScriptWorkerRetryException("first")
return "yay"
async def req(_, url, **kwargs):
if url == "poll":
return await fake_request()
if raises:
raise ScriptWorkerRetryException("died in req")
async def claim(*args, **kwargs):
if counters['claim_task']:
return None
counters['claim_task'] = counters['claim_task'] + 1
raise ScriptWorkerRetryException("died in claim")