def receive_build(request, cid):
'''receive_build will receive the post from Google Cloud Build.
we check the response header against the jwt token to authenticate,
and then check other metadata and permissions in complete_build.
'''
print(request.body)
print(cid)
if request.method == "POST":
# Must be an existing container
container = get_container(cid)
if container is None:
return JsonResponseMessage(message="Invalid request.")
# Decode parameters
params = json.loads(request.body.decode('utf-8'))
# Must include a jwt token that is valid for the container
if not validate_jwt(container, params):
return JsonResponseMessage(message="Invalid request.")
scheduler = django_rq.get_scheduler('default')
scheduler.enqueue_in(timedelta(seconds=10),
complete_build,
cid=container.id,
params=params)
return JsonResponseMessage(message="Notification Received",
status=200,
status_message="Received")
return JsonResponseMessage(message="Invalid request.")
def delete_build(cid, client=None):
'''Delete artifacts for a container build, if they exist, along
with the container object. This is called
as a django-rq task for a worker to do from views.py
Parameters
==========
cid: the container id to finish the build for, expected to have an id
'''
from shub.apps.main.views import get_container
container = get_container(cid)
# if being called from delete_container_collection, just instantiate once
if client is None:
context = get_build_context()
client = get_client(debug=True, **context)
# If the container has an image, delete it
image = container.get_image() or ""
if container.metadata['builder']['name'] == "google_build":
# Delete the image
if "storage.googleapis.com" in image:
print("deleting container %s" % image)
container_name = os.path.basename(image)
client.delete(container_name, force=True)
# Finally, delete the container
container.delete()
def globus_transfer(request, cid=None, endpoints=None):
""" a main portal for working with globus. If the user has navigated
here with a container id, it is presented with option to do a
transfer. If the method is a POST, we also do a custom search
for a set of containers.
"""
container = None
if cid is not None:
container = get_container(cid)
context = {
"user": request.user,
"container": container,
"endpoint_search_term": "Search for...",
}
# Does the user want to search endpoints?
if request.method == "POST":
term = request.POST.get("term")
if term is not None:
endpoints = search_endpoints(term=term, user=request.user)
context["endpoint_search_term"] = term.capitalize()
# If we don't have any endpoints still
if endpoints is None:
endpoints = get_endpoints(request.user)
context["endpoints"] = endpoints
return render(request, "globus/transfer.html", context)
def calculate_version(cid):
'''calculate version is run as a separate task after a container upload.
Instead of using md5 provided by nginx we calculate sha256 sum and
then include as the version variable.
'''
from shub.apps.main.views import get_container
from sregistry.utils import get_file_hash
print("Calculating version for upload.")
container = get_container(cid)
version = "sha256.%s" % get_file_hash(container.image.datafile.path,
"sha256")
container.version = version
container.save()
def delete_container(request, cid):
'''delete a container, including it's corresponding files
that are stored in Google Build (if they exist)
'''
container = get_container(cid)
if not container.has_edit_permission(request):
messages.info(request, "This action is not permitted.")
return redirect('collections')
# Send a job to the worker to delete the build files
django_rq.enqueue(delete_build, cid=container.id)
messages.info(request, 'Container successfully deleted.')
return redirect(container.collection.get_absolute_url())
def generate_share(request, cid):
"""generate a temporary share link for a container
Parameters
==========
cid: the container to generate a share link for
"""
container = get_container(cid)
edit_permission = container.has_edit_permission(request)
if edit_permission:
days = request.POST.get("days", None)
if days is not None:
days = int(days)
try:
expire_date = calculate_expiration_date(days)
share, _ = Share.objects.get_or_create(container=container,
expire_date=expire_date)
share.save()
# Generate an expiration task
django_rq.enqueue(expire_share, sid=share.id, eta=expire_date)
link = reverse(
"download_share",
kwargs={
"cid": container.id,
"secret": share.secret
},
)
expire_date = datetime.strftime(expire_date, "%b %m, %Y")
response = {
"status": "success",
"days": days,
"expire": expire_date,
"link": link,
}
except:
response = {"status": "error", "days": days}
return JsonResponse(response)
return JsonResponse(
{"error": "You are not allowed to perform this action."})
def submit_transfer(request, endpoint, cid):
"""submit a transfer request for a container id to an endpoint, also
based on id
"""
container = get_container(cid)
if container is None:
m = "This container could not be found."
else:
result = do_transfer(user=request.user,
endpoint=endpoint,
container=container)
link = "https://globus.org/app/activity/%s" % result["task_id"]
m = result["message"]
m = "%s: <a target='_blank' href='%s'>view task</a>" % (m, link)
status = {"message": m}
return JsonResponse(status)
def globus_endpoint(request, endpoint_id=None, cid=None):
''' Show information for a single endpoint only.
'''
container = None
if cid is not None:
container = get_container(cid)
context = {'user': request.user,
'container': container,
'endpoint_search_term': "Search for..." }
# Get the endpoint
try:
client = get_transfer_client(request.user)
endpoints = [client.get_endpoint(endpoint_id).data]
except TransferAPIError:
endpoints = get_endpoints(request.user)
context['endpoints'] = endpoints
return render(request, 'globus/transfer.html', context)
def globus_endpoint(request, endpoint_id=None, cid=None):
"""Show information for a single endpoint only."""
container = None
if cid is not None:
container = get_container(cid)
context = {
"user": request.user,
"container": container,
"endpoint_search_term": "Search for...",
}
# Get the endpoint
try:
client = get_transfer_client(request.user)
endpoints = [client.get_endpoint(endpoint_id).data]
except TransferAPIError:
endpoints = get_endpoints(request.user)
context["endpoints"] = endpoints
return render(request, "globus/transfer.html", context)
def complete_build(cid, params, check_again_seconds=10):
'''finish a build, meaning obtaining the original build_id for the container
and checking for completion.
Parameters
==========
cid: the container id to finish the build for, expected to have an id
params: the parameters from the build. They must have matching build it.
check_again_seconds: if the build is still working, check again in this
many seconds. By default, we multiply by 2 each time
(exponential backoff).
'''
from shub.apps.main.views import get_container
print("RUNNING COMPLETE BUILD")
container = get_container(cid)
# Case 1: No id provided
if "id" not in params:
return JsonResponseMessage(message="Invalid request.")
# Case 2: the container is already finished or not a google build
if "build_metadata" not in container.metadata or "builder" not in container.metadata:
return JsonResponseMessage(message="Invalid request.")
# Case 3: It's not a Google Build
if container.metadata['builder'].get('name') != "google_build":
return JsonResponseMessage(message="Invalid request.")
# Google build will have an id here
build_id = container.metadata['build_metadata']['build']['id']
status = container.metadata['build_metadata']['build']['status']
# Case 4: Build is already finished
active = ["QUEUED", "WORKING"]
if status not in active:
return JsonResponseMessage(message="Invalid request.")
# Case 5: Build id doesn't match
if build_id != params['id']:
return JsonResponseMessage(message="Invalid request.")
context = get_build_context()
# Instantiate client with context (connects to buckets)
client = get_client(debug=True, **context)
# Get an updated status
response = client._finish_build(build_id)
print("RESPONSE")
print(response)
if "public_url" in response:
container.metadata['image'] = response['public_url']
elif "media_link" in response:
container.metadata['image'] = response['media_link']
elif "status" in response:
# If it's still working, schedule to check with exponential backoff
if response["status"] in ["QUEUED", "WORKING"]:
check_again_seconds = check_again_seconds*2
print("Build status WORKING: checking in %s seconds" % check_again_seconds)
# Get the scheduler, submit to check again
scheduler = django_rq.get_scheduler('default')
scheduler.enqueue_in(timedelta(seconds=check_again_seconds),
complete_build,
cid=container.id,
params=params,
check_again_seconds=check_again_seconds)
# This is an invalid status, and no action to take
else:
print("Invalid response, no container link and status not working.")
return
# Save the build finish
container.metadata['build_finish'] = response
# Clear the container metadata
container = clear_container_payload(container)
# Add response metrics (size and file_hash)
if "size" in response:
container.metrics["size_mb"] = round(convert_size(response['size'], "MB"), 3)
# Update the status
if "status" in response:
container.metadata['build_metadata']['build']['status'] = response["status"]
# If a file hash is included, we use this as the version (not commit)
if "crc32" in response:
container.metrics["crc32"] = response["crc32"]
# Add the version, also calculated by builder
if "sha256sum" in response:
container.metrics["sha256"] = "sha256.%s" % response['sha256sum']
container.version = "sha256.%s" % response['sha256sum']
# Keep an md5, for posterity
if "md5sum" in response:
container.metrics["md5"] = "md5.%s" % response['md5sum']
# Calculate total time
if "startTime" in response and "finishTime" in response:
total_time = parse(response['finishTime']) - parse(response['startTime'])
container.metrics['build_seconds'] = total_time.total_seconds()
# Created date
if "createTime" in response:
created_at = datetime.strftime(parse(response['createTime']), '%h %d, %Y')
container.metrics['created_at'] = created_at
container.save()
