def receive_build(request, cid): '''receive_build will receive the post from Google Cloud Build. we check the response header against the jwt token to authenticate, and then check other metadata and permissions in complete_build. ''' print(request.body) print(cid) if request.method == "POST": # Must be an existing container container = get_container(cid) if container is None: return JsonResponseMessage(message="Invalid request.") # Decode parameters params = json.loads(request.body.decode('utf-8')) # Must include a jwt token that is valid for the container if not validate_jwt(container, params): return JsonResponseMessage(message="Invalid request.") scheduler = django_rq.get_scheduler('default') scheduler.enqueue_in(timedelta(seconds=10), complete_build, cid=container.id, params=params) return JsonResponseMessage(message="Notification Received", status=200, status_message="Received") return JsonResponseMessage(message="Invalid request.")
def delete_build(cid, client=None): '''Delete artifacts for a container build, if they exist, along with the container object. This is called as a django-rq task for a worker to do from views.py Parameters ========== cid: the container id to finish the build for, expected to have an id ''' from shub.apps.main.views import get_container container = get_container(cid) # if being called from delete_container_collection, just instantiate once if client is None: context = get_build_context() client = get_client(debug=True, **context) # If the container has an image, delete it image = container.get_image() or "" if container.metadata['builder']['name'] == "google_build": # Delete the image if "storage.googleapis.com" in image: print("deleting container %s" % image) container_name = os.path.basename(image) client.delete(container_name, force=True) # Finally, delete the container container.delete()
def globus_transfer(request, cid=None, endpoints=None): """ a main portal for working with globus. If the user has navigated here with a container id, it is presented with option to do a transfer. If the method is a POST, we also do a custom search for a set of containers. """ container = None if cid is not None: container = get_container(cid) context = { "user": request.user, "container": container, "endpoint_search_term": "Search for...", } # Does the user want to search endpoints? if request.method == "POST": term = request.POST.get("term") if term is not None: endpoints = search_endpoints(term=term, user=request.user) context["endpoint_search_term"] = term.capitalize() # If we don't have any endpoints still if endpoints is None: endpoints = get_endpoints(request.user) context["endpoints"] = endpoints return render(request, "globus/transfer.html", context)
def calculate_version(cid): '''calculate version is run as a separate task after a container upload. Instead of using md5 provided by nginx we calculate sha256 sum and then include as the version variable. ''' from shub.apps.main.views import get_container from sregistry.utils import get_file_hash print("Calculating version for upload.") container = get_container(cid) version = "sha256.%s" % get_file_hash(container.image.datafile.path, "sha256") container.version = version container.save()
def delete_container(request, cid): '''delete a container, including it's corresponding files that are stored in Google Build (if they exist) ''' container = get_container(cid) if not container.has_edit_permission(request): messages.info(request, "This action is not permitted.") return redirect('collections') # Send a job to the worker to delete the build files django_rq.enqueue(delete_build, cid=container.id) messages.info(request, 'Container successfully deleted.') return redirect(container.collection.get_absolute_url())
def generate_share(request, cid): """generate a temporary share link for a container Parameters ========== cid: the container to generate a share link for """ container = get_container(cid) edit_permission = container.has_edit_permission(request) if edit_permission: days = request.POST.get("days", None) if days is not None: days = int(days) try: expire_date = calculate_expiration_date(days) share, _ = Share.objects.get_or_create(container=container, expire_date=expire_date) share.save() # Generate an expiration task django_rq.enqueue(expire_share, sid=share.id, eta=expire_date) link = reverse( "download_share", kwargs={ "cid": container.id, "secret": share.secret }, ) expire_date = datetime.strftime(expire_date, "%b %m, %Y") response = { "status": "success", "days": days, "expire": expire_date, "link": link, } except: response = {"status": "error", "days": days} return JsonResponse(response) return JsonResponse( {"error": "You are not allowed to perform this action."})
def submit_transfer(request, endpoint, cid): """submit a transfer request for a container id to an endpoint, also based on id """ container = get_container(cid) if container is None: m = "This container could not be found." else: result = do_transfer(user=request.user, endpoint=endpoint, container=container) link = "https://globus.org/app/activity/%s" % result["task_id"] m = result["message"] m = "%s: <a target='_blank' href='%s'>view task</a>" % (m, link) status = {"message": m} return JsonResponse(status)
def globus_endpoint(request, endpoint_id=None, cid=None): ''' Show information for a single endpoint only. ''' container = None if cid is not None: container = get_container(cid) context = {'user': request.user, 'container': container, 'endpoint_search_term': "Search for..." } # Get the endpoint try: client = get_transfer_client(request.user) endpoints = [client.get_endpoint(endpoint_id).data] except TransferAPIError: endpoints = get_endpoints(request.user) context['endpoints'] = endpoints return render(request, 'globus/transfer.html', context)
def globus_endpoint(request, endpoint_id=None, cid=None): """Show information for a single endpoint only.""" container = None if cid is not None: container = get_container(cid) context = { "user": request.user, "container": container, "endpoint_search_term": "Search for...", } # Get the endpoint try: client = get_transfer_client(request.user) endpoints = [client.get_endpoint(endpoint_id).data] except TransferAPIError: endpoints = get_endpoints(request.user) context["endpoints"] = endpoints return render(request, "globus/transfer.html", context)
def complete_build(cid, params, check_again_seconds=10): '''finish a build, meaning obtaining the original build_id for the container and checking for completion. Parameters ========== cid: the container id to finish the build for, expected to have an id params: the parameters from the build. They must have matching build it. check_again_seconds: if the build is still working, check again in this many seconds. By default, we multiply by 2 each time (exponential backoff). ''' from shub.apps.main.views import get_container print("RUNNING COMPLETE BUILD") container = get_container(cid) # Case 1: No id provided if "id" not in params: return JsonResponseMessage(message="Invalid request.") # Case 2: the container is already finished or not a google build if "build_metadata" not in container.metadata or "builder" not in container.metadata: return JsonResponseMessage(message="Invalid request.") # Case 3: It's not a Google Build if container.metadata['builder'].get('name') != "google_build": return JsonResponseMessage(message="Invalid request.") # Google build will have an id here build_id = container.metadata['build_metadata']['build']['id'] status = container.metadata['build_metadata']['build']['status'] # Case 4: Build is already finished active = ["QUEUED", "WORKING"] if status not in active: return JsonResponseMessage(message="Invalid request.") # Case 5: Build id doesn't match if build_id != params['id']: return JsonResponseMessage(message="Invalid request.") context = get_build_context() # Instantiate client with context (connects to buckets) client = get_client(debug=True, **context) # Get an updated status response = client._finish_build(build_id) print("RESPONSE") print(response) if "public_url" in response: container.metadata['image'] = response['public_url'] elif "media_link" in response: container.metadata['image'] = response['media_link'] elif "status" in response: # If it's still working, schedule to check with exponential backoff if response["status"] in ["QUEUED", "WORKING"]: check_again_seconds = check_again_seconds*2 print("Build status WORKING: checking in %s seconds" % check_again_seconds) # Get the scheduler, submit to check again scheduler = django_rq.get_scheduler('default') scheduler.enqueue_in(timedelta(seconds=check_again_seconds), complete_build, cid=container.id, params=params, check_again_seconds=check_again_seconds) # This is an invalid status, and no action to take else: print("Invalid response, no container link and status not working.") return # Save the build finish container.metadata['build_finish'] = response # Clear the container metadata container = clear_container_payload(container) # Add response metrics (size and file_hash) if "size" in response: container.metrics["size_mb"] = round(convert_size(response['size'], "MB"), 3) # Update the status if "status" in response: container.metadata['build_metadata']['build']['status'] = response["status"] # If a file hash is included, we use this as the version (not commit) if "crc32" in response: container.metrics["crc32"] = response["crc32"] # Add the version, also calculated by builder if "sha256sum" in response: container.metrics["sha256"] = "sha256.%s" % response['sha256sum'] container.version = "sha256.%s" % response['sha256sum'] # Keep an md5, for posterity if "md5sum" in response: container.metrics["md5"] = "md5.%s" % response['md5sum'] # Calculate total time if "startTime" in response and "finishTime" in response: total_time = parse(response['finishTime']) - parse(response['startTime']) container.metrics['build_seconds'] = total_time.total_seconds() # Created date if "createTime" in response: created_at = datetime.strftime(parse(response['createTime']), '%h %d, %Y') container.metrics['created_at'] = created_at container.save()