def delete_container_collection(cid, uid):
'''Delete artifacts for a container build, if they exist, and then
the entire collection. This is called
as a django-rq task for a worker to do from views.py
Parameters
==========
cid: the collection id to delete.
uid: the user id requesting permission
'''
from shub.apps.main.views import get_collection
from .github import delete_webhook
collection = get_collection(cid)
# Delete files before containers
containers = Container.objects.filter(collection=collection)
# Create a client to share
context = get_build_context()
client = get_client(debug=True, **context)
# Delete container build objects first
for container in containers:
delete_build(cid=container.id, client=client)
# Now handle the webhook (a separate task)
if "github" in collection.metadata:
django_rq.enqueue(delete_webhook,
user=uid,
repo=collection.metadata['github']['repo_name'],
hook_id=collection.metadata['github']['webhook']['id'])
# Finally, delete the collection
print("%s deleting." % collection)
collection.delete()
def delete_build(cid, client=None):
'''Delete artifacts for a container build, if they exist, along
with the container object. This is called
as a django-rq task for a worker to do from views.py
Parameters
==========
cid: the container id to finish the build for, expected to have an id
'''
from shub.apps.main.views import get_container
container = get_container(cid)
# if being called from delete_container_collection, just instantiate once
if client is None:
context = get_build_context()
client = get_client(debug=True, **context)
# If the container has an image, delete it
image = container.get_image() or ""
if container.metadata['builder']['name'] == "google_build":
# Delete the image
if "storage.googleapis.com" in image:
print("deleting container %s" % image)
container_name = os.path.basename(image)
client.delete(container_name, force=True)
# Finally, delete the container
container.delete()
def is_over_limit(limit=None):
'''check if we are over the limit for active builds. Returns a boolean to
indicate yes or no, based on filtering the number of total builds
by those with status "QUEUED" or "WORKING."
Parameters
==========
limit: an integer limit for the maximum concurrent waiting or active
builds. If not set, we use the default in settings.
'''
# Allow the function to set a custom limit
limit = limit or settings.SREGISTRY_GOOGLE_BUILD_LIMIT
# Instantiate client with context (connects to buckets)
context = get_build_context()
client = get_client(debug=True, **context)
project = settings.SREGISTRY_GOOGLE_PROJECT
result = client._build_service.projects().builds().list(projectId=project,
filter='status="QUEUED" OR status="WORKING"').execute()
return len(result) > limit
def trigger_build(sender, instance, **kwargs):
'''Trigger build will send a recipe directly to Google Cloud Build,
and create a container that will send a curl response back to
an endpoint here to signal that the build is complete.
Triggered by RecipePushSerializer.
Parameters
==========
sender: should be the sending model, which is an RecipeFile instance
instance: is the instance of the RecipeFile
'''
collection = Collection.objects.get(name=instance.collection)
context = get_build_context()
print("IN TRIGGER BUILD")
# Instantiate client with context (connects to buckets)
client = get_client(debug=True, **context)
# Assemble the name
name = "%s/%s:%s" %(instance.collection, instance.name, instance.tag)
# The recipe needs to be in PWD to create the build package
recipe = instance.datafile.name
working_dir = os.path.dirname(recipe)
# Create a container (with status google-build) for the user to watch
try:
container = collection.containers.get(tag=instance.tag,
name=instance.name)
except ObjectDoesNotExist:
container = Container.objects.create(collection=collection,
tag=instance.tag,
name=instance.name)
# If it's frozen, don't submit
if container.frozen:
return JsonResponseMessage(message="Container is frozen.")
# Webhook response
webhook = "%s%s" % (settings.DOMAIN_NAME,
reverse('receive_build', kwargs={"cid": container.id}))
# Generate a one time use secret for jwt web token
container.metadata['builder'] = {"name": "google_build"}
payload = create_container_payload(container) # does not save
# Generate the jwt token
jwt_token = generate_jwt_token(secret=container.metadata['builder']['secret'],
payload=payload)
# Submit the build
response = client.build(name,
recipe=recipe,
working_dir=working_dir,
headless=True,
webhook=webhook,
extra_data={"token": jwt_token})
# Update the status for the container
if "status" in response:
container.metadata['build_metadata']['build']['status'] = response["status"]
# Add the metadata
container.metadata['build_metadata'] = response['metadata']
container.save()
print(response)
return JsonResponseMessage(message="Build received.")
def complete_build(cid, params, check_again_seconds=10):
'''finish a build, meaning obtaining the original build_id for the container
and checking for completion.
Parameters
==========
cid: the container id to finish the build for, expected to have an id
params: the parameters from the build. They must have matching build it.
check_again_seconds: if the build is still working, check again in this
many seconds. By default, we multiply by 2 each time
(exponential backoff).
'''
from shub.apps.main.views import get_container
print("RUNNING COMPLETE BUILD")
container = get_container(cid)
# Case 1: No id provided
if "id" not in params:
return JsonResponseMessage(message="Invalid request.")
# Case 2: the container is already finished or not a google build
if "build_metadata" not in container.metadata or "builder" not in container.metadata:
return JsonResponseMessage(message="Invalid request.")
# Case 3: It's not a Google Build
if container.metadata['builder'].get('name') != "google_build":
return JsonResponseMessage(message="Invalid request.")
# Google build will have an id here
build_id = container.metadata['build_metadata']['build']['id']
status = container.metadata['build_metadata']['build']['status']
# Case 4: Build is already finished
active = ["QUEUED", "WORKING"]
if status not in active:
return JsonResponseMessage(message="Invalid request.")
# Case 5: Build id doesn't match
if build_id != params['id']:
return JsonResponseMessage(message="Invalid request.")
context = get_build_context()
# Instantiate client with context (connects to buckets)
client = get_client(debug=True, **context)
# Get an updated status
response = client._finish_build(build_id)
print("RESPONSE")
print(response)
if "public_url" in response:
container.metadata['image'] = response['public_url']
elif "media_link" in response:
container.metadata['image'] = response['media_link']
elif "status" in response:
# If it's still working, schedule to check with exponential backoff
if response["status"] in ["QUEUED", "WORKING"]:
check_again_seconds = check_again_seconds*2
print("Build status WORKING: checking in %s seconds" % check_again_seconds)
# Get the scheduler, submit to check again
scheduler = django_rq.get_scheduler('default')
scheduler.enqueue_in(timedelta(seconds=check_again_seconds),
complete_build,
cid=container.id,
params=params,
check_again_seconds=check_again_seconds)
# This is an invalid status, and no action to take
else:
print("Invalid response, no container link and status not working.")
return
# Save the build finish
container.metadata['build_finish'] = response
# Clear the container metadata
container = clear_container_payload(container)
# Add response metrics (size and file_hash)
if "size" in response:
container.metrics["size_mb"] = round(convert_size(response['size'], "MB"), 3)
# Update the status
if "status" in response:
container.metadata['build_metadata']['build']['status'] = response["status"]
# If a file hash is included, we use this as the version (not commit)
if "crc32" in response:
container.metrics["crc32"] = response["crc32"]
# Add the version, also calculated by builder
if "sha256sum" in response:
container.metrics["sha256"] = "sha256.%s" % response['sha256sum']
container.version = "sha256.%s" % response['sha256sum']
# Keep an md5, for posterity
if "md5sum" in response:
container.metrics["md5"] = "md5.%s" % response['md5sum']
# Calculate total time
if "startTime" in response and "finishTime" in response:
total_time = parse(response['finishTime']) - parse(response['startTime'])
container.metrics['build_seconds'] = total_time.total_seconds()
# Created date
if "createTime" in response:
created_at = datetime.strftime(parse(response['createTime']), '%h %d, %Y')
container.metrics['created_at'] = created_at
container.save()
def receive_build(collection, recipes, branch):
'''receive_build will receive a build from GitHub, and then trigger
the same Google Cloud Build but using a GitHub repository (recommended).
Parameters
==========
collection: the collection
recipes: a dictionary of modified recipe files to build
branch: the repository branch (kept as metadata)
'''
from .github import get_auth_token
context = get_build_context()
# Instantiate client with context (connects to buckets)
client = get_client(debug=True, **context)
print("RECIPES: %s" % recipes)
# Derive tag from the recipe, or default to latest
for recipe, metadata in recipes.items():
# First preference to command line, then recipe tag
tag = get_recipe_tag(recipe) or "latest"
# Get a container, if it exists, we've already written file here
try:
container = collection.containers.get(tag=tag)
except: # DoesNotExist
container = Container.objects.create(collection=collection,
tag=tag,
name=collection.name)
# If the container is frozen, no go
if container.frozen:
bot.debug('%s is frozen, will not trigger build.' % container)
continue
# Recipe path on Github
reponame = container.collection.metadata['github']['repo_name']
# If we don't have a commit, just send to recipe
if metadata['commit'] is None:
deffile = "https://www.github.com/%s/tree/%s/%s" %(reponame,
branch,
recipe)
else:
deffile = "https://www.github.com/%s/blob/%s/%s" %(reponame,
metadata['commit'],
recipe)
# Webhook response
webhook = "%s%s" % (settings.DOMAIN_NAME,
reverse('receive_build', kwargs={"cid": container.id}))
# Generate a one time use secret for jwt web token
container.metadata['builder'] = {"name":"google_build",
"deffile": deffile}
payload = create_container_payload(container) # does not save
# Generate the jwt token
jwt_token = generate_jwt_token(secret=container.metadata['builder']['secret'],
payload=payload)
# If the repo is private, we need to account for that
token = None
if collection.metadata['github'].get('private', False) is True:
token = get_auth_token(collection.owners.first())
# Submit the build with the GitHub repo and commit
response = client.build_repo("github.com/%s" % metadata['name'],
recipe=recipe,
headless=True,
token=token,
commit=metadata['commit'],
webhook=webhook,
extra_data={"token": jwt_token})
# Add the metadata
container.metadata['build_metadata'] = response['metadata']
container.save()