def get_bucket_name(self):
'''get or return the s3 bucket name. If not yet defined via an environment
variable or setting, we create a name with the pattern.
sregistry-<robotnamer>-<1234>
You can use the following environment variables to determine
interaction with the bucket:
SREGISTRY_S3_BUCKET: the bucket name (all lowercase, no underscore)
'''
# Get bucket name
bucket_name = 'sregistry-%s' % RobotNamer().generate()
self.bucket_name = self._get_and_update_setting(
'SREGISTRY_S3_BUCKET', bucket_name)
def create_container_payload(container):
'''a helper function to create a consistent container payload.
Parameters
==========
container: a container object to create a payload for
'''
if "builder" not in container.metadata:
container.metadata['builder'] = {}
if "robot_name" not in container.metadata["builder"]:
container.metadata['builder']['robot_name'] = RobotNamer().generate()
# Always create a new secret
container.metadata['builder']['secret'] = str(uuid.uuid4())
container.save()
return get_container_payload(container)
def setup_build(self,
name,
repo,
config,
tag=None,
commit=None,
recipe="Singularity",
startup_script=None):
'''setup the build based on the selected configuration file, meaning
producing the configuration file filled in based on the user's input
Parameters
==========
config: the complete configuration file provided by the client
template: an optional custom start script to use
tag: a user specified tag for the build, derived from uri or manual
recipe: a recipe, if defined, overrides recipe set in config.
commit: a commit to check out, if needed
start_script: the start script to use, if not defined
defaults to apt (or manager) base in main/templates/build
'''
manager = self._get_and_update_setting('SREGISTRY_BUILDER_MANAGER', 'apt')
startup_script = get_build_template(startup_script, manager)
# Read in the config to know what we can edit
config = self._load_build_config(config)
if not config:
bot.error('Cannot find config, check path or URI.')
sys.exit(1)
# Ensure that the builder config is intended for the client
self._client_tagged(config['data']['tags'])
# Compute settings that are parsed into runscript via metadata
defaults = config['data']['metadata']
selfLink = config['links']['self']
# Make sure the builder repository and folder is passed forward
builder_repo = config['data']['repo']
builder_bundle = config['data']['path']
builder_id = config['data']['id']
config = config['data']['config']
# Config settings from the environment, fall back to defaults
image_project = defaults.get('GOOGLE_COMPUTE_PROJECT', 'debian-cloud')
image_family = defaults.get('GOOGLE_COMPUTE_IMAGE_FAMILY', 'debian-8')
# Generate names, description is for repo, name is random
instance_name = "%s-builder %s" % (name.replace('/', '-'), selfLink)
robot_name = RobotNamer().generate()
project = self._get_project()
zone = self._get_zone()
# Machine Type
machine_type = defaults.get('SREGISTRY_BUILDER_machine_type',
'n1-standard-1')
machine_type = "zones/%s/machineTypes/%s" % (zone, machine_type)
# Disk Size
disk_size = defaults.get('SREGISTRY_BUILDER_disk_size', '100')
# Get the image type
image_response = self._compute_service.images().getFromFamily(
project=image_project, family=image_family).execute()
source_disk_image = image_response['selfLink']
storage_bucket = self._bucket_name
# Add the machine parameters to the config
config['name'] = robot_name
config['description'] = instance_name
config['machineType'] = machine_type
config['disks'].append({
"autoDelete": True,
"boot": True,
"initializeParams": {
'sourceImage': source_disk_image,
'diskSizeGb': disk_size
}
})
# Metadata base
metadata = {
'items': [
{
'key': 'startup-script',
'value': startup_script
},
# Storage Settings from Host
{
'key': 'SREGISTRY_BUILDER_STORAGE_BUCKET',
'value': self._bucket_name
}
]
}
# Runtime variables take priority over defaults from config
# and so here we update the defaults with runtime
# ([defaults], [config-key], [runtime-setting])
# User Repository
defaults = setconfig(defaults, 'SREGISTRY_USER_REPO', repo)
# Container Namespace (without tag/version)
defaults = setconfig(defaults, 'SREGISTRY_CONTAINER_NAME', name)
# User Repository Commit
defaults = setconfig(defaults, 'SREGISTRY_USER_COMMIT', commit)
# User Repository Branch
defaults = setconfig(defaults, 'SREGISTRY_USER_BRANCH', "master")
# User Repository Tag
defaults = setconfig(defaults, 'SREGISTRY_USER_TAG', tag)
# Builder repository url
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_REPO', builder_repo)
# Builder commit
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_COMMIT')
# Builder default runscript
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_RUNSCRIPT', "run.sh")
# Builder repository url
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_BRANCH', "master")
# Builder id in repository
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_ID', builder_id)
# Builder repository relative folder path
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_BUNDLE', builder_bundle)
# Number of extra hours to debug
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_DEBUGHOURS', "4")
# Hours to kill running job
defaults = setconfig(defaults, 'SREGISTRY_BUILDER_KILLHOURS', "10")
# Recipe set at runtime
defaults = setconfig(defaults, 'SINGULARITY_RECIPE', recipe)
# Branch of Singularity to install
defaults = setconfig(defaults, 'SINGULARITY_BRANCH')
# Singularity commit to use (if needed)
defaults = setconfig(defaults, 'SINGULARITY_COMMIT')
# Singularity Repo to Use
defaults = setconfig(defaults, 'SINGULARITY_REPO',
'https://github.com/cclerget/singularity.git')
# Update metadata config object
seen = ['SREGISTRY_BUILDER_STORAGE_BUCKET', 'startup-script']
for key, value in defaults.items():
# This also appends empty values, they are meaningful
if value not in seen:
entry = {"key": key, 'value': value}
metadata['items'].append(entry)
seen.append(key)
config['metadata'] = metadata
return config
def setup_build(
self,
name,
repo,
config,
tag=None,
commit=None,
recipe="Singularity",
startup_script=None,
):
"""setup the build based on the selected configuration file, meaning
producing the configuration file filled in based on the user's input
Parameters
==========
config: the complete configuration file provided by the client
template: an optional custom start script to use
tag: a user specified tag for the build, derived from uri or manual
recipe: a recipe, if defined, overrides recipe set in config.
commit: a commit to check out, if needed
start_script: the start script to use, if not defined
defaults to apt (or manager) base in main/templates/build
"""
manager = self._get_and_update_setting("SREGISTRY_BUILDER_MANAGER", "apt")
startup_script = get_build_template(startup_script, manager)
# Read in the config to know what we can edit
config = self._load_build_config(config)
if not config:
bot.exit("Cannot find config, check path or URI.")
# Ensure that the builder config is intended for the client
self._client_tagged(config["data"]["tags"])
# Compute settings that are parsed into runscript via metadata
defaults = config["data"]["metadata"]
selfLink = config["links"]["self"]
# Make sure the builder repository and folder is passed forward
builder_repo = config["data"]["repo"]
builder_bundle = config["data"]["path"]
builder_id = config["data"]["id"]
config = config["data"]["config"]
# Config settings from the environment, fall back to defaults
image_project = defaults.get("GOOGLE_COMPUTE_PROJECT", "debian-cloud")
image_family = defaults.get("GOOGLE_COMPUTE_IMAGE_FAMILY", "debian-8")
# Generate names, description is for repo, name is random
instance_name = "%s-builder %s" % (name.replace("/", "-"), selfLink)
robot_name = RobotNamer().generate()
zone = self._get_zone()
# Machine Type
machine_type = defaults.get("SREGISTRY_BUILDER_machine_type", "n1-standard-1")
machine_type = "zones/%s/machineTypes/%s" % (zone, machine_type)
# Disk Size
disk_size = defaults.get("SREGISTRY_BUILDER_disk_size", "100")
# Get the image type
image_response = (
self._compute_service.images()
.getFromFamily(project=image_project, family=image_family)
.execute()
)
source_disk_image = image_response["selfLink"]
# Add the machine parameters to the config
config["name"] = robot_name
config["description"] = instance_name
config["machineType"] = machine_type
config["disks"].append(
{
"autoDelete": True,
"boot": True,
"initializeParams": {
"sourceImage": source_disk_image,
"diskSizeGb": disk_size,
},
}
)
# Metadata base
metadata = {
"items": [
{"key": "startup-script", "value": startup_script},
# Storage Settings from Host
{"key": "SREGISTRY_BUILDER_STORAGE_BUCKET", "value": self._bucket_name},
]
}
# Runtime variables take priority over defaults from config
# and so here we update the defaults with runtime
# ([defaults], [config-key], [runtime-setting])
# User Repository
defaults = setconfig(defaults, "SREGISTRY_USER_REPO", repo)
# Container Namespace (without tag/version)
defaults = setconfig(defaults, "SREGISTRY_CONTAINER_NAME", name)
# User Repository Commit
defaults = setconfig(defaults, "SREGISTRY_USER_COMMIT", commit)
# User Repository Branch
defaults = setconfig(defaults, "SREGISTRY_USER_BRANCH", "master")
# User Repository Tag
defaults = setconfig(defaults, "SREGISTRY_USER_TAG", tag)
# Builder repository url
defaults = setconfig(defaults, "SREGISTRY_BUILDER_REPO", builder_repo)
# Builder commit
defaults = setconfig(defaults, "SREGISTRY_BUILDER_COMMIT")
# Builder default runscript
defaults = setconfig(defaults, "SREGISTRY_BUILDER_RUNSCRIPT", "run.sh")
# Builder repository url
defaults = setconfig(defaults, "SREGISTRY_BUILDER_BRANCH", "master")
# Builder id in repository
defaults = setconfig(defaults, "SREGISTRY_BUILDER_ID", builder_id)
# Builder repository relative folder path
defaults = setconfig(defaults, "SREGISTRY_BUILDER_BUNDLE", builder_bundle)
# Number of extra hours to debug
defaults = setconfig(defaults, "SREGISTRY_BUILDER_DEBUGHOURS", "4")
# Hours to kill running job
defaults = setconfig(defaults, "SREGISTRY_BUILDER_KILLHOURS", "10")
# Recipe set at runtime
defaults = setconfig(defaults, "SINGULARITY_RECIPE", recipe)
# Branch of Singularity to install
defaults = setconfig(defaults, "SINGULARITY_BRANCH")
# Singularity commit to use (if needed)
defaults = setconfig(defaults, "SINGULARITY_COMMIT")
# Singularity Repo to Use
defaults = setconfig(
defaults, "SINGULARITY_REPO", "https://github.com/cclerget/singularity.git"
)
# Update metadata config object
seen = ["SREGISTRY_BUILDER_STORAGE_BUCKET", "startup-script"]
for key, value in defaults.items():
# This also appends empty values, they are meaningful
if value not in seen:
entry = {"key": key, "value": value}
metadata["items"].append(entry)
seen.append(key)
config["metadata"] = metadata
return config