def _getCloudConfigUserData(self,
role,
masterPublicKey=None,
keyPath=None,
preemptable=False):
if role == 'leader':
entryPoint = 'mesos-master'
mesosArgs = self.MESOS_LOG_DIR + self.LEADER_DOCKER_ARGS.format(
name=self.clusterName)
elif role == 'worker':
entryPoint = 'mesos-slave'
mesosArgs = self.MESOS_LOG_DIR + self.WORKER_DOCKER_ARGS.format(
ip=self._leaderPrivateIP, preemptable=preemptable)
else:
raise RuntimeError("Unknown role %s" % role)
template = self.cloudConfigTemplate
if masterPublicKey:
template += self.sshTemplate
if keyPath:
mesosArgs = keyPath + ' ' + mesosArgs
entryPoint = "waitForKey.sh"
customDockerInitCommand = customDockerInitCmd()
if customDockerInitCommand:
mesosArgs = " ".join(
["'" + customDockerInitCommand + "'", entryPoint, mesosArgs])
entryPoint = "customDockerInit.sh"
templateArgs = dict(
role=role,
dockerImage=applianceSelf(),
entrypoint=entryPoint,
sshKey=masterPublicKey, # ignored if None
mesosArgs=mesosArgs)
userData = template.format(**templateArgs)
return userData
def addToilService(self,
config: InstanceConfiguration,
role: str,
keyPath: str = None,
preemptable: bool = False):
"""
Add the Toil leader or worker service to an instance configuration.
Will run Mesos master or agent as appropriate in Mesos clusters.
For Kubernetes clusters, will just sleep to provide a place to shell
into on the leader, and shouldn't run on the worker.
:param role: Should be 'leader' or 'worker'. Will not work for 'worker' until leader credentials have been collected.
:param keyPath: path on the node to a server-side encryption key that will be added to the node after it starts. The service will wait until the key is present before starting.
:param preemptable: Whether a woeker should identify itself as preemptable or not to the scheduler.
"""
# If keys are rsynced, then the mesos-agent needs to be started after the keys have been
# transferred. The waitForKey.sh script loops on the new VM until it finds the keyPath file, then it starts the
# mesos-agent. If there are multiple keys to be transferred, then the last one to be transferred must be
# set to keyPath.
MESOS_LOG_DIR = '--log_dir=/var/lib/mesos '
LEADER_DOCKER_ARGS = '--registry=in_memory --cluster={name}'
# --no-systemd_enable_support is necessary in Ubuntu 16.04 (otherwise,
# Mesos attempts to contact systemd but can't find its run file)
WORKER_DOCKER_ARGS = '--work_dir=/var/lib/mesos --master={ip}:5050 --attributes=preemptable:{preemptable} --no-hostname_lookup --no-systemd_enable_support'
if self.clusterType == 'mesos':
if role == 'leader':
entryPoint = 'mesos-master'
entryPointArgs = MESOS_LOG_DIR + LEADER_DOCKER_ARGS.format(
name=self.clusterName)
elif role == 'worker':
entryPoint = 'mesos-agent'
entryPointArgs = MESOS_LOG_DIR + WORKER_DOCKER_ARGS.format(
ip=self._leaderPrivateIP, preemptable=preemptable)
else:
raise RuntimeError("Unknown role %s" % role)
elif self.clusterType == 'kubernetes':
if role == 'leader':
# We need *an* entry point or the leader container will finish
# and go away, and thus not be available to take user logins.
entryPoint = 'sleep'
entryPointArgs = 'infinity'
else:
raise RuntimeError(
'Toil service not needed for %s nodes in a %s cluster',
role, self.clusterType)
else:
raise RuntimeError('Toil service not needed in a %s cluster',
self.clusterType)
if keyPath:
entryPointArgs = keyPath + ' ' + entryPointArgs
entryPoint = "waitForKey.sh"
customDockerInitCommand = customDockerInitCmd()
if customDockerInitCommand:
entryPointArgs = " ".join([
"'" + customDockerInitCommand + "'", entryPoint, entryPointArgs
])
entryPoint = "customDockerInit.sh"
config.addUnit(f"toil-{role}.service",
content=textwrap.dedent(f'''\
[Unit]
Description=toil-{role} container
After=docker.service
After=create-kubernetes-cluster.service
[Service]
Restart=on-failure
RestartSec=2
ExecStartPre=-/usr/bin/docker rm toil_{role}
ExecStartPre=-/usr/bin/bash -c '{customInitCmd()}'
ExecStart=/usr/bin/docker run \\
--entrypoint={entryPoint} \\
--net=host \\
-v /var/run/docker.sock:/var/run/docker.sock \\
-v /var/lib/mesos:/var/lib/mesos \\
-v /var/lib/docker:/var/lib/docker \\
-v /var/lib/toil:/var/lib/toil \\
-v /var/lib/cwl:/var/lib/cwl \\
-v /tmp:/tmp \\
-v /opt:/opt \\
-v /etc/kubernetes:/etc/kubernetes \\
-v /etc/kubernetes/admin.conf:/root/.kube/config \\
--name=toil_{role} \\
{applianceSelf()} \\
{entryPointArgs}
'''))