def provision(self):
self.status = ClusterStatus.PROVISIONING
base_url = cumulus.config.girder.baseUrl
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
playbook = get_property('config.provision.spec',
self.cluster,
default=self.DEFAULT_PLAYBOOK)
playbook_params = get_property('config.provision.params',
self.cluster,
default={})
provision_ssh_user = get_property('config.provision.ssh.user',
self.cluster,
default='ubuntu')
playbook_params['cluster_state'] = ClusterStatus.RUNNING
playbook_params['ansible_ssh_user'] = provision_ssh_user
cumulus.ansible.tasks.cluster.provision_cluster \
.delay(playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key, playbook_params,
girder_token, log_write_url, ClusterStatus.RUNNING)
return self.cluster
def provision(self):
self.status = ClusterStatus.PROVISIONING
base_url = getApiUrl()
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
playbook = get_property(
'config.provision.spec', self.cluster,
default=self.DEFAULT_PLAYBOOK)
playbook_params = get_property(
'config.provision.params', self.cluster, default={})
provision_ssh_user = get_property(
'config.provision.ssh.user', self.cluster, default='ubuntu')
playbook_params['cluster_state'] = ClusterStatus.RUNNING
playbook_params['ansible_ssh_user'] = provision_ssh_user
cumulus.ansible.tasks.cluster.provision_cluster \
.delay(playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key, playbook_params,
girder_token, log_write_url, ClusterStatus.RUNNING)
return self.cluster
def create_profile(user, params):
body = getBodyJson()
requireParams([
'name', 'accessKeyId', 'secretAccessKey', 'regionName',
'availabilityZone'
], body)
profile_type = 'ec2' if 'cloudProvider' not in body.keys() \
else body['cloudProvider']
model = ModelImporter.model('aws', 'cumulus')
profile = model.create_profile(user['_id'], body['name'], profile_type,
body['accessKeyId'],
body['secretAccessKey'], body['regionName'],
body['availabilityZone'],
body.get('publicIPs', False))
# Now fire of a task to create a key pair for this profile
try:
cumulus.aws.ec2.tasks.key.generate_key_pair.delay(
_filter(profile),
get_task_token()['_id'])
cherrypy.response.status = 201
cherrypy.response.headers['Location'] \
= '/user/%s/aws/profile/%s' % (str(user['_id']),
str(profile['_id']))
return model.filter(profile, getCurrentUser())
except Exception:
# Remove profile if error occurs fire of task
model.remove(profile)
raise
def submit_job(self, job):
log_url = '%s/jobs/%s/log' % (getApiUrl(), job['_id'])
girder_token = get_task_token(self.cluster)['_id']
cumulus.tasks.job.submit(
girder_token,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
job, log_url)
def delete(self):
super(TraditionClusterAdapter, self).delete()
# Clean up key associate with cluster
cumulus.ssh.tasks.key.delete_key_pair.delay(
self._model.filter(self.cluster,
getCurrentUser(),
passphrase=False),
get_task_token()['_id'])
def start(self, request_body):
log_write_url = '%s/clusters/%s/log' % (getApiUrl(),
self.cluster['_id'])
girder_token = get_task_token(self.cluster)['_id']
cumulus.tasks.cluster.test_connection \
.delay(
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
log_write_url=log_write_url, girder_token=girder_token)
def start(self, request_body):
log_write_url = '%s/clusters/%s/log' % (cumulus.config.girder.baseUrl,
self.cluster['_id'])
girder_token = get_task_token(self.cluster)['_id']
cumulus.tasks.cluster.test_connection \
.delay(
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
log_write_url=log_write_url, girder_token=girder_token)
def submit_job(self, job):
log_url = '%s/jobs/%s/log' % (cumulus.config.girder.baseUrl,
job['_id'])
girder_token = get_task_token(self.cluster)['_id']
cumulus.tasks.job.submit(
girder_token,
self._model.filter(self.cluster,
getCurrentUser(),
passphrase=False), job, log_url)
def start(self, request_body):
if self.cluster['status'] == ClusterStatus.CREATING:
raise RestException('Cluster is not ready to start.', code=400)
log_write_url = '%s/clusters/%s/log' % (getApiUrl(),
self.cluster['_id'])
girder_token = get_task_token()['_id']
cumulus.tasks.cluster.test_connection \
.delay(self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
log_write_url=log_write_url,
girder_token=girder_token)
def start(self, request_body):
if self.cluster['status'] == ClusterStatus.CREATING:
raise RestException('Cluster is not ready to start.', code=400)
log_write_url = '%s/clusters/%s/log' % (cumulus.config.girder.baseUrl,
self.cluster['_id'])
girder_token = get_task_token()['_id']
cumulus.tasks.cluster.test_connection \
.delay(self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
log_write_url=log_write_url,
girder_token=girder_token)
def detach(self, volume, params):
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': get_task_token()['_id']
}
log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume['_id'])
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume is None or aws_volume['state'] != VolumeState.INUSE:
raise RestException('This volume is not attached '
'to a cluster', 400)
if 'clusterId' not in volume:
raise RestException('clusterId is not set on this volume!', 400)
try:
volume['path']
except KeyError:
raise RestException('path is not set on this volume!', 400)
cluster = ModelImporter.model('cluster',
'cumulus').load(volume['clusterId'],
user=getCurrentUser(),
level=AccessType.ADMIN)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!', 400)
user = getCurrentUser()
cluster = ModelImporter.model('cluster',
'cumulus').filter(cluster,
user,
passphrase=False)
cumulus.ansible.tasks.volume.detach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, user),
secret_key, log_write_url, girder_callback_info)
volume['status'] = VolumeState.DETACHING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
def start(self, request_body):
"""
Adapters may implement this if they support a start operation.
"""
self.status = ClusterStatus.LAUNCHING
self.cluster['config'].setdefault('provision', {})\
.setdefault('params', {}).update(request_body)
self.cluster = self.model('cluster', 'cumulus').save(self.cluster)
base_url = cumulus.config.girder.baseUrl
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
# Launch
launch_playbook = get_property('config.launch.spec',
self.cluster,
default=self.DEFAULT_PLAYBOOK)
launch_playbook_params = get_property('config.launch.params',
self.cluster,
default={})
launch_playbook_params['cluster_state'] = ClusterStatus.RUNNING
# Provision
provision_playbook = get_property('config.provision.spec',
self.cluster,
default='gridengine/site')
provision_playbook_params = get_property('config.provision.params',
self.cluster,
default={})
provision_ssh_user = get_property('config.provision.ssh.user',
self.cluster,
default='ubuntu')
provision_playbook_params['ansible_ssh_user'] = provision_ssh_user
provision_playbook_params['cluster_state'] = ClusterStatus.RUNNING
cumulus.ansible.tasks.cluster.start_cluster \
.delay(launch_playbook,
# provision playbook
provision_playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key,
launch_playbook_params, provision_playbook_params,
girder_token, log_write_url)
def attach(self, volume, cluster, params):
body = getBodyJson()
self.requireParams(['path'], body)
path = body['path']
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': get_task_token()['_id']
}
log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume['_id'])
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
# If volume exists it needs to be available to be attached. If
# it doesn't exist it will be created as part of the attach
# playbook.
if aws_volume is not None and \
aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException(
'This volume is not available to attach '
'to a cluster', 400)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!', 400)
cluster = ModelImporter.model('cluster',
'cumulus').filter(cluster,
getCurrentUser(),
passphrase=False)
cumulus.ansible.tasks.volume.attach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, getCurrentUser()), path,
secret_key, log_write_url, girder_callback_info)
volume['status'] = VolumeState.ATTACHING
volume = self._model.update_volume(getCurrentUser(), volume)
return self._model.filter(volume, getCurrentUser())
def delete(self, volume, params):
if 'clusterId' in volume:
raise RestException('Unable to delete attached volume')
# If the volume is in state created and it has no ec2 volume id
# associated with it, we should be able to just delete it
if volume['status'] in (VolumeState.CREATED, VolumeState.ERROR):
if 'id' in volume['ec2'] and volume['ec2']['id'] is not None:
raise RestException('Unable to delete volume, it is '
'associated with an ec2 volume %s' %
volume['ec2']['id'])
self._model.remove(volume)
return None
log_write_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume['_id'])
# Call EC2 to delete volume
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': get_task_token()['_id']
}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException(
'Volume must be in an "%s" status to be deleted' %
VolumeState.AVAILABLE, 400)
user = getCurrentUser()
cumulus.ansible.tasks.volume.delete_volume\
.delay(profile, self._model.filter(volume, user),
secret_key, log_write_url, girder_callback_info)
volume['status'] = VolumeState.DELETING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
def detach(self, volume, params):
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': getApiUrl(),
'girder_token': get_task_token()['_id']}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume is None or aws_volume['state'] != VolumeState.INUSE:
raise RestException('This volume is not attached '
'to a cluster',
400)
if 'clusterId' not in volume:
raise RestException('clusterId is not set on this volume!', 400)
try:
volume['path']
except KeyError:
raise RestException('path is not set on this volume!', 400)
cluster = self.model('cluster', 'cumulus').load(volume['clusterId'],
user=getCurrentUser(),
level=AccessType.ADMIN)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!',
400)
user = getCurrentUser()
cluster = self.model('cluster', 'cumulus').filter(
cluster, user, passphrase=False)
cumulus.ansible.tasks.volume.detach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, user),
secret_key, girder_callback_info)
volume['status'] = VolumeState.DETACHING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
def attach(self, volume, cluster, params):
body = getBodyJson()
self.requireParams(['path'], body)
path = body['path']
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': getApiUrl(),
'girder_token': get_task_token()['_id']}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
# If volume exists it needs to be available to be attached. If
# it doesn't exist it will be created as part of the attach
# playbook.
if aws_volume is not None and \
aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException('This volume is not available to attach '
'to a cluster',
400)
master = p.get_master_instance(cluster['_id'])
if master['state'] != InstanceState.RUNNING:
raise RestException('Master instance is not running!',
400)
cluster = self.model('cluster', 'cumulus').filter(
cluster, getCurrentUser(), passphrase=False)
cumulus.ansible.tasks.volume.attach_volume\
.delay(profile, cluster, master,
self._model.filter(volume, getCurrentUser()), path,
secret_key, girder_callback_info)
volume['status'] = VolumeState.ATTACHING
volume = self._model.update_volume(getCurrentUser(), volume)
return self._model.filter(volume, getCurrentUser())
def terminate(self):
self.status = ClusterStatus.TERMINATING
base_url = getApiUrl()
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
playbook = get_property(
'config.launch.spec', self.cluster, default=self.DEFAULT_PLAYBOOK)
playbook_params = get_property(
'config.launch.params', self.cluster, default={})
playbook_params['cluster_state'] = 'absent'
cumulus.ansible.tasks.cluster.terminate_cluster \
.delay(playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key, playbook_params, girder_token,
log_write_url, ClusterStatus.TERMINATED)
def delete(self, volume, params):
if 'clusterId' in volume:
raise RestException('Unable to delete attached volume')
# If the volume is in state created and it has no ec2 volume id
# associated with it, we should be able to just delete it
if volume['status'] == VolumeState.CREATED:
if 'id' in volume['ec2'] and volume['ec2']['id'] is not None:
raise RestException(
'Unable to delete volume, it is '
'associated with an ec2 volume %s' % volume['ec2']['id'])
self._model.remove(volume)
return None
# Call EC2 to delete volume
profile_id = parse('profileId').find(volume)[0].value
profile, secret_key = _get_profile(profile_id)
girder_callback_info = {
'girder_api_url': getApiUrl(),
'girder_token': get_task_token()['_id']}
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
aws_volume = p.get_volume(volume)
if aws_volume['state'] != VolumeState.AVAILABLE:
raise RestException(
'Volume must be in an "%s" status to be deleted'
% VolumeState.AVAILABLE, 400)
user = getCurrentUser()
cumulus.ansible.tasks.volume.delete_volume\
.delay(profile, self._model.filter(volume, user),
secret_key, girder_callback_info)
volume['status'] = VolumeState.DELETING
volume = self._model.update_volume(user, volume)
return self._model.filter(volume, user)
def start(self, request_body):
"""
Adapters may implement this if they support a start operation.
"""
self.status = ClusterStatus.LAUNCHING
self.cluster['config'].setdefault('provision', {})\
.setdefault('params', {}).update(request_body)
self.cluster = self.model('cluster', 'cumulus').save(self.cluster)
base_url = getApiUrl()
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
# Launch
launch_playbook = get_property(
'config.launch.spec', self.cluster, default=self.DEFAULT_PLAYBOOK)
launch_playbook_params = get_property(
'config.launch.params', self.cluster, default={})
launch_playbook_params['cluster_state'] = ClusterStatus.RUNNING
# Provision
provision_playbook = get_property(
'config.provision.spec', self.cluster, default='gridengine/site')
provision_playbook_params = get_property(
'config.provision.params', self.cluster, default={})
provision_ssh_user = get_property(
'config.provision.ssh.user', self.cluster, default='ubuntu')
provision_playbook_params['ansible_ssh_user'] = provision_ssh_user
provision_playbook_params['cluster_state'] = ClusterStatus.RUNNING
cumulus.ansible.tasks.cluster.start_cluster \
.delay(launch_playbook,
# provision playbook
provision_playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key,
launch_playbook_params, provision_playbook_params,
girder_token, log_write_url)
def terminate(self):
self.status = ClusterStatus.TERMINATING
base_url = cumulus.config.girder.baseUrl
log_write_url = '%s/clusters/%s/log' % (base_url, self.cluster['_id'])
girder_token = get_task_token()['_id']
profile, secret_key = _get_profile(self.cluster['profileId'])
playbook = get_property('config.launch.spec',
self.cluster,
default=self.DEFAULT_PLAYBOOK)
playbook_params = get_property('config.launch.params',
self.cluster,
default={})
playbook_params['cluster_state'] = 'absent'
cumulus.ansible.tasks.cluster.terminate_cluster \
.delay(playbook,
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
profile, secret_key, playbook_params, girder_token,
log_write_url, ClusterStatus.TERMINATED)
def delete_profile(user, profile, params):
query = {'profileId': profile['_id']}
if ModelImporter.model('volume', 'cumulus').findOne(query):
raise RestException(
'Unable to delete profile as it is associated with'
' a volume', 400)
if ModelImporter.model('cluster', 'cumulus').findOne(query):
raise RestException(
'Unable to delete profile as it is associated with'
' a cluster', 400)
# Clean up key associate with profile
cumulus.aws.ec2.tasks.key.delete_key_pair.delay(_filter(profile),
get_task_token()['_id'])
client = get_ec2_client(profile)
client.delete_key_pair(KeyName=str(profile['_id']))
ModelImporter.model('aws', 'cumulus').remove(profile)
def get_task_token(self, cluster=None):
return get_task_token(cluster)
def delete(self):
super(TraditionClusterAdapter, self).delete()
# Clean up key associate with cluster
cumulus.ssh.tasks.key.delete_key_pair.delay(
self._model.filter(self.cluster, getCurrentUser(), passphrase=False),
get_task_token()['_id'])
