def provision_cluster(playbook, cluster, profile, secret_key, extra_vars,
girder_token, log_write_url, post_status):
playbook = get_playbook_path(playbook)
playbook_variables = get_playbook_variables(cluster, profile, extra_vars)
env = os.environ.copy()
env.update({
'AWS_ACCESS_KEY_ID': profile['accessKeyId'],
'AWS_SECRET_ACCESS_KEY': secret_key,
'GIRDER_TOKEN': girder_token,
'LOG_WRITE_URL': log_write_url,
'CLUSTER_ID': cluster['_id'],
'REGION_NAME': profile['regionName'],
'ANSIBLE_HOST_KEY_CHECKING': 'false',
'ANSIBLE_CALLBACK_PLUGINS': get_callback_plugins_path()
})
inventory = os.path.join(os.path.dirname(__file__), 'providers', 'ec2.py')
ansible = run_playbook(playbook,
inventory,
playbook_variables,
env=env,
verbose=3)
check_girder_cluster_status(cluster, girder_token, post_status)
check_ansible_return_code(ansible, cluster, girder_token)
def launch_cluster(playbook, cluster, profile, secret_key, extra_vars,
girder_token, log_write_url, post_status):
playbook = get_playbook_path(playbook)
playbook_variables = get_playbook_variables(cluster, profile, extra_vars)
env = os.environ.copy()
env.update({'AWS_ACCESS_KEY_ID': profile['accessKeyId'],
'AWS_SECRET_ACCESS_KEY': secret_key,
'GIRDER_TOKEN': girder_token,
'LOG_WRITE_URL': log_write_url,
'CLUSTER_ID': cluster['_id']})
inventory = simple_inventory('localhost')
with inventory.to_tempfile() as inventory_path:
ansible = run_playbook(playbook, inventory_path, playbook_variables,
env=env, verbose=3)
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
master = p.get_master_instance(cluster['_id'])
status_url = '%s/clusters/%s' % (cumulus.config.girder.baseUrl,
cluster['_id'])
updates = {
'config': {
'host': master['public_ip']
}
}
headers = {'Girder-Token': girder_token}
r = requests.patch(status_url, headers=headers, json=updates)
check_status(r)
check_ansible_return_code(ansible, cluster, girder_token)
check_girder_cluster_status(cluster, girder_token, post_status)
def provision_ec2_cluster(task, cluster, profile, provision_params={}):
if 'ansible_ssh_user' not in provision_params:
provision_params['ansible_ssh_user'] = '******'
if 'cluster_state' not in provision_params:
provision_params['cluster_state'] = 'running'
girder_token = task.taskflow.girder_token
client = create_girder_client(task.taskflow.girder_api_url, girder_token)
secret_key = profile['secretAccessKey']
profile = profile.copy()
del profile['secretAccessKey']
log_write_url = '%s/clusters/%s/log' % (task.taskflow.girder_api_url,
cluster['_id'])
check_girder_cluster_status(cluster, girder_token, 'provisioning')
cumulus.ansible.tasks.cluster.provision_cluster(PROVISION_SPEC, cluster,
profile, secret_key,
provision_params,
girder_token,
log_write_url, 'running')
# Now update the cluster state to 'running' Is this needed?
check_girder_cluster_status(cluster, girder_token, 'running')
# Get the update to data cluster
cluster = client.get('clusters/%s' % cluster['_id'])
return cluster
def start_cluster(launch_playbook, provision_playbook, cluster, profile,
secret_key, launch_extra_vars, provision_extra_vars,
girder_token, log_write_url):
launch_cluster(launch_playbook, cluster, profile, secret_key,
launch_extra_vars, girder_token, log_write_url, 'running')
check_girder_cluster_status(cluster, girder_token, 'provisioning')
provision_cluster(provision_playbook, cluster, profile, secret_key,
provision_extra_vars, girder_token, log_write_url,
'running')
# Now update the cluster state to 'running'
check_girder_cluster_status(cluster, girder_token, 'running')
def start_cluster(launch_playbook, provision_playbook, cluster, profile,
secret_key, launch_extra_vars, provision_extra_vars,
girder_token, log_write_url):
launch_cluster(launch_playbook, cluster, profile, secret_key,
launch_extra_vars, girder_token, log_write_url, 'running')
check_girder_cluster_status(cluster, girder_token, 'provisioning')
provision_cluster(provision_playbook, cluster, profile, secret_key,
provision_extra_vars, girder_token, log_write_url,
'running')
# Now update the cluster state to 'running'
check_girder_cluster_status(cluster, girder_token, 'running')
def terminate_cluster(playbook, cluster, profile, secret_key, extra_vars,
girder_token, log_write_url, post_status):
playbook = get_playbook_path(playbook)
playbook_variables = get_playbook_variables(cluster, profile, extra_vars)
env = os.environ.copy()
env.update({
'AWS_ACCESS_KEY_ID': profile['accessKeyId'],
'AWS_SECRET_ACCESS_KEY': secret_key,
'GIRDER_TOKEN': girder_token,
'LOG_WRITE_URL': log_write_url,
'CLUSTER_ID': cluster['_id']
})
# if there are any volumes, make sure to detach them first.
if 'volumes' in cluster and len(cluster['volumes']):
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
master = p.get_master_instance(cluster['_id'])
for volume_id in cluster['volumes']:
r = requests.get('%s/volumes/%s' %
(cumulus.config.girder.baseUrl, volume_id),
headers={'Girder-Token': girder_token})
check_status(r)
volume = r.json()
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': girder_token
}
vol_log_url = '%s/volumes/%s/log' % (cumulus.config.girder.baseUrl,
volume_id)
detach_volume(profile, cluster, master, volume, secret_key,
vol_log_url, girder_callback_info)
inventory = simple_inventory('localhost')
with inventory.to_tempfile() as inventory_path:
ansible = run_playbook(playbook,
inventory_path,
playbook_variables,
env=env,
verbose=3)
check_ansible_return_code(ansible, cluster, girder_token)
check_girder_cluster_status(cluster, girder_token, post_status)
def terminate_cluster(playbook, cluster, profile, secret_key, extra_vars,
girder_token, log_write_url, post_status):
playbook = get_playbook_path(playbook)
playbook_variables = get_playbook_variables(cluster, profile, extra_vars)
env = os.environ.copy()
env.update({'AWS_ACCESS_KEY_ID': profile['accessKeyId'],
'AWS_SECRET_ACCESS_KEY': secret_key,
'GIRDER_TOKEN': girder_token,
'LOG_WRITE_URL': log_write_url,
'CLUSTER_ID': cluster['_id']})
# if there are any volumes, make sure to detach them first.
if 'volumes' in cluster and len(cluster['volumes']):
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
master = p.get_master_instance(cluster['_id'])
for volume_id in cluster['volumes']:
r = requests.get('%s/volumes/%s' %
(cumulus.config.girder.baseUrl, volume_id),
headers={'Girder-Token': girder_token})
check_status(r)
volume = r.json()
girder_callback_info = {
'girder_api_url': cumulus.config.girder.baseUrl,
'girder_token': girder_token}
detach_volume(profile, cluster, master, volume,
secret_key, girder_callback_info)
inventory = simple_inventory('localhost')
with inventory.to_tempfile() as inventory_path:
ansible = run_playbook(playbook, inventory_path, playbook_variables,
env=env, verbose=3)
check_ansible_return_code(ansible, cluster, girder_token)
check_girder_cluster_status(cluster, girder_token, post_status)
def provision_cluster(playbook, cluster, profile, secret_key, extra_vars,
girder_token, log_write_url, post_status):
playbook = get_playbook_path(playbook)
playbook_variables = get_playbook_variables(cluster, profile, extra_vars)
env = os.environ.copy()
env.update({'AWS_ACCESS_KEY_ID': profile['accessKeyId'],
'AWS_SECRET_ACCESS_KEY': secret_key,
'GIRDER_TOKEN': girder_token,
'LOG_WRITE_URL': log_write_url,
'CLUSTER_ID': cluster['_id'],
'REGION_NAME': profile['regionName'],
'ANSIBLE_HOST_KEY_CHECKING': 'false',
'ANSIBLE_CALLBACK_PLUGINS': get_callback_plugins_path()})
inventory = os.path.join(os.path.dirname(__file__), 'providers', 'ec2.py')
ansible = run_playbook(playbook, inventory, playbook_variables,
env=env, verbose=3)
check_girder_cluster_status(cluster, girder_token, post_status)
check_ansible_return_code(ansible, cluster, girder_token)
def launch_cluster(playbook, cluster, profile, secret_key, extra_vars,
girder_token, log_write_url, post_status):
playbook = get_playbook_path(playbook)
playbook_variables = get_playbook_variables(cluster, profile, extra_vars)
env = os.environ.copy()
env.update({
'AWS_ACCESS_KEY_ID': profile['accessKeyId'],
'AWS_SECRET_ACCESS_KEY': secret_key,
'GIRDER_TOKEN': girder_token,
'LOG_WRITE_URL': log_write_url,
'CLUSTER_ID': cluster['_id']
})
inventory = simple_inventory('localhost')
with inventory.to_tempfile() as inventory_path:
ansible = run_playbook(playbook,
inventory_path,
playbook_variables,
env=env,
verbose=3)
p = CloudProvider(dict(secretAccessKey=secret_key, **profile))
master = p.get_master_instance(cluster['_id'])
status_url = '%s/clusters/%s' % (cumulus.config.girder.baseUrl,
cluster['_id'])
updates = {'config': {'host': master['public_ip']}}
headers = {'Girder-Token': girder_token}
r = requests.patch(status_url, headers=headers, json=updates)
check_status(r)
check_ansible_return_code(ansible, cluster, girder_token)
check_girder_cluster_status(cluster, girder_token, post_status)
def setup_cluster(task, *args, **kwargs):
cluster = kwargs['cluster']
profile = kwargs.get('profile')
volume = kwargs.get('volume')
new = False
if '_id' in cluster:
task.taskflow.logger.info('We are using an existing cluster: %s' %
cluster['name'])
else:
new = True
task.taskflow.logger.info('We are creating an EC2 cluster.')
task.logger.info('Cluster name %s' % cluster['name'])
kwargs['machine'] = cluster.get('machine')
if volume:
config = cluster.setdefault('config', {})
config['jobOutputDir'] = '/data'
# Create the model in Girder
cluster = create_ec2_cluster(task, cluster, profile,
kwargs['image_spec'])
# Now launch the cluster
cluster = launch_ec2_cluster(task, cluster, profile)
task.logger.info('Cluster started.')
if volume and '_id' in volume:
task.taskflow.logger.info('We are using an existing volume: %s' %
volume['name'])
elif volume:
task.taskflow.logger.info('We are creating a new volume: "%s"' %
volume['name'])
volume = create_volume(task, volume, profile)
# Now provision
if new:
provision_params = {}
girder_token = task.taskflow.girder_token
check_girder_cluster_status(cluster, girder_token, 'provisioning')
# attach volume
if volume:
volume = _attach_volume(task, profile, volume, cluster)
path = volume.get('path')
if path:
provision_params['master_nfs_exports_extra'] = [path]
cluster = provision_ec2_cluster(task, cluster, profile,
provision_params)
# Call any follow on task
if 'next' in kwargs:
kwargs['cluster'] = cluster
next = Signature.from_dict(kwargs['next'])
if next.task == 'celery.chain':
# If we are dealing with a chain we want to update the arg and
# kwargs passed into the chain.
first_task = next.kwargs['tasks'][0]
if first_task:
if args:
first_task.args = tuple(args) + tuple(first_task.args)
if kwargs:
first_task.kwargs = dict(first_task.kwargs, **kwargs)
next.delay(*args, **kwargs)