def populate_anti_affinity_choices(self, request, context):
sahara = saharaclient(request)
plugin, version = whelpers.get_plugin_and_hadoop_version(request)
version_details = sahara.plugins.get_version_details(plugin, version)
process_choices = []
for service, processes in version_details.node_processes.items():
for process in processes:
process_choices.append((process, process))
cluster_template_id = request.REQUEST.get("cluster_template_id", None)
if cluster_template_id is None:
selected_processes = request.REQUEST.get("aa_groups", [])
else:
cluster_template = sahara.cluster_templates.get(cluster_template_id)
selected_processes = cluster_template.anti_affinity
checked_dict = dict()
for process in selected_processes:
checked_dict[process] = process
self.fields['anti_affinity'].initial = checked_dict
return process_choices
def handle(self, request, context):
sahara = saharaclient(request)
node_groups = None
plugin, hadoop_version = (
whelpers.get_plugin_and_hadoop_version(request))
ct_id = context["cluster_general_cluster_template"] or None
user_keypair = context["cluster_general_keypair"] or None
cluster = sahara.clusters.create(
context["cluster_general_cluster_name"],
plugin, hadoop_version,
cluster_template_id=ct_id,
default_image_id=context["cluster_general_image"],
description=context["cluster_general_description"],
node_groups=node_groups,
user_keypair_id=user_keypair,
is_transient=not(context["cluster_general_persist_cluster"]),
net_id=context.get("cluster_general_neutron_management_network",
None))
sahara.job_executions.create(
context["job_general_job"],
cluster.id,
context["job_general_job_input"],
context["job_general_job_output"],
context["job_config"])
return True
def get_object(self, *args, **kwargs):
if not hasattr(self, "_object"):
template_id = self.kwargs['template_id']
sahara = saharaclient(self.request)
template = sahara.cluster_templates.get(template_id)
self._object = template
return self._object
def __init__(self, request, *args, **kwargs):
super(JobConfigAction, self).__init__(request, *args, **kwargs)
job_ex_id = request.REQUEST.get("job_execution_id")
if job_ex_id is not None:
client = saharaclient(request)
job_ex_id = request.REQUEST.get("job_execution_id")
job_ex = client.job_executions.get(job_ex_id)
job_configs = job_ex.job_configs
edp_configs = {}
if 'configs' in job_configs:
configs, edp_configs = (
self.clean_edp_configs(job_configs['configs']))
self.fields['job_configs'].initial = (
json.dumps(configs))
if 'params' in job_configs:
self.fields['job_params'].initial = (
json.dumps(job_configs['params']))
job_args = json.dumps(job_configs['args'])
self.fields['job_args_array'].initial = job_args
if self.MAIN_CLASS in edp_configs:
self.fields['main_class'].initial = (
edp_configs[self.MAIN_CLASS])
if self.JAVA_OPTS in edp_configs:
self.fields['java_opts'].initial = (
edp_configs[self.JAVA_OPTS])
if self.EDP_MAPPER in edp_configs:
self.fields['streaming_mapper'].initial = (
edp_configs[self.EDP_MAPPER])
if self.EDP_REDUCER in edp_configs:
self.fields['streaming_reducer'].initial = (
edp_configs[self.EDP_REDUCER])
def populate_anti_affinity_choices(self, request, context):
sahara = saharaclient(request)
plugin, version = whelpers.get_plugin_and_hadoop_version(request)
version_details = sahara.plugins.get_version_details(plugin, version)
process_choices = []
for service, processes in version_details.node_processes.items():
for process in processes:
process_choices.append((process, process))
cluster_template_id = request.REQUEST.get("cluster_template_id", None)
if cluster_template_id is None:
selected_processes = request.REQUEST.get("aa_groups", [])
else:
cluster_template = sahara.cluster_templates.get(cluster_template_id)
selected_processes = cluster_template.anti_affinity
checked_dict = dict()
for process in selected_processes:
checked_dict[process] = process
self.fields['anti_affinity'].initial = checked_dict
return process_choices
def get_context_data(self, request):
cluster_id = self.tab_group.kwargs['cluster_id']
sahara = saharaclient(request)
cluster = sahara.clusters.get(cluster_id)
for info_key, info_val in cluster.info.items():
for key, val in info_val.items():
if str(val).startswith(('http://', 'https://')):
cluster.info[info_key][key] = build_link(val)
base_image = glance.image_get(request, cluster.default_image_id)
if getattr(cluster, 'cluster_template_id', None):
cluster_template = helpers.safe_call(sahara.cluster_templates.get,
cluster.cluster_template_id)
else:
cluster_template = None
if getattr(cluster, 'neutron_management_network', None):
net_id = cluster.neutron_management_network
network = neutron.network_get(request, net_id)
network.set_id_as_name_if_empty()
net_name = network.name
else:
net_name = None
return {
"cluster": cluster,
"base_image": base_image,
"cluster_template": cluster_template,
"network": net_name
}
def get_context_data(self, request):
cluster_id = self.tab_group.kwargs['cluster_id']
sahara = saharaclient(request)
cluster = sahara.clusters.get(cluster_id)
for info_key, info_val in cluster.info.items():
for key, val in info_val.items():
if str(val).startswith(('http://', 'https://')):
cluster.info[info_key][key] = build_link(val)
base_image = glance.image_get(request,
cluster.default_image_id)
if getattr(cluster, 'cluster_template_id', None):
cluster_template = helpers.safe_call(sahara.cluster_templates.get,
cluster.cluster_template_id)
else:
cluster_template = None
if getattr(cluster, 'neutron_management_network', None):
net_id = cluster.neutron_management_network
network = neutron.network_get(request, net_id)
network.set_id_as_name_if_empty()
net_name = network.name
else:
net_name = None
return {"cluster": cluster,
"base_image": base_image,
"cluster_template": cluster_template,
"network": net_name}
def populate_main_binary_choices(self, request, context):
sahara = saharaclient(request)
job_binaries = sahara.job_binaries.list()
choices = [(job_binary.id, job_binary.name)
for job_binary in job_binaries]
choices.insert(0, ('', '-- not selected --'))
return choices
def get_context_data(self, request):
job_execution_id = self.tab_group.kwargs['job_execution_id']
sahara = saharaclient(request)
job_execution = sahara.job_executions.get(job_execution_id)
object_names = self.get_object_names(job_execution, sahara)
return {"job_execution": job_execution,
"object_names": object_names}
def populate_property_name_choices(self, request, context):
client = saharaclient(request)
job_id = request.REQUEST.get("job_id") or request.REQUEST.get("job")
job_type = client.jobs.get(job_id).type
job_configs = client.jobs.get_configs(job_type).job_config
choices = [(param['value'], param['name'])
for param in job_configs['configs']]
return choices
def populate_job_choices(self, request):
sahara = saharaclient(request)
jobs = sahara.jobs.list()
choices = [(job.id, job.name)
for job in jobs]
return choices
def populate_cluster_choices(self, request, context):
sahara = saharaclient(request)
clusters = sahara.clusters.list()
choices = [(cluster.id, cluster.name)
for cluster in clusters]
return choices
def populate_main_binary_choices(self, request, context):
sahara = saharaclient(request)
job_binaries = sahara.job_binaries.list()
choices = [(job_binary.id, job_binary.name)
for job_binary in job_binaries]
choices.insert(0, ('', '-- not selected --'))
return choices
def get(self, request, *args, **kwargs):
if request.is_ajax():
if request.REQUEST.get("json", None):
job_id = request.REQUEST.get("job_id")
sahara = saharaclient(request)
job_type = sahara.jobs.get(job_id).type
return HttpResponse(json.dumps({"job_type": job_type}),
mimetype='application/json')
return super(LaunchJobView, self).get(request, args, kwargs)
def get_data_source_choices(self, request, context):
sahara = saharaclient(request)
data_sources = sahara.data_sources.list()
choices = [(data_source.id, data_source.name)
for data_source in data_sources]
choices.insert(0, (None, 'None'))
return choices
def get(self, request, *args, **kwargs):
if request.is_ajax():
if request.REQUEST.get("json", None):
job_id = request.REQUEST.get("job_id")
sahara = saharaclient(request)
job_type = sahara.jobs.get(job_id).type
return HttpResponse(json.dumps({"job_type": job_type}),
mimetype='application/json')
return super(LaunchJobView, self).get(request, args, kwargs)
def handle(self, request, context):
sahara = saharaclient(request)
sahara.data_sources.create(
context["general_data_source_name"],
context["general_data_source_description"],
context["general_data_source_type"],
context["source_url"],
context.get("general_data_source_credential_user", None),
context.get("general_data_source_credential_pass", None))
return True
def handle(self, request, context):
sahara = saharaclient(request)
sahara.data_sources.create(
context["general_data_source_name"],
context["general_data_source_description"],
context["general_data_source_type"],
context["source_url"],
context.get("general_data_source_credential_user", None),
context.get("general_data_source_credential_pass", None))
return True
def handle(self, request, context):
sahara = saharaclient(request)
sahara.job_executions.create(
context["job_general_job"],
context["job_general_cluster"],
context["job_general_job_input"],
context["job_general_job_output"],
context["job_config"])
return True
def get_data(self, request, instance_id):
sahara = saharaclient(request)
try:
return sahara.clusters.get(instance_id)
except api_base.APIException as e:
if e.error_code == 404:
# returning 404 to the ajax call removes the
# row from the table on the ui
raise Http404
else:
messages.error(request, e)
def get_context_data(self, request):
template_id = self.tab_group.kwargs['template_id']
sahara = saharaclient(request)
template = sahara.cluster_templates.get(template_id)
for ng in template.node_groups:
if not ng["flavor_id"]:
continue
ng["flavor_name"] = nova.flavor_get(request, ng["flavor_id"]).name
ng["node_group_template"] = helpers.safe_call(
sahara.node_group_templates.get,
ng.get("node_group_template_id", None))
return {"template": template}
def get_context_data(self, request):
template_id = self.tab_group.kwargs['template_id']
sahara = saharaclient(request)
template = sahara.cluster_templates.get(template_id)
for ng in template.node_groups:
if not ng["flavor_id"]:
continue
ng["flavor_name"] = nova.flavor_get(request, ng["flavor_id"]).name
ng["node_group_template"] = helpers.safe_call(
sahara.node_group_templates.get,
ng.get("node_group_template_id", None))
return {"template": template}
def __init__(self, request, context_seed, entry_point, *args, **kwargs):
sahara = saharaclient(request)
template_id = context_seed["template_id"]
template = sahara.cluster_templates.get(template_id)
self._set_configs_to_copy(template.cluster_configs)
request.GET = request.GET.copy()
request.GET.update({"plugin_name": template.plugin_name})
request.GET.update({"hadoop_version": template.hadoop_version})
request.GET.update({"aa_groups": template.anti_affinity})
super(CopyClusterTemplate, self).__init__(request, context_seed,
entry_point, *args, **kwargs)
#init Node Groups
for step in self.steps:
if isinstance(step, create_flow.ConfigureNodegroups):
ng_action = step.action
template_ngs = template.node_groups
if 'forms_ids' not in request.POST:
ng_action.groups = []
for id in range(0, len(template_ngs), 1):
group_name = "group_name_" + str(id)
template_id = "template_id_" + str(id)
count = "count_" + str(id)
templ_ng = template_ngs[id]
ng_action.groups.append({
"name":
templ_ng["name"],
"template_id":
templ_ng["node_group_template_id"],
"count":
templ_ng["count"],
"id":
id,
"deletable":
"true"
})
build_node_group_fields(ng_action, group_name,
template_id, count)
elif isinstance(step, create_flow.GeneralConfig):
fields = step.action.fields
fields["cluster_template_name"].initial = \
template.name + "-copy"
fields["description"].initial = template.description
def __init__(self, request, context_seed, entry_point, *args, **kwargs):
sahara = saharaclient(request)
template_id = context_seed["template_id"]
template = sahara.node_group_templates.get(template_id)
self._set_configs_to_copy(template.node_configs)
plugin = template.plugin_name
hadoop_version = template.hadoop_version
request.GET = request.GET.copy()
request.GET.update({"plugin_name": plugin})
request.GET.update({"hadoop_version": hadoop_version})
super(CopyNodegroupTemplate,
self).__init__(request, context_seed, entry_point, *args,
**kwargs)
for step in self.steps:
if not isinstance(step, create_flow.GeneralConfig):
continue
fields = step.action.fields
fields["nodegroup_name"].initial = template.name + "-copy"
fields["description"].initial = template.description
fields["flavor"].initial = template.flavor_id
storage = "cinder_volume" if template.volumes_per_node > 0 \
else "ephemeral_drive"
volumes_per_node = template.volumes_per_node
volumes_size = template.volumes_size
fields["storage"].initial = storage
fields["volumes_per_node"].initial = volumes_per_node
fields["volumes_size"].initial = volumes_size
if template.floating_ip_pool:
fields['floating_ip_pool'].initial = template.floating_ip_pool
processes_dict = dict()
plugin_details = sahara.plugins.get_version_details(
plugin, hadoop_version)
plugin_node_processes = plugin_details.node_processes
for process in template.node_processes:
#need to know the service
_service = None
for service, processes in plugin_node_processes.items():
if process in processes:
_service = service
break
processes_dict.update({"%s:%s" % (_service, process): process})
fields["processes"].initial = processes_dict
def __init__(self, request, context_seed, entry_point, *args, **kwargs):
sahara = saharaclient(request)
template_id = context_seed["template_id"]
template = sahara.cluster_templates.get(template_id)
self._set_configs_to_copy(template.cluster_configs)
request.GET = request.GET.copy()
request.GET.update({"plugin_name": template.plugin_name})
request.GET.update({"hadoop_version": template.hadoop_version})
request.GET.update({"aa_groups": template.anti_affinity})
super(CopyClusterTemplate, self).__init__(request, context_seed,
entry_point, *args,
**kwargs)
#init Node Groups
for step in self.steps:
if isinstance(step, create_flow.ConfigureNodegroups):
ng_action = step.action
template_ngs = template.node_groups
if 'forms_ids' not in request.POST:
ng_action.groups = []
for id in range(0, len(template_ngs), 1):
group_name = "group_name_" + str(id)
template_id = "template_id_" + str(id)
count = "count_" + str(id)
templ_ng = template_ngs[id]
ng_action.groups.append(
{"name": templ_ng["name"],
"template_id": templ_ng["node_group_template_id"],
"count": templ_ng["count"],
"id": id,
"deletable": "true"})
build_node_group_fields(ng_action,
group_name,
template_id,
count)
elif isinstance(step, create_flow.GeneralConfig):
fields = step.action.fields
fields["cluster_template_name"].initial = \
template.name + "-copy"
fields["description"].initial = template.description
def handle(self, request, context):
sahara = saharaclient(request)
main_locations = []
lib_locations = []
for k in context.keys():
if k.startswith('lib_'):
lib_locations.append(context.get(k))
if context.get("main_binary", None):
main_locations.append(context["main_binary"])
sahara.jobs.create(context["job_name"], context["job_type"],
main_locations, lib_locations,
context["job_description"])
return True
def get_cluster_instances_data(self):
cluster_id = self.tab_group.kwargs['cluster_id']
sahara = saharaclient(self.request)
cluster = sahara.clusters.get(cluster_id)
instances = []
for ng in cluster.node_groups:
for instance in ng["instances"]:
instances.append(Instance(
name=instance["instance_name"],
id=instance["instance_id"],
internal_ip=instance.get("internal_ip",
"Not assigned"),
management_ip=instance.get("management_ip",
"Not assigned")))
return instances
def get_cluster_instances_data(self):
cluster_id = self.tab_group.kwargs['cluster_id']
sahara = saharaclient(self.request)
cluster = sahara.clusters.get(cluster_id)
instances = []
for ng in cluster.node_groups:
for instance in ng["instances"]:
instances.append(
Instance(name=instance["instance_name"],
id=instance["instance_id"],
internal_ip=instance.get("internal_ip",
"Not assigned"),
management_ip=instance.get(
"management_ip", "Not assigned")))
return instances
def update_context_with_plugin_tags(request, context):
sahara = saharaclient(request)
plugins = sahara.plugins.list()
plugins_object = dict()
for plugin in plugins:
plugins_object[plugin.name] = dict()
for version in plugin.versions:
plugins_object[plugin.name][version] = []
details = sahara.plugins.get_version_details(plugin.name, version)
for tag in details.required_image_tags:
plugins_object[plugin.name][version].append(tag)
context["plugins"] = plugins_object
return context
def __init__(self, request, *args, **kwargs):
super(SelectHadoopPluginAction, self).__init__(request,
*args,
**kwargs)
self.fields["job_id"] = forms.ChoiceField(
label=_("Plugin name"),
required=True,
initial=request.GET.get("job_id") or request.POST.get("job_id"),
widget=forms.HiddenInput(attrs={"class": "hidden_create_field"}))
self.fields["job_configs"] = forms.ChoiceField(
label=_("Job configs"),
required=True,
widget=forms.HiddenInput(attrs={"class": "hidden_create_field"}))
self.fields["job_args"] = forms.ChoiceField(
label=_("Job args"),
required=True,
widget=forms.HiddenInput(attrs={"class": "hidden_create_field"}))
self.fields["job_params"] = forms.ChoiceField(
label=_("Job params"),
required=True,
widget=forms.HiddenInput(attrs={"class": "hidden_create_field"}))
job_ex_id = request.REQUEST.get("job_execution_id")
if job_ex_id is not None:
self.fields["job_execution_id"] = forms.ChoiceField(
label=_("Job Execution Id"),
required=True,
initial=request.REQUEST.get("job_execution_id"),
widget=forms.HiddenInput(
attrs={"class": "hidden_create_field"}))
client = saharaclient(request)
job_ex_id = request.REQUEST.get("job_execution_id")
job_configs = client.job_executions.get(job_ex_id).job_configs
if "configs" in job_configs:
self.fields["job_configs"].initial = (
json.dumps(job_configs["configs"]))
if "params" in job_configs:
self.fields["job_params"].initial = (
json.dumps(job_configs["params"]))
if "args" in job_configs:
self.fields["job_args"].initial = (
json.dumps(job_configs["args"]))
def get(self, request, job_binary_id=None):
try:
sahara = saharaclient(request)
jb = sahara.job_binaries.get(job_binary_id)
data = sahara.job_binaries.get_file(job_binary_id)
except Exception:
redirect = reverse('horizon:sahara:job_binaries:index')
exceptions.handle(self.request,
_('Unable to fetch job binary: %(exc)s'),
redirect=redirect)
response = http.HttpResponse(mimetype='application/binary')
response['Content-Disposition'] = \
'attachment; filename=%s' % slugify(jb.name)
response.write(data)
response['Content-Length'] = str(len(data))
return response
def update_context_with_plugin_tags(request, context):
sahara = saharaclient(request)
plugins = sahara.plugins.list()
plugins_object = dict()
for plugin in plugins:
plugins_object[plugin.name] = dict()
for version in plugin.versions:
plugins_object[plugin.name][version] = []
details = sahara.plugins.get_version_details(plugin.name,
version)
for tag in details.required_image_tags:
plugins_object[plugin.name][version].append(tag)
context["plugins"] = plugins_object
return context
def get(self, request, job_binary_id=None):
try:
sahara = saharaclient(request)
jb = sahara.job_binaries.get(job_binary_id)
data = sahara.job_binaries.get_file(job_binary_id)
except Exception:
redirect = reverse('horizon:sahara:job_binaries:index')
exceptions.handle(self.request,
_('Unable to fetch job binary: %(exc)s'),
redirect=redirect)
response = http.HttpResponse(mimetype='application/binary')
response['Content-Disposition'] = \
'attachment; filename=%s' % slugify(jb.name)
response.write(data)
response['Content-Length'] = str(len(data))
return response
def handle(self, request, context):
sahara = saharaclient(request)
main_locations = []
lib_locations = []
for k in context.keys():
if k.startswith('lib_'):
lib_locations.append(context.get(k))
if context.get("main_binary", None):
main_locations.append(context["main_binary"])
sahara.jobs.create(
context["job_name"],
context["job_type"],
main_locations,
lib_locations,
context["job_description"])
return True
def action(self, request, obj_id):
sahara = saharaclient(request)
sahara.images.unregister_image(obj_id)
def get_data(self):
sahara = saharaclient(self.request)
nodegroup_templates = sahara.node_group_templates.list()
return nodegroup_templates
def action(self, request, obj_id):
sahara = saharaclient(request)
sahara.data_sources.delete(obj_id)
def get_context_data(self, request):
template_id = self.tab_group.kwargs['template_id']
sahara = saharaclient(request)
template = sahara.node_group_templates.get(template_id)
flavor = nova.flavor_get(request, template.flavor_id)
return {"template": template, "flavor": flavor}
def action(self, request, obj_id):
sahara = saharaclient(request)
sahara.clusters.delete(obj_id)
def get_data(self, request, job_execution_id):
sahara = saharaclient(request)
job_execution = sahara.job_executions.get(job_execution_id)
return job_execution
def get_context_data(self, request):
plugin_id = self.tab_group.kwargs['plugin_id']
sahara = saharaclient(request)
plugin = sahara.plugins.get(plugin_id)
return {"plugin": plugin}
def action(self, request, obj_id):
sahara = saharaclient(request)
sahara.jobs.delete(obj_id)
def get_data(self):
sahara = saharaclient(self.request)
jobs = sahara.job_executions.list()
return jobs
def get_context_data(self, request):
template_id = self.tab_group.kwargs['template_id']
sahara = saharaclient(request)
template = sahara.cluster_templates.get(template_id)
return {"template": template}
def action(self, request, template_id):
sahara = saharaclient(request)
sahara.cluster_templates.delete(template_id)
def get_context_data(self, request):
job_binary_id = self.tab_group.kwargs['job_binary_id']
sahara = saharaclient(request)
job_binary = sahara.job_binaries.get(job_binary_id)
return {"job_binary": job_binary}
def action(self, request, obj_id):
sahara = saharaclient(request)
sahara.data_sources.delete(obj_id)
def get_data(self):
sahara = saharaclient(self.request)
data_sources = sahara.data_sources.list()
return data_sources
def get_data(self):
sahara = saharaclient(self.request)
return sahara.plugins.list()
def action(self, request, template_id):
sahara = saharaclient(request)
sahara.node_group_templates.delete(template_id)
def get_context_data(self, request):
template_id = self.tab_group.kwargs['template_id']
sahara = saharaclient(request)
template = sahara.node_group_templates.get(template_id)
return {"template": template}
def get_data(self):
sahara = saharaclient(self.request)
job_binaries = sahara.job_binaries.list()
return job_binaries
def action(self, request, obj_id):
sahara = saharaclient(request)
sahara.images.unregister_image(obj_id)
def get_context_data(self, request):
plugin_id = self.tab_group.kwargs['plugin_id']
sahara = saharaclient(request)
plugin = sahara.plugins.get(plugin_id)
return {"plugin": plugin}
def get_data(self):
sahara = saharaclient(self.request)
cluster_templates = sahara.cluster_templates.list()
return cluster_templates
def get_data(self, request, instance_id):
sahara = saharaclient(request)
instance = sahara.clusters.get(instance_id)
return instance
def get_context_data(self, request):
data_source_id = self.tab_group.kwargs['data_source_id']
sahara = saharaclient(request)
data_source = sahara.data_sources.get(data_source_id)
return {"data_source": data_source}
def get_data(self):
sahara = saharaclient(self.request)
return sahara.plugins.list()