def _copy_files(self, deployment_dir, oozie_xml, oozie_properties):
"""
Copy XML and the jar_path files from Java or MR actions to the deployment directory.
This should run as the workflow user.
"""
xml_path = self.fs.join(deployment_dir, self.job.XML_FILE_NAME)
self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml))
LOG.debug("Created %s" % (xml_path,))
properties_path = self.fs.join(deployment_dir, 'job.properties')
self.fs.create(properties_path, overwrite=True, permission=0644, data=smart_str('\n'.join(['%s=%s' % (key, val) for key, val in oozie_properties.iteritems()])))
LOG.debug("Created %s" % (properties_path,))
# List jar files
files = []
lib_path = self.fs.join(deployment_dir, 'lib')
if hasattr(self.job, 'node_list'):
for node in self.job.node_list:
if hasattr(node, 'jar_path') and not node.jar_path.startswith(lib_path):
files.append(node.jar_path)
# Copy the jar files to the workspace lib
if files:
for jar_file in files:
LOG.debug("Updating %s" % jar_file)
jar_lib_path = self.fs.join(lib_path, self.fs.basename(jar_file))
# Refresh if needed
if self.fs.exists(jar_lib_path):
stat_src = self.fs.stats(jar_file)
stat_dest = self.fs.stats(jar_lib_path)
if stat_src.fileId != stat_dest.fileId:
self.fs.remove(jar_lib_path, skip_trash=True)
self.fs.copyfile(jar_file, jar_lib_path)
def copy_configs(fields, unique_key_field, df):
# Create temporary copy of solr configs
tmp_path = tempfile.mkdtemp()
solr_config_path = os.path.join(tmp_path, os.path.basename(conf.CONFIG_TEMPLATE_PATH.get()))
shutil.copytree(conf.CONFIG_TEMPLATE_PATH.get(), solr_config_path)
if fields or unique_key_field:
# Get complete schema.xml
with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f:
schemaxml = SchemaXml(f.read())
schemaxml.uniqueKeyField(unique_key_field)
schemaxml.fields(fields)
# Write complete schema.xml to copy
with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
f.write(smart_str(schemaxml.xml))
if df:
# Get complete solrconfig.xml
with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/solrconfig.xml')) as f:
solrconfigxml = SolrConfigXml(f.read())
solrconfigxml.defaultField(df)
# Write complete solrconfig.xml to copy
with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
f.write(smart_str(solrconfigxml.xml))
return tmp_path, solr_config_path
def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True):
# Create temporary copy of solr configs
tmp_path = tempfile.mkdtemp()
config_template_path = get_config_template_path(solr_cloud_mode)
solr_config_path = os.path.join(tmp_path, 'solr_configs')
shutil.copytree(config_template_path, solr_config_path)
if fields or unique_key_field:
# Get complete schema.xml
with open(os.path.join(config_template_path, 'conf/schema.xml')) as f:
schemaxml = SchemaXml(f.read())
schemaxml.uniqueKeyField(unique_key_field)
schemaxml.fields(fields)
# Write complete schema.xml to copy
with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f:
f.write(smart_str(schemaxml.xml))
if df:
# Get complete solrconfig.xml
with open(os.path.join(config_template_path, 'conf/solrconfig.xml')) as f:
solrconfigxml = SolrConfigXml(f.read())
solrconfigxml.defaultField(df)
# Write complete solrconfig.xml to copy
with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f:
f.write(smart_str(solrconfigxml.xml))
return tmp_path, solr_config_path
def process_response(self, request, response):
if not _has_tidylib or not self._is_html(request, response):
return response
html, errors = tidylib.tidy_document(response.content, self._options, keep_doc=True)
if not errors:
return response
# Filter out what we care about
err_list = errors.rstrip().split("\n")
err_list = self._filter_warnings(err_list)
if not err_list:
return response
try:
fn = urlresolvers.resolve(request.path)[0]
fn_name = "%s.%s" % (fn.__module__, fn.__name__)
except:
fn_name = "<unresolved_url>"
# Write the two versions of html out for offline debugging
filename = os.path.join(self._outdir, fn_name)
result = (
"HTML tidy result: %s [%s]:"
"\n\t%s"
"\nPlease see %s.orig %s.tidy\n-------" % (request.path, fn_name, "\n\t".join(err_list), filename, filename)
)
file(filename + ".orig", "w").write(i18n.smart_str(response.content))
file(filename + ".tidy", "w").write(i18n.smart_str(html))
file(filename + ".info", "w").write(i18n.smart_str(result))
self._logger.error(result)
return response
def save_query(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
form = QueryForm()
api = get_api(request.user)
app_names = api.jars()
try:
form.bind(request.POST)
form.query.fields['appName'].choices = ((key, key) for key in app_names)
if form.is_valid():
design = save_design(request, form, query_type, design, True)
response['design_id'] = design.id
response['status'] = 0
else:
response['message'] = smart_str(form.query.errors) + smart_str(form.saveform.errors)
except RuntimeError, e:
response['message'] = str(e)
def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''):
for c in counts:
fq_fields = (fields + ':' if fields else '') + c['field']
fq_values = (smart_str(values) + ':' if values else '') + smart_str(c['value'])
if 'pivot' in c:
_augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values)
fq_filter = selected_values.get((facet_id, fq_fields, 'field'), [])
_selected_values = [f['value'] for f in fq_filter]
c['selected'] = fq_values in _selected_values
c['exclude'] = False
c['fq_fields'] = fq_fields
c['fq_values'] = fq_values
def _get_fq(self, collection, query):
params = ()
timeFilter = {}
if collection:
timeFilter = self._get_range_borders(collection, query)
if timeFilter and not timeFilter.get('time_filter_overrides'):
params += (('fq', urllib.unquote(utf_quoter('%(field)s:[%(from)s TO %(to)s]' % timeFilter))),)
# Merge facets queries on same fields
grouped_fqs = groupby(query['fqs'], lambda x: (x['type'], x['field']))
merged_fqs = []
for key, group in grouped_fqs:
field_fq = next(group)
for fq in group:
for f in fq['filter']:
field_fq['filter'].append(f)
merged_fqs.append(field_fq)
for fq in merged_fqs:
if fq['type'] == 'field':
fields = fq['field'] if type(fq['field']) == list else [fq['field']] # 2D facets support
for field in fields:
f = []
for _filter in fq['filter']:
values = _filter['value'] if type(_filter['value']) == list else [_filter['value']] # 2D facets support
if fields.index(field) < len(values): # Lowest common field denominator
value = values[fields.index(field)]
exclude = '-' if _filter['exclude'] else ''
if value is not None and ' ' in smart_str(value):
value = smart_str(value).replace('"', '\\"')
f.append('%s%s:"%s"' % (exclude, field, value))
else:
f.append('%s{!field f=%s}%s' % (exclude, field, value))
_params ='{!tag=%(id)s}' % fq + ' '.join(f)
params += (('fq', urllib.unquote(utf_quoter(_params))),)
elif fq['type'] == 'range':
params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote(
utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'], f['to']))) for field, f in zip(fq['filter'], fq['properties'])])),)
elif fq['type'] == 'range-up':
params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote(
utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'] if fq['is_up'] else '*', '*' if fq['is_up'] else f['from'])))
for field, f in zip(fq['filter'], fq['properties'])])),)
elif fq['type'] == 'map':
_keys = fq.copy()
_keys.update(fq['properties'])
params += (('fq', '{!tag=%(id)s}' % fq + urllib.unquote(
utf_quoter('%(lat)s:[%(lat_sw)s TO %(lat_ne)s} AND %(lon)s:[%(lon_sw)s TO %(lon_ne)s}' % _keys))),)
return params
def _create_workflow(self, pig_script, params):
workflow = Workflow.objects.new_workflow(self.user)
workflow.name = OozieApi.WORKFLOW_NAME
workflow.is_history = True
if pig_script.use_hcatalog:
workflow.add_parameter("oozie.action.sharelib.for.pig", "pig,hcatalog")
workflow.save()
Workflow.objects.initialize(workflow, self.fs)
script_path = workflow.deployment_dir + "/script.pig"
if self.fs: # For testing, difficult to mock
self.fs.do_as_user(
self.user.username, self.fs.create, script_path, data=smart_str(pig_script.dict["script"])
)
files = []
archives = []
popup_params = json.loads(params)
popup_params_names = [param["name"] for param in popup_params]
pig_params = self._build_parameters(popup_params)
script_params = [param for param in pig_script.dict["parameters"] if param["name"] not in popup_params_names]
pig_params += self._build_parameters(script_params)
job_properties = [
{"name": prop["name"], "value": prop["value"]} for prop in pig_script.dict["hadoopProperties"]
]
for resource in pig_script.dict["resources"]:
if resource["type"] == "file":
files.append(resource["value"])
if resource["type"] == "archive":
archives.append({"dummy": "", "name": resource["value"]})
action = Pig.objects.create(
name="pig",
script_path=script_path,
workflow=workflow,
node_type="pig",
params=json.dumps(pig_params),
files=json.dumps(files),
archives=json.dumps(archives),
job_properties=json.dumps(job_properties),
)
if pig_script.use_hcatalog and self.oozie_api.security_enabled:
action.credentials.append([{"name": "hcat", "value": True}])
action.save()
if pig_script.use_hbase and self.oozie_api.security_enabled:
action.credentials.append([{"name": "hbase", "value": True}])
action.save()
action.add_node(workflow.end)
start_link = workflow.start.get_link()
start_link.child = action
start_link.save()
return workflow
def get_logs(request):
response = {'status': -1}
notebook = json.loads(request.POST.get('notebook', '{}'))
snippet = json.loads(request.POST.get('snippet', '{}'))
startFrom = request.POST.get('from')
startFrom = int(startFrom) if startFrom else None
size = request.POST.get('size')
size = int(size) if size else None
db = get_api(request, snippet)
full_log = smart_str(request.POST.get('full_log', ''))
logs = db.get_log(notebook, snippet, startFrom=startFrom, size=size)
full_log += logs
jobs = db.get_jobs(notebook, snippet, full_log)
response['logs'] = logs.strip()
response['progress'] = min(db.progress(snippet, full_log), 99) if snippet['status'] != 'available' and snippet['status'] != 'success' else 100
response['jobs'] = jobs
response['isFullLogs'] = snippet.get('interface') == 'oozie'
response['status'] = 0
return JsonResponse(response)
def _create_file(self, deployment_dir, file_name, data, do_as=False):
file_path = self.fs.join(deployment_dir, file_name)
if do_as:
self.fs.do_as_user(self.user, self.fs.create, file_path, overwrite=True, permission=0644, data=smart_str(data))
else:
self.fs.create(file_path, overwrite=True, permission=0644, data=smart_str(data))
LOG.debug("Created/Updated %s" % (file_path,))
def _submit_workflow_helper(request, workflow, submit_action):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
cluster = json.loads(request.POST.get('cluster', '{}'))
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
mapping['send_email'] = request.POST.get('email_checkbox') == 'on'
if '/submit_single_action/' in submit_action:
mapping['submit_single_action'] = True
if 'altus' in cluster.get('type', ''):
mapping['cluster'] = cluster.get('id')
try:
job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
except Exception, e:
raise PopupException(_('Workflow submission failed'), detail=smart_str(e), error_code=200)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'workflow'}, safe=False)
else:
request.info(_('Workflow submitted'))
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def import_documents(request):
if request.FILES.get("documents"):
documents = request.FILES["documents"].read()
else:
documents = json.loads(request.POST.get("documents"))
documents = json.loads(documents)
docs = []
for doc in documents:
if not request.user.is_superuser:
doc["fields"]["owner"] = [request.user.username]
owner = doc["fields"]["owner"][0]
doc["fields"]["tags"] = []
if Document2.objects.filter(uuid=doc["fields"]["uuid"], owner__username=owner).exists():
doc["pk"] = Document2.objects.get(uuid=doc["fields"]["uuid"], owner__username=owner).pk
else:
doc["pk"] = None
docs.append(doc)
f = tempfile.NamedTemporaryFile(mode="w+", suffix=".json")
f.write(json.dumps(docs))
f.flush()
stdout = StringIO.StringIO()
try:
management.call_command("loaddata", f.name, stdout=stdout)
except Exception, e:
return JsonResponse({"message": smart_str(e)})
def import_documents(request):
if request.FILES.get('documents'):
documents = request.FILES['documents'].read()
else:
documents = json.loads(request.POST.get('documents'))
documents = json.loads(documents)
docs = []
uuids_map = dict((doc['fields']['uuid'], None) for doc in documents)
for doc in documents:
# If doc is not owned by current user, make a copy of the document with current user as owner
if doc['fields']['owner'][0] != request.user.username:
doc = _copy_document_with_owner(doc, request.user, uuids_map)
else: # Update existing doc or create new
doc = _create_or_update_document_with_owner(doc, request.user, uuids_map)
# Set last modified date to now
doc['fields']['last_modified'] = datetime.now().replace(microsecond=0).isoformat()
docs.append(doc)
f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
f.write(json.dumps(docs))
f.flush()
stdout = StringIO.StringIO()
try:
management.call_command('loaddata', f.name, stdout=stdout)
except Exception, e:
return JsonResponse({'message': smart_str(e)})
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def search_entities(request):
"""
For displaying results.
"""
api = NavigatorApi(request.user)
query_s = json.loads(request.POST.get('query_s', ''))
query_s = smart_str(query_s)
offset = request.POST.get('offset', 0)
limit = request.POST.get('limit', 100)
sources = json.loads(request.POST.get('sources')) or []
query_s = query_s.strip() or '*'
entities = api.search_entities(query_s, limit=limit, offset=offset, sources=sources)
_augment_highlighting(query_s, entities)
response = {
'entities': entities,
'count': len(entities),
'offset': offset,
'limit': limit,
'query_s': query_s,
'status': 0
}
return JsonResponse(response)
def import_documents(request):
if request.FILES.get('documents'):
documents = request.FILES['documents'].read()
else:
documents = json.loads(request.POST.get('documents'))
documents = json.loads(documents)
docs = []
for doc in documents:
if not request.user.is_superuser:
doc['fields']['owner'] = [request.user.username]
owner = doc['fields']['owner'][0]
doc['fields']['tags'] = []
# TODO: Check if this should be replaced by get_by_uuid
if Document2.objects.filter(uuid=doc['fields']['uuid'], owner__username=owner).exists():
doc['pk'] = Document2.objects.get(uuid=doc['fields']['uuid'], owner__username=owner).pk
else:
doc['pk'] = None
docs.append(doc)
f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
f.write(json.dumps(docs))
f.flush()
stdout = StringIO.StringIO()
try:
management.call_command('loaddata', f.name, stdout=stdout)
except Exception, e:
return JsonResponse({'message': smart_str(e)})
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
mapping[application_class.get_application_path_key()] = application_path
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def _fixup(self):
"""
Fixup fields:
- expand actions
- time fields are struct_time
- run is integer
- configuration dict
- log
- definition
"""
# TODO(bc) Can we get the log and definition lazily?
if self.startTime:
self.startTime = parse_timestamp(self.startTime)
if self.endTime:
self.endTime = parse_timestamp(self.endTime)
if self.createdTime:
self.createdTime = parse_timestamp(self.createdTime)
if self.lastModTime:
self.lastModTime = parse_timestamp(self.lastModTime)
self.run = int(self.run)
self.actions = [ Action(act_dict) for act_dict in self.actions ]
if self.conf is not None:
xml = StringIO(i18n.smart_str(self.conf))
self.conf_dict = hadoop.confparse.ConfParse(xml)
else:
self.conf_dict = { }
def _copy_files(self, deployment_dir, oozie_xml):
"""
Copy XML and the jar_path files from Java or MR actions to the deployment directory.
This should run as the workflow user.
"""
xml_path = self.fs.join(deployment_dir, self.job.get_application_filename())
self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml))
LOG.debug("Created %s" % (xml_path,))
# List jar files
files = []
lib_path = self.fs.join(deployment_dir, "lib")
if hasattr(self.job, "node_list"):
for node in self.job.node_list:
if hasattr(node, "jar_path") and not node.jar_path.startswith(lib_path):
files.append(node.jar_path)
# Copy the jar files to the workspace lib
if files:
for jar_file in files:
LOG.debug("Updating %s" % jar_file)
jar_lib_path = self.fs.join(lib_path, self.fs.basename(jar_file))
if self.fs.exists(jar_lib_path):
self.fs.remove(jar_lib_path, skip_trash=True)
self.fs.copyfile(jar_file, jar_lib_path)
def export_documents(request):
if request.GET.get("documents"):
selection = json.loads(request.GET.get("documents"))
else:
selection = json.loads(request.POST.get("documents"))
# Only export documents the user has permissions to read
docs = (
Document2.objects.documents(user=request.user, perms="both", include_history=True, include_trashed=True)
.filter(id__in=selection)
.order_by("-id")
)
# Add any dependencies to the set of exported documents
export_doc_set = _get_dependencies(docs)
# For directories, add any children docs to the set of exported documents
export_doc_set.update(_get_dependencies(docs, deps_mode=False))
# Get PKs of documents to export
doc_ids = [doc.pk for doc in export_doc_set]
num_docs = len(doc_ids)
filename = "hue-documents-%s-(%s)" % (datetime.today().strftime("%Y-%m-%d"), num_docs)
f = StringIO.StringIO()
if doc_ids:
doc_ids = ",".join(map(str, doc_ids))
management.call_command(
"dumpdata",
"desktop.Document2",
primary_keys=doc_ids,
indent=2,
use_natural_keys=True,
verbosity=2,
stdout=f,
)
if request.GET.get("format") == "json":
return JsonResponse(f.getvalue(), safe=False)
elif request.GET.get("format") == "zip":
zfile = zipfile.ZipFile(f, "w")
zfile.writestr("hue.json", f.getvalue())
for doc in docs:
if doc.type == "notebook":
try:
from spark.models import Notebook
zfile.writestr(
"notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str())
)
except Exception, e:
LOG.exception(e)
zfile.close()
response = HttpResponse(content_type="application/zip")
response["Content-Length"] = len(f.getvalue())
response["Content-Disposition"] = 'attachment; filename="%s".zip' % filename
response.write(f.getvalue())
return response
def _sync_definition(self, deployment_dir, mapping):
""" This is helper function for 'Sync Workflow' functionality in a Coordinator.
It copies updated workflow changes into HDFS """
self._create_file(deployment_dir, self.job.XML_FILE_NAME, self.job.to_xml(mapping=mapping), do_as=True)
data_properties = smart_str('\n'.join(['%s=%s' % (key, val) for key, val in mapping.iteritems()]))
self._create_file(deployment_dir, 'job.properties', data_properties, do_as=True)
def force_dict_to_strings(dictionary):
if not dictionary:
return dictionary
new_dict = {}
for k in dictionary:
new_key = smart_str(k)
if isinstance(dictionary[k], basestring):
# Strings should not be unicode.
new_dict[new_key] = smart_str(dictionary[k])
elif isinstance(dictionary[k], dict):
# Recursively force dicts to strings.
new_dict[new_key] = force_dict_to_strings(dictionary[k])
else:
# Normal objects, or other literals, should not be converted.
new_dict[new_key] = dictionary[k]
return new_dict
def open_old_workflow(request):
doc_id = request.GET.get('workflow')
workflow = Document.objects.get(id=doc_id).content_object.get_full_node()
try:
_workflow = import_workflow_from_hue_3_7(workflow)
return _edit_workflow(request, None, _workflow)
except Exception, e:
LOG.warn('Could not open old worklow: %s' % smart_str(e))
return old_edit_workflow(request, workflow=workflow.id)
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def is_valid(self):
assert self._is_bound
r = True
# Explicitly iterate through all of them; we don't want
# to abort early, since we want each form's is_valid to be run.
for f in self._forms.values():
if not f.is_valid():
LOG.error(smart_str(f.errors))
r = False
return r
def _copy_files(self, deployment_dir, oozie_xml, oozie_properties):
"""
Copy XML and the jar_path files from Java or MR actions to the deployment directory.
This should run as the workflow user.
"""
xml_path = self.fs.join(deployment_dir, self.job.XML_FILE_NAME)
self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml))
LOG.debug("Created %s" % (xml_path,))
properties_path = self.fs.join(deployment_dir, "job.properties")
self.fs.create(
properties_path,
overwrite=True,
permission=0644,
data=smart_str("\n".join(["%s=%s" % (key, val) for key, val in oozie_properties.iteritems()])),
)
LOG.debug("Created %s" % (properties_path,))
# List jar files
files = []
lib_path = self.fs.join(deployment_dir, "lib")
if hasattr(self.job, "nodes"):
for node in self.job.nodes:
jar_path = node.data["properties"].get("jar_path")
if jar_path:
if not jar_path.startswith("/"): # If workspace relative path
jar_path = self.fs.join(self.job.deployment_dir, jar_path)
if not jar_path.startswith(lib_path): # If not already in lib
files.append(jar_path)
# Copy the jar files to the workspace lib
if files:
for jar_file in files:
LOG.debug("Updating %s" % jar_file)
jar_lib_path = self.fs.join(lib_path, self.fs.basename(jar_file))
# Refresh if needed
if self.fs.exists(jar_lib_path) and self.fs.exists(jar_file):
stat_src = self.fs.stats(jar_file)
stat_dest = self.fs.stats(jar_lib_path)
if stat_src.fileId != stat_dest.fileId:
self.fs.remove(jar_lib_path, skip_trash=True)
self.fs.copyfile(jar_file, jar_lib_path)
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
def _create_workflow(self, pig_script, params):
workflow = Workflow.objects.new_workflow(self.user)
workflow.name = OozieApi.WORKFLOW_NAME
workflow.is_history = True
if pig_script.use_hcatalog:
workflow.add_parameter("oozie.action.sharelib.for.pig", "pig,hcatalog")
workflow.save()
Workflow.objects.initialize(workflow, self.fs)
script_path = workflow.deployment_dir + '/script.pig'
if self.fs: # For testing, difficult to mock
self.fs.do_as_user(self.user.username, self.fs.create, script_path, data=smart_str(pig_script.dict['script']))
files = []
archives = []
popup_params = json.loads(params)
popup_params_names = [param['name'] for param in popup_params]
pig_params = self._build_parameters(popup_params)
script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names]
pig_params += self._build_parameters(script_params)
job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']]
for resource in pig_script.dict['resources']:
if resource['type'] == 'file':
files.append(resource['value'])
if resource['type'] == 'archive':
archives.append({"dummy": "", "name": resource['value']})
action = Pig.objects.create(
name='pig',
script_path=script_path,
workflow=workflow,
node_type='pig',
params=json.dumps(pig_params),
files=json.dumps(files),
archives=json.dumps(archives),
job_properties=json.dumps(job_properties)
)
if pig_script.use_hcatalog and get_oozie(self.user).security_enabled:
action.credentials = [{'name': 'hcat', 'value': True}]
action.save()
action.add_node(workflow.end)
start_link = workflow.start.get_link()
start_link.child = action
start_link.save()
return workflow
def _submit_workflow(user, fs, workflow, mapping):
try:
submission = Submission(user, workflow, fs, mapping)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'urlopen error' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
def invalidate_tables(self, database, tables):
handle = None
for table in tables:
try:
hql = "INVALIDATE METADATA `%s`.`%s`" % (database, table)
query = hql_query(hql, database, query_type=QUERY_TYPES[1])
handle = self.execute_and_wait(query, timeout_sec=10.0)
except Exception, e:
LOG.warn("Refresh tables cache out of sync: %s" % smart_str(e))
finally:
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
History.objects.create_from_submission(submission)
return job_id
except RestException as ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.error(smart_str(detail))
raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
def test_connection(self, throw_exception=True):
try:
self.connect()
return True
except Exception as e:
message = force_unicode(smart_str(e))
if throw_exception:
if 'Access denied' in message:
raise AuthenticationRequired()
raise
else:
return False
finally:
self.close()
def decorator(*args, **kwargs):
try:
return func(*args, **kwargs)
except AuthenticationRequired as e:
raise e
except Exception as e:
message = force_unicode(smart_str(e))
if 'error occurred while trying to connect to the Java server' in message:
raise QueryError, _('%s: is the DB Proxy server running?'
) % message, sys.exc_info()[2]
elif 'Access denied' in message:
raise AuthenticationRequired, '', sys.exc_info()[2]
else:
raise QueryError, message, sys.exc_info()[2]
def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''):
for c in counts:
fq_fields = (fields if fields else []) + [c['field']]
fq_values = (values if values else []) + [smart_str(c['value'])]
if 'pivot' in c:
_augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values)
fq_filter = selected_values.get(facet_id, [])
_selected_values = [f['value'] for f in fq_filter]
c['selected'] = fq_values in _selected_values
c['exclude'] = False
c['fq_fields'] = fq_fields
c['fq_values'] = fq_values
def _submit_workflow(user, fs, jt, workflow, mapping):
try:
submission = Submission(user, workflow, fs, jt, mapping)
job_id = submission.run()
workflow.document.add_to_history(submission.user, {'properties': submission.properties, 'oozie_id': submission.oozie_id})
return job_id
except RestException, ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception('Error submitting workflow: %s' % smart_str(detail))
raise PopupException(_("Error submitting workflow %s: %s") % (workflow, detail))
def process_response(self, request, response):
if not _has_tidylib or not self._is_html(request, response):
return response
html, errors = tidylib.tidy_document(response.content,
self._options,
keep_doc=True)
if not errors:
return response
# Filter out what we care about
err_list = errors.rstrip().split('\n')
err_list = self._filter_warnings(err_list)
if not err_list:
return response
try:
fn = urlresolvers.resolve(request.path)[0]
fn_name = '%s.%s' % (fn.__module__, fn.__name__)
except:
LOG.exception('failed to resolve url')
fn_name = '<unresolved_url>'
# Write the two versions of html out for offline debugging
filename = os.path.join(self._outdir, fn_name)
result = "HTML tidy result: %s [%s]:" \
"\n\t%s" \
"\nPlease see %s.orig %s.tidy\n-------" % \
(request.path, fn_name, '\n\t'.join(err_list), filename, filename)
file(filename + '.orig', 'w').write(i18n.smart_str(response.content))
file(filename + '.tidy', 'w').write(i18n.smart_str(html))
file(filename + '.info', 'w').write(i18n.smart_str(result))
self._logger.error(result)
return response
def export_documents(request):
if request.GET.get('documents'):
selection = json.loads(request.GET.get('documents'))
else:
selection = json.loads(request.POST.get('documents'))
# Only export documents the user has permissions to read
docs = Document2.objects.documents(user=request.user, perms='both', include_history=True, include_trashed=True).\
filter(id__in=selection).order_by('-id')
# Add any dependencies to the set of exported documents
export_doc_set = _get_dependencies(docs)
# For directories, add any children docs to the set of exported documents
export_doc_set.update(_get_dependencies(docs, deps_mode=False))
# Get PKs of documents to export
doc_ids = [doc.pk for doc in export_doc_set]
num_docs = len(doc_ids)
if len(selection) == 1 and num_docs >= len(selection) and docs[0].name:
filename = docs[0].name
else:
filename = 'hue-documents-%s-(%s)' % (datetime.today().strftime('%Y-%m-%d'), num_docs)
f = StringIO.StringIO()
if doc_ids:
doc_ids = ','.join(map(str, doc_ids))
management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_foreign_keys=True, verbosity=2, stdout=f)
if request.GET.get('format') == 'json':
return JsonResponse(f.getvalue(), safe=False)
elif request.GET.get('format') == 'zip':
zfile = zipfile.ZipFile(f, 'w')
zfile.writestr("hue.json", f.getvalue())
for doc in docs:
if doc.type == 'notebook':
try:
from spark.models import Notebook
zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()))
except Exception, e:
LOG.exception(e)
zfile.close()
response = HttpResponse(content_type="application/zip")
response["Content-Length"] = len(f.getvalue())
response['Content-Disposition'] = 'attachment; filename="%s".zip' % filename
response.write(f.getvalue())
return response
def _create_workflow(self, pig_script, params):
workflow = Workflow.objects.new_workflow(self.user)
workflow.name = OozieApi.WORKFLOW_NAME
workflow.is_history = True
if pig_script.use_hcatalog:
workflow.add_parameter("oozie.action.sharelib.for.pig", "pig,hcatalog")
workflow.save()
Workflow.objects.initialize(workflow, self.fs)
script_path = workflow.deployment_dir + '/script.pig'
if self.fs: # For testing, difficult to mock
self.fs.do_as_user(self.user.username, self.fs.create, script_path, data=smart_str(pig_script.dict['script']))
files = []
archives = []
popup_params = json.loads(params)
popup_params_names = [param['name'] for param in popup_params]
pig_params = self._build_parameters(popup_params)
script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names]
pig_params += self._build_parameters(script_params)
job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']]
for resource in pig_script.dict['resources']:
if resource['type'] == 'file':
files.append(resource['value'])
if resource['type'] == 'archive':
archives.append({"dummy": "", "name": resource['value']})
action = Pig.objects.create(
name='pig',
script_path=script_path,
workflow=workflow,
node_type='pig',
params=json.dumps(pig_params),
files=json.dumps(files),
archives=json.dumps(archives),
job_properties=json.dumps(job_properties),
)
action.add_node(workflow.end)
start_link = workflow.start.get_link()
start_link.child = action
start_link.save()
return workflow
def invalidate_tables(self, database, tables):
handle = None
for table in tables:
try:
hql = "INVALIDATE METADATA `%s`.`%s`" % (
database,
table,
)
query = hql_query(hql, database, query_type=QUERY_TYPES[1])
handle = self.execute_and_wait(query, timeout_sec=10.0)
except Exception, e:
LOG.warn('Refresh tables cache out of sync: %s' % smart_str(e))
finally:
def get_logs(request):
response = {'status': -1}
notebook = json.loads(request.POST.get('notebook', '{}'))
snippet = json.loads(request.POST.get('snippet', '{}'))
startFrom = request.POST.get('from')
startFrom = int(startFrom) if startFrom else None
size = request.POST.get('size')
size = int(size) if size else None
full_log = smart_str(request.POST.get('full_log', ''))
db = get_api(request, snippet)
with opentracing.tracer.start_span('notebook-get_logs') as span:
logs = smart_str(
db.get_log(notebook, snippet, startFrom=startFrom, size=size))
span.set_tag('user-id', request.user.username)
span.set_tag(
'query-id', snippet['result']['handle']['guid']
if snippet['result'].get('handle')
and snippet['result']['handle'].get('guid') else None)
full_log += logs
jobs = db.get_jobs(notebook, snippet, full_log)
response['logs'] = logs.strip()
response['progress'] = min(
db.progress(notebook, snippet, logs=full_log),
99) if snippet['status'] != 'available' and snippet[
'status'] != 'success' else 100
response['jobs'] = jobs
response['isFullLogs'] = db.get_log_is_full_log(notebook, snippet)
response['status'] = 0
return JsonResponse(response)
def query_error_handler(func):
def decorator(*args, **kwargs):
try:
return func(*args, **kwargs)
except AuthenticationRequired, e:
raise e
except Exception, e:
message = force_unicode(smart_str(e))
if 'error occurred while trying to connect to the Java server' in message:
raise QueryError(
_('%s: is the DB Proxy server running?') % message)
elif 'Access denied' in message:
raise AuthenticationRequired()
else:
raise QueryError(message)
def _fixup(self):
"""
Fixup:
- time fields as struct_time
- config dict
"""
super(BundleAction, self)._fixup()
self.type = 'coord-action'
self.name = self.coordJobName
if self.conf:
xml = StringIO(i18n.smart_str(self.conf))
self.conf_dict = hadoop.confparse.ConfParse(xml)
else:
self.conf_dict = {}
def download_log_view(request):
"""
Zip up the log buffer and then return as a file attachment.
"""
if not is_admin(request.user):
return HttpResponse(_("You must be a superuser."))
l = logging.getLogger()
for h in l.handlers:
if isinstance(h, desktop.log.log_buffer.FixedBufferHandler):
try:
# We want to avoid doing a '\n'.join of the entire log in memory
# in case it is rather big. So we write it to a file line by line
# and pass that file to zipfile, which might follow a more efficient path.
tmp = tempfile.NamedTemporaryFile()
log_tmp = tempfile.NamedTemporaryFile(
"w+t"
) if sys.version_info[0] == 2 else tempfile.NamedTemporaryFile(
"w+t", encoding='utf-8')
for l in h.buf:
log_tmp.write(smart_str(l, errors='replace') + '\n')
# This is not just for show - w/out flush, we often get truncated logs
log_tmp.flush()
t = time.time()
zip = zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED)
zip.write(log_tmp.name, "hue-logs/hue-%s.log" % t)
zip.close()
length = tmp.tell()
# if we don't seek to start of file, no bytes will be written
tmp.seek(0)
wrapper = FileWrapper(tmp)
response = HttpResponse(wrapper,
content_type="application/zip")
response[
'Content-Disposition'] = 'attachment; filename=hue-logs-%s.zip' % t
response['Content-Length'] = length
return response
except Exception as e:
LOG.exception("Couldn't construct zip file to write logs")
return log_view(request)
return render_to_response(
"logs.mako",
dict(log=[_("No logs found.")],
is_embeddable=request.GET.get('is_embeddable', False)))
def import_documents(request):
if request.FILES.get('documents'):
documents = request.FILES['documents'].read()
else:
documents = json.loads(request.POST.get('documents'))
documents = json.loads(documents)
docs = []
uuids_map = dict((doc['fields']['uuid'], None) for doc in documents)
for doc in documents:
# Remove any deprecated fields
if 'tags' in doc['fields']:
doc['fields'].pop('tags')
# If doc is not owned by current user, make a copy of the document with current user as owner
if doc['fields']['owner'][0] != request.user.username:
doc = _copy_document_with_owner(doc, request.user, uuids_map)
else: # Update existing doc or create new
doc = _create_or_update_document_with_owner(
doc, request.user, uuids_map)
# If the doc contains any history dependencies, ignore them
# NOTE: this assumes that each dependency is exported as an array using the natural PK [uuid, version, is_history]
deps_minus_history = [
dep for dep in doc.get('dependencies', [])
if len(dep) >= 3 and not dep[2]
]
doc['dependencies'] = deps_minus_history
# Set last modified date to now
doc['fields']['last_modified'] = datetime.now().replace(
microsecond=0).isoformat()
docs.append(doc)
f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
f.write(json.dumps(docs))
f.flush()
stdout = StringIO.StringIO()
try:
management.call_command('loaddata', f.name, stdout=stdout)
except Exception, e:
return JsonResponse({'message': smart_str(e)})
def _submit_workflow_helper(request, workflow, submit_action):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
cluster = json.loads(request.POST.get('cluster', '{}'))
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
mapping['send_email'] = request.POST.get('email_checkbox') == 'on'
if '/submit_single_action/' in submit_action:
mapping['submit_single_action'] = True
try:
job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
except Exception as e:
raise PopupException(_('Workflow submission failed'), detail=smart_str(e), error_code=200)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'workflow'}, safe=False)
else:
request.info(_('Workflow submitted'))
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
cluster_json = request.GET.get('cluster', '{}')
parameters = workflow and workflow.find_all_parameters() or []
initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
params_form = ParametersFormSet(initial=initial_params)
return render(
'/scheduler/submit_job_popup.mako',
request, {
'params_form': params_form,
'name': workflow.name,
'action': submit_action,
'show_dryrun': True,
'email_id': request.user.email,
'is_oozie_mail_enabled': _is_oozie_mail_enabled(request.user),
'return_json': request.GET.get('format') == 'json',
'cluster_json': cluster_json
},
force_template=True
)
def export_documents(request):
if request.GET.get('documents'):
selection = json.loads(request.GET.get('documents'))
else:
selection = json.loads(request.POST.get('documents'))
# If non admin, only export documents the user owns
docs = Document2.objects
if not request.user.is_superuser:
docs = docs.filter(owner=request.user)
docs = docs.filter(id__in=selection).order_by('-id')
doc_ids = docs.values_list('id', flat=True)
f = StringIO.StringIO()
if doc_ids:
doc_ids = ','.join(map(str, doc_ids))
management.call_command('dumpdata',
'desktop.Document2',
primary_keys=doc_ids,
indent=2,
use_natural_keys=True,
verbosity=2,
stdout=f)
if request.GET.get('format') == 'json':
return JsonResponse(f.getvalue(), safe=False)
elif request.GET.get('format') == 'zip':
zfile = zipfile.ZipFile(f, 'w')
zfile.writestr("hue.json", f.getvalue())
for doc in docs:
if doc.type == 'notebook':
try:
from spark.models import Notebook
zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id),
smart_str(Notebook(document=doc).get_str()))
except Exception, e:
print e
LOG.exception(e)
zfile.close()
response = HttpResponse(content_type="application/zip")
response["Content-Length"] = len(f.getvalue())
response[
'Content-Disposition'] = 'attachment; filename="hue-documents.zip"'
response.write(f.getvalue())
return response
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
mapping[application_class.get_application_path_key()] = os.path.dirname(application_path)
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException as ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
else:
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
initial_params = ParameterForm.get_initial_params(parameters)
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor/submit_job_popup.mako', request, {
'params_form': params_form,
'name': _('Job'),
'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}),
'show_dryrun': os.path.basename(application_path) != 'bundle.xml',
'return_json': request.GET.get('format') == 'json'
}, force_template=True).content
return JsonResponse(popup, safe=False)
def kill(self, app_id):
data = {'state': 'KILLED'}
token = None
# Tokens are managed within the kill method but should be moved out when not alpha anymore or we support submitting an app.
if self.security_enabled and False:
full_token = self.delegation_token()
if 'token' not in full_token:
raise PopupException(_('YARN did not return any token field.'), detail=smart_str(full_token))
data['X-Hadoop-Delegation-Token'] = token = full_token.pop('token')
LOG.debug('Received delegation token %s' % full_token)
try:
params = self._get_params()
return self._execute(self._root.put, 'cluster/apps/%(app_id)s/state' % {'app_id': app_id}, params=params, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
finally:
if token:
self.cancel_token(token)
def _submit_workflow_helper(request, workflow, submit_action):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
try:
job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
except Exception, e:
raise PopupException(_('Workflow submission failed'), detail=smart_str(e))
request.info(_('Workflow submitted'))
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def query_and_fetch(db, statement, n=None):
data = None
try:
db.connect()
curs = db.cursor()
try:
if curs.execute(statement):
data = curs.fetchmany(n)
meta = curs.description
return data, meta
finally:
curs.close()
except Exception, e:
message = force_unicode(smart_str(e))
if 'Access denied' in message:
raise AuthenticationRequired()
raise
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator')
doc = None
if coordinator_id:
doc = Document2.objects.get(id=coordinator_id)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def submit_workflow(request, doc_id):
workflow = Workflow(document=Document2.objects.get(id=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
try:
job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping)
except Exception, e:
raise PopupException(_('Workflow submission failed'), detail=smart_str(e))
request.info(_('Workflow submitted'))
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
def get_indexes(self, include_cores=False):
indexes = []
try:
if self.is_solr_cloud_mode():
collections = self.api.collections2()
for name in collections:
indexes.append({
'name': name,
'type': 'collection',
'collections': []
})
if self.is_solr_cloud_mode():
try:
if self.is_solr_six_or_more():
solr_aliases = self.api.list_aliases()
else:
solr_aliases = self.api.aliases()
for name in solr_aliases:
collections = solr_aliases[name].split()
indexes.append({
'name': name,
'type': 'alias',
'collections': collections
})
except Exception:
LOG.exception('Aliases could not be retrieved')
if not self.is_solr_cloud_mode() or include_cores:
solr_cores = self.api.cores()
for name in solr_cores:
indexes.append({
'name': name,
'type': 'core',
'collections': []
})
except Exception as e:
msg = _('Solr server could not be contacted properly: %s') % e
LOG.warn(msg)
raise PopupException(msg, detail=smart_str(e))
return sorted(indexes, key=lambda index: index['name'])
def _fixup(self):
"""
Fixup:
- time fields as struct_time
- config dict
"""
super(BundleAction, self)._fixup()
self.type = 'coord-action'
self.name = self.coordJobName
if self.conf:
conf_data = i18n.smart_str(self.conf)
if not isinstance(conf_data, bytes):
conf_data = conf_data.encode('utf-8')
xml = string_io(conf_data)
self.conf_dict = hadoop.confparse.ConfParse(xml)
else:
self.conf_dict = {}
def force_list_to_strings(lst):
if not lst:
return lst
new_list = []
for item in lst:
if isinstance(item, string_types):
# Strings should not be unicode.
new_list.append(smart_str(item))
elif isinstance(item, dict):
# Recursively force dicts to strings.
new_list.append(force_dict_to_strings(item))
elif isinstance(item, list):
new_list.append(force_list_to_strings(item))
else:
# Normal objects, or other literals, should not be converted.
new_list.append(item)
return new_list
def _submit_workflow_helper(request, workflow, submit_action):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
cluster = json.loads(request.POST.get('cluster', '{}'))
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value'])
for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
mapping['send_email'] = request.POST.get('email_checkbox') == 'on'
if '/submit_single_action/' in submit_action:
mapping['submit_single_action'] = True
if 'altus' in cluster.get('type', ''):
mapping['cluster'] = cluster.get('id')
try:
job_id = _submit_workflow(request.user, request.fs, request.jt,
workflow, mapping)
except Exception, e:
raise PopupException(_('Workflow submission failed'),
detail=smart_str(e),
error_code=200)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse(
{
'status': 0,
'job_id': job_id,
'type': 'workflow'
},
safe=False)
else:
request.info(_('Workflow submitted'))
return redirect(
reverse('oozie:list_oozie_workflow',
kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' %
params_form.errors))
def invalidate_tables(self, database, tables=None):
handle = None
try:
if tables:
for table in tables:
hql = "INVALIDATE METADATA `%s`.`%s`" % (
database,
table,
)
print hql
query = hql_query(hql, database, query_type=QUERY_TYPES[1])
handle = self.execute_and_wait(query, timeout_sec=10.0)
else: # call INVALIDATE on entire DB to pick up newly created tables
hql = "INVALIDATE METADATA `%s`" % database
print hql
query = hql_query(hql, database, query_type=QUERY_TYPES[1])
handle = self.execute_and_wait(query, timeout_sec=10.0)
except Exception, e:
LOG.warn('Refresh tables cache out of sync: %s' % smart_str(e))
def _fixup(self):
"""
Fixup:
- time fields as struct_time
- config dict
"""
super(CoordinatorAction, self)._fixup()
if self.createdTime:
self.createdTime = parse_timestamp(self.createdTime)
if self.nominalTime:
self.nominalTime = parse_timestamp(self.nominalTime)
if self.lastModifiedTime:
self.lastModifiedTime = parse_timestamp(self.lastModifiedTime)
if self.runConf:
xml = StringIO(i18n.smart_str(self.runConf))
self.conf_dict = hadoop.confparse.ConfParse(xml)
else:
self.conf_dict = {}
def _fixup(self):
"""
Fixup:
- time fields as struct_time
- config dict
"""
super(WorkflowAction, self)._fixup()
if self.startTime:
self.startTime = parse_timestamp(self.startTime)
if self.endTime:
self.endTime = parse_timestamp(self.endTime)
if self.retries:
self.retries = int(self.retries)
if self.conf:
xml = StringIO(i18n.smart_str(self.conf))
self.conf_dict = hadoop.confparse.ConfParse(xml)
else:
self.conf_dict = {}
