def _copy_files(self, deployment_dir, oozie_xml, oozie_properties): """ Copy XML and the jar_path files from Java or MR actions to the deployment directory. This should run as the workflow user. """ xml_path = self.fs.join(deployment_dir, self.job.XML_FILE_NAME) self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml)) LOG.debug("Created %s" % (xml_path,)) properties_path = self.fs.join(deployment_dir, 'job.properties') self.fs.create(properties_path, overwrite=True, permission=0644, data=smart_str('\n'.join(['%s=%s' % (key, val) for key, val in oozie_properties.iteritems()]))) LOG.debug("Created %s" % (properties_path,)) # List jar files files = [] lib_path = self.fs.join(deployment_dir, 'lib') if hasattr(self.job, 'node_list'): for node in self.job.node_list: if hasattr(node, 'jar_path') and not node.jar_path.startswith(lib_path): files.append(node.jar_path) # Copy the jar files to the workspace lib if files: for jar_file in files: LOG.debug("Updating %s" % jar_file) jar_lib_path = self.fs.join(lib_path, self.fs.basename(jar_file)) # Refresh if needed if self.fs.exists(jar_lib_path): stat_src = self.fs.stats(jar_file) stat_dest = self.fs.stats(jar_lib_path) if stat_src.fileId != stat_dest.fileId: self.fs.remove(jar_lib_path, skip_trash=True) self.fs.copyfile(jar_file, jar_lib_path)
def copy_configs(fields, unique_key_field, df): # Create temporary copy of solr configs tmp_path = tempfile.mkdtemp() solr_config_path = os.path.join(tmp_path, os.path.basename(conf.CONFIG_TEMPLATE_PATH.get())) shutil.copytree(conf.CONFIG_TEMPLATE_PATH.get(), solr_config_path) if fields or unique_key_field: # Get complete schema.xml with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/schema.xml')) as f: schemaxml = SchemaXml(f.read()) schemaxml.uniqueKeyField(unique_key_field) schemaxml.fields(fields) # Write complete schema.xml to copy with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f: f.write(smart_str(schemaxml.xml)) if df: # Get complete solrconfig.xml with open(os.path.join(conf.CONFIG_TEMPLATE_PATH.get(), 'conf/solrconfig.xml')) as f: solrconfigxml = SolrConfigXml(f.read()) solrconfigxml.defaultField(df) # Write complete solrconfig.xml to copy with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f: f.write(smart_str(solrconfigxml.xml)) return tmp_path, solr_config_path
def copy_configs(fields, unique_key_field, df, solr_cloud_mode=True): # Create temporary copy of solr configs tmp_path = tempfile.mkdtemp() config_template_path = get_config_template_path(solr_cloud_mode) solr_config_path = os.path.join(tmp_path, 'solr_configs') shutil.copytree(config_template_path, solr_config_path) if fields or unique_key_field: # Get complete schema.xml with open(os.path.join(config_template_path, 'conf/schema.xml')) as f: schemaxml = SchemaXml(f.read()) schemaxml.uniqueKeyField(unique_key_field) schemaxml.fields(fields) # Write complete schema.xml to copy with open(os.path.join(solr_config_path, 'conf/schema.xml'), 'w') as f: f.write(smart_str(schemaxml.xml)) if df: # Get complete solrconfig.xml with open(os.path.join(config_template_path, 'conf/solrconfig.xml')) as f: solrconfigxml = SolrConfigXml(f.read()) solrconfigxml.defaultField(df) # Write complete solrconfig.xml to copy with open(os.path.join(solr_config_path, 'conf/solrconfig.xml'), 'w') as f: f.write(smart_str(solrconfigxml.xml)) return tmp_path, solr_config_path
def process_response(self, request, response): if not _has_tidylib or not self._is_html(request, response): return response html, errors = tidylib.tidy_document(response.content, self._options, keep_doc=True) if not errors: return response # Filter out what we care about err_list = errors.rstrip().split("\n") err_list = self._filter_warnings(err_list) if not err_list: return response try: fn = urlresolvers.resolve(request.path)[0] fn_name = "%s.%s" % (fn.__module__, fn.__name__) except: fn_name = "<unresolved_url>" # Write the two versions of html out for offline debugging filename = os.path.join(self._outdir, fn_name) result = ( "HTML tidy result: %s [%s]:" "\n\t%s" "\nPlease see %s.orig %s.tidy\n-------" % (request.path, fn_name, "\n\t".join(err_list), filename, filename) ) file(filename + ".orig", "w").write(i18n.smart_str(response.content)) file(filename + ".tidy", "w").write(i18n.smart_str(html)) file(filename + ".info", "w").write(i18n.smart_str(result)) self._logger.error(result) return response
def save_query(request, design_id=None): response = {'status': -1, 'message': ''} if request.method != 'POST': response['message'] = _('A POST request is required.') app_name = get_app_name(request) query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name] design = safe_get_design(request, query_type, design_id) form = QueryForm() api = get_api(request.user) app_names = api.jars() try: form.bind(request.POST) form.query.fields['appName'].choices = ((key, key) for key in app_names) if form.is_valid(): design = save_design(request, form, query_type, design, True) response['design_id'] = design.id response['status'] = 0 else: response['message'] = smart_str(form.query.errors) + smart_str(form.saveform.errors) except RuntimeError, e: response['message'] = str(e)
def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''): for c in counts: fq_fields = (fields + ':' if fields else '') + c['field'] fq_values = (smart_str(values) + ':' if values else '') + smart_str(c['value']) if 'pivot' in c: _augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values) fq_filter = selected_values.get((facet_id, fq_fields, 'field'), []) _selected_values = [f['value'] for f in fq_filter] c['selected'] = fq_values in _selected_values c['exclude'] = False c['fq_fields'] = fq_fields c['fq_values'] = fq_values
def _get_fq(self, collection, query): params = () timeFilter = {} if collection: timeFilter = self._get_range_borders(collection, query) if timeFilter and not timeFilter.get('time_filter_overrides'): params += (('fq', urllib.unquote(utf_quoter('%(field)s:[%(from)s TO %(to)s]' % timeFilter))),) # Merge facets queries on same fields grouped_fqs = groupby(query['fqs'], lambda x: (x['type'], x['field'])) merged_fqs = [] for key, group in grouped_fqs: field_fq = next(group) for fq in group: for f in fq['filter']: field_fq['filter'].append(f) merged_fqs.append(field_fq) for fq in merged_fqs: if fq['type'] == 'field': fields = fq['field'] if type(fq['field']) == list else [fq['field']] # 2D facets support for field in fields: f = [] for _filter in fq['filter']: values = _filter['value'] if type(_filter['value']) == list else [_filter['value']] # 2D facets support if fields.index(field) < len(values): # Lowest common field denominator value = values[fields.index(field)] exclude = '-' if _filter['exclude'] else '' if value is not None and ' ' in smart_str(value): value = smart_str(value).replace('"', '\\"') f.append('%s%s:"%s"' % (exclude, field, value)) else: f.append('%s{!field f=%s}%s' % (exclude, field, value)) _params ='{!tag=%(id)s}' % fq + ' '.join(f) params += (('fq', urllib.unquote(utf_quoter(_params))),) elif fq['type'] == 'range': params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote( utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'], f['to']))) for field, f in zip(fq['filter'], fq['properties'])])),) elif fq['type'] == 'range-up': params += (('fq', '{!tag=%(id)s}' % fq + ' '.join([urllib.unquote( utf_quoter('%s%s:[%s TO %s}' % ('-' if field['exclude'] else '', fq['field'], f['from'] if fq['is_up'] else '*', '*' if fq['is_up'] else f['from']))) for field, f in zip(fq['filter'], fq['properties'])])),) elif fq['type'] == 'map': _keys = fq.copy() _keys.update(fq['properties']) params += (('fq', '{!tag=%(id)s}' % fq + urllib.unquote( utf_quoter('%(lat)s:[%(lat_sw)s TO %(lat_ne)s} AND %(lon)s:[%(lon_sw)s TO %(lon_ne)s}' % _keys))),) return params
def _create_workflow(self, pig_script, params): workflow = Workflow.objects.new_workflow(self.user) workflow.name = OozieApi.WORKFLOW_NAME workflow.is_history = True if pig_script.use_hcatalog: workflow.add_parameter("oozie.action.sharelib.for.pig", "pig,hcatalog") workflow.save() Workflow.objects.initialize(workflow, self.fs) script_path = workflow.deployment_dir + "/script.pig" if self.fs: # For testing, difficult to mock self.fs.do_as_user( self.user.username, self.fs.create, script_path, data=smart_str(pig_script.dict["script"]) ) files = [] archives = [] popup_params = json.loads(params) popup_params_names = [param["name"] for param in popup_params] pig_params = self._build_parameters(popup_params) script_params = [param for param in pig_script.dict["parameters"] if param["name"] not in popup_params_names] pig_params += self._build_parameters(script_params) job_properties = [ {"name": prop["name"], "value": prop["value"]} for prop in pig_script.dict["hadoopProperties"] ] for resource in pig_script.dict["resources"]: if resource["type"] == "file": files.append(resource["value"]) if resource["type"] == "archive": archives.append({"dummy": "", "name": resource["value"]}) action = Pig.objects.create( name="pig", script_path=script_path, workflow=workflow, node_type="pig", params=json.dumps(pig_params), files=json.dumps(files), archives=json.dumps(archives), job_properties=json.dumps(job_properties), ) if pig_script.use_hcatalog and self.oozie_api.security_enabled: action.credentials.append([{"name": "hcat", "value": True}]) action.save() if pig_script.use_hbase and self.oozie_api.security_enabled: action.credentials.append([{"name": "hbase", "value": True}]) action.save() action.add_node(workflow.end) start_link = workflow.start.get_link() start_link.child = action start_link.save() return workflow
def get_logs(request): response = {'status': -1} notebook = json.loads(request.POST.get('notebook', '{}')) snippet = json.loads(request.POST.get('snippet', '{}')) startFrom = request.POST.get('from') startFrom = int(startFrom) if startFrom else None size = request.POST.get('size') size = int(size) if size else None db = get_api(request, snippet) full_log = smart_str(request.POST.get('full_log', '')) logs = db.get_log(notebook, snippet, startFrom=startFrom, size=size) full_log += logs jobs = db.get_jobs(notebook, snippet, full_log) response['logs'] = logs.strip() response['progress'] = min(db.progress(snippet, full_log), 99) if snippet['status'] != 'available' and snippet['status'] != 'success' else 100 response['jobs'] = jobs response['isFullLogs'] = snippet.get('interface') == 'oozie' response['status'] = 0 return JsonResponse(response)
def _create_file(self, deployment_dir, file_name, data, do_as=False): file_path = self.fs.join(deployment_dir, file_name) if do_as: self.fs.do_as_user(self.user, self.fs.create, file_path, overwrite=True, permission=0644, data=smart_str(data)) else: self.fs.create(file_path, overwrite=True, permission=0644, data=smart_str(data)) LOG.debug("Created/Updated %s" % (file_path,))
def _submit_workflow_helper(request, workflow, submit_action): ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': cluster = json.loads(request.POST.get('cluster', '{}')) params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' mapping['send_email'] = request.POST.get('email_checkbox') == 'on' if '/submit_single_action/' in submit_action: mapping['submit_single_action'] = True if 'altus' in cluster.get('type', ''): mapping['cluster'] = cluster.get('id') try: job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping) except Exception, e: raise PopupException(_('Workflow submission failed'), detail=smart_str(e), error_code=200) jsonify = request.POST.get('format') == 'json' if jsonify: return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'workflow'}, safe=False) else: request.info(_('Workflow submitted')) return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors))
def import_documents(request): if request.FILES.get("documents"): documents = request.FILES["documents"].read() else: documents = json.loads(request.POST.get("documents")) documents = json.loads(documents) docs = [] for doc in documents: if not request.user.is_superuser: doc["fields"]["owner"] = [request.user.username] owner = doc["fields"]["owner"][0] doc["fields"]["tags"] = [] if Document2.objects.filter(uuid=doc["fields"]["uuid"], owner__username=owner).exists(): doc["pk"] = Document2.objects.get(uuid=doc["fields"]["uuid"], owner__username=owner).pk else: doc["pk"] = None docs.append(doc) f = tempfile.NamedTemporaryFile(mode="w+", suffix=".json") f.write(json.dumps(docs)) f.flush() stdout = StringIO.StringIO() try: management.call_command("loaddata", f.name, stdout=stdout) except Exception, e: return JsonResponse({"message": smart_str(e)})
def import_documents(request): if request.FILES.get('documents'): documents = request.FILES['documents'].read() else: documents = json.loads(request.POST.get('documents')) documents = json.loads(documents) docs = [] uuids_map = dict((doc['fields']['uuid'], None) for doc in documents) for doc in documents: # If doc is not owned by current user, make a copy of the document with current user as owner if doc['fields']['owner'][0] != request.user.username: doc = _copy_document_with_owner(doc, request.user, uuids_map) else: # Update existing doc or create new doc = _create_or_update_document_with_owner(doc, request.user, uuids_map) # Set last modified date to now doc['fields']['last_modified'] = datetime.now().replace(microsecond=0).isoformat() docs.append(doc) f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json') f.write(json.dumps(docs)) f.flush() stdout = StringIO.StringIO() try: management.call_command('loaddata', f.name, stdout=stdout) except Exception, e: return JsonResponse({'message': smart_str(e)})
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) if request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def search_entities(request): """ For displaying results. """ api = NavigatorApi(request.user) query_s = json.loads(request.POST.get('query_s', '')) query_s = smart_str(query_s) offset = request.POST.get('offset', 0) limit = request.POST.get('limit', 100) sources = json.loads(request.POST.get('sources')) or [] query_s = query_s.strip() or '*' entities = api.search_entities(query_s, limit=limit, offset=offset, sources=sources) _augment_highlighting(query_s, entities) response = { 'entities': entities, 'count': len(entities), 'offset': offset, 'limit': limit, 'query_s': query_s, 'status': 0 } return JsonResponse(response)
def import_documents(request): if request.FILES.get('documents'): documents = request.FILES['documents'].read() else: documents = json.loads(request.POST.get('documents')) documents = json.loads(documents) docs = [] for doc in documents: if not request.user.is_superuser: doc['fields']['owner'] = [request.user.username] owner = doc['fields']['owner'][0] doc['fields']['tags'] = [] # TODO: Check if this should be replaced by get_by_uuid if Document2.objects.filter(uuid=doc['fields']['uuid'], owner__username=owner).exists(): doc['pk'] = Document2.objects.get(uuid=doc['fields']['uuid'], owner__username=owner).pk else: doc['pk'] = None docs.append(doc) f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json') f.write(json.dumps(docs)) f.flush() stdout = StringIO.StringIO() try: management.call_command('loaddata', f.name, stdout=stdout) except Exception, e: return JsonResponse({'message': smart_str(e)})
def submit_external_job(request, application_path): ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' application_name = os.path.basename(application_path) application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow() mapping[application_class.get_application_path_key()] = application_path try: submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping) job_id = submission.run(application_path) except RestException, ex: detail = ex._headers.get('oozie-error-message', ex) if 'Max retries exceeded with url' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.exception(smart_str(detail)) raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail) request.info(_('Oozie job submitted')) view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow' return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors))
def _fixup(self): """ Fixup fields: - expand actions - time fields are struct_time - run is integer - configuration dict - log - definition """ # TODO(bc) Can we get the log and definition lazily? if self.startTime: self.startTime = parse_timestamp(self.startTime) if self.endTime: self.endTime = parse_timestamp(self.endTime) if self.createdTime: self.createdTime = parse_timestamp(self.createdTime) if self.lastModTime: self.lastModTime = parse_timestamp(self.lastModTime) self.run = int(self.run) self.actions = [ Action(act_dict) for act_dict in self.actions ] if self.conf is not None: xml = StringIO(i18n.smart_str(self.conf)) self.conf_dict = hadoop.confparse.ConfParse(xml) else: self.conf_dict = { }
def _copy_files(self, deployment_dir, oozie_xml): """ Copy XML and the jar_path files from Java or MR actions to the deployment directory. This should run as the workflow user. """ xml_path = self.fs.join(deployment_dir, self.job.get_application_filename()) self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml)) LOG.debug("Created %s" % (xml_path,)) # List jar files files = [] lib_path = self.fs.join(deployment_dir, "lib") if hasattr(self.job, "node_list"): for node in self.job.node_list: if hasattr(node, "jar_path") and not node.jar_path.startswith(lib_path): files.append(node.jar_path) # Copy the jar files to the workspace lib if files: for jar_file in files: LOG.debug("Updating %s" % jar_file) jar_lib_path = self.fs.join(lib_path, self.fs.basename(jar_file)) if self.fs.exists(jar_lib_path): self.fs.remove(jar_lib_path, skip_trash=True) self.fs.copyfile(jar_file, jar_lib_path)
def export_documents(request): if request.GET.get("documents"): selection = json.loads(request.GET.get("documents")) else: selection = json.loads(request.POST.get("documents")) # Only export documents the user has permissions to read docs = ( Document2.objects.documents(user=request.user, perms="both", include_history=True, include_trashed=True) .filter(id__in=selection) .order_by("-id") ) # Add any dependencies to the set of exported documents export_doc_set = _get_dependencies(docs) # For directories, add any children docs to the set of exported documents export_doc_set.update(_get_dependencies(docs, deps_mode=False)) # Get PKs of documents to export doc_ids = [doc.pk for doc in export_doc_set] num_docs = len(doc_ids) filename = "hue-documents-%s-(%s)" % (datetime.today().strftime("%Y-%m-%d"), num_docs) f = StringIO.StringIO() if doc_ids: doc_ids = ",".join(map(str, doc_ids)) management.call_command( "dumpdata", "desktop.Document2", primary_keys=doc_ids, indent=2, use_natural_keys=True, verbosity=2, stdout=f, ) if request.GET.get("format") == "json": return JsonResponse(f.getvalue(), safe=False) elif request.GET.get("format") == "zip": zfile = zipfile.ZipFile(f, "w") zfile.writestr("hue.json", f.getvalue()) for doc in docs: if doc.type == "notebook": try: from spark.models import Notebook zfile.writestr( "notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str()) ) except Exception, e: LOG.exception(e) zfile.close() response = HttpResponse(content_type="application/zip") response["Content-Length"] = len(f.getvalue()) response["Content-Disposition"] = 'attachment; filename="%s".zip' % filename response.write(f.getvalue()) return response
def _sync_definition(self, deployment_dir, mapping): """ This is helper function for 'Sync Workflow' functionality in a Coordinator. It copies updated workflow changes into HDFS """ self._create_file(deployment_dir, self.job.XML_FILE_NAME, self.job.to_xml(mapping=mapping), do_as=True) data_properties = smart_str('\n'.join(['%s=%s' % (key, val) for key, val in mapping.iteritems()])) self._create_file(deployment_dir, 'job.properties', data_properties, do_as=True)
def force_dict_to_strings(dictionary): if not dictionary: return dictionary new_dict = {} for k in dictionary: new_key = smart_str(k) if isinstance(dictionary[k], basestring): # Strings should not be unicode. new_dict[new_key] = smart_str(dictionary[k]) elif isinstance(dictionary[k], dict): # Recursively force dicts to strings. new_dict[new_key] = force_dict_to_strings(dictionary[k]) else: # Normal objects, or other literals, should not be converted. new_dict[new_key] = dictionary[k] return new_dict
def open_old_workflow(request): doc_id = request.GET.get('workflow') workflow = Document.objects.get(id=doc_id).content_object.get_full_node() try: _workflow = import_workflow_from_hue_3_7(workflow) return _edit_workflow(request, None, _workflow) except Exception, e: LOG.warn('Could not open old worklow: %s' % smart_str(e)) return old_edit_workflow(request, workflow=workflow.id)
def _edit_workflow(request, doc, workflow): workflow_data = workflow.get_data() api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def is_valid(self): assert self._is_bound r = True # Explicitly iterate through all of them; we don't want # to abort early, since we want each form's is_valid to be run. for f in self._forms.values(): if not f.is_valid(): LOG.error(smart_str(f.errors)) r = False return r
def _copy_files(self, deployment_dir, oozie_xml, oozie_properties): """ Copy XML and the jar_path files from Java or MR actions to the deployment directory. This should run as the workflow user. """ xml_path = self.fs.join(deployment_dir, self.job.XML_FILE_NAME) self.fs.create(xml_path, overwrite=True, permission=0644, data=smart_str(oozie_xml)) LOG.debug("Created %s" % (xml_path,)) properties_path = self.fs.join(deployment_dir, "job.properties") self.fs.create( properties_path, overwrite=True, permission=0644, data=smart_str("\n".join(["%s=%s" % (key, val) for key, val in oozie_properties.iteritems()])), ) LOG.debug("Created %s" % (properties_path,)) # List jar files files = [] lib_path = self.fs.join(deployment_dir, "lib") if hasattr(self.job, "nodes"): for node in self.job.nodes: jar_path = node.data["properties"].get("jar_path") if jar_path: if not jar_path.startswith("/"): # If workspace relative path jar_path = self.fs.join(self.job.deployment_dir, jar_path) if not jar_path.startswith(lib_path): # If not already in lib files.append(jar_path) # Copy the jar files to the workspace lib if files: for jar_file in files: LOG.debug("Updating %s" % jar_file) jar_lib_path = self.fs.join(lib_path, self.fs.basename(jar_file)) # Refresh if needed if self.fs.exists(jar_lib_path) and self.fs.exists(jar_file): stat_src = self.fs.stats(jar_file) stat_dest = self.fs.stats(jar_lib_path) if stat_src.fileId != stat_dest.fileId: self.fs.remove(jar_lib_path, skip_trash=True) self.fs.copyfile(jar_file, jar_lib_path)
def _submit_workflow(user, fs, jt, workflow, mapping): try: submission = Submission(user, workflow, fs, jt, mapping) job_id = submission.run() return job_id except RestException, ex: detail = ex._headers.get('oozie-error-message', ex) if 'Max retries exceeded with url' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.error(smart_str(detail)) raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
def _create_workflow(self, pig_script, params): workflow = Workflow.objects.new_workflow(self.user) workflow.name = OozieApi.WORKFLOW_NAME workflow.is_history = True if pig_script.use_hcatalog: workflow.add_parameter("oozie.action.sharelib.for.pig", "pig,hcatalog") workflow.save() Workflow.objects.initialize(workflow, self.fs) script_path = workflow.deployment_dir + '/script.pig' if self.fs: # For testing, difficult to mock self.fs.do_as_user(self.user.username, self.fs.create, script_path, data=smart_str(pig_script.dict['script'])) files = [] archives = [] popup_params = json.loads(params) popup_params_names = [param['name'] for param in popup_params] pig_params = self._build_parameters(popup_params) script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names] pig_params += self._build_parameters(script_params) job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']] for resource in pig_script.dict['resources']: if resource['type'] == 'file': files.append(resource['value']) if resource['type'] == 'archive': archives.append({"dummy": "", "name": resource['value']}) action = Pig.objects.create( name='pig', script_path=script_path, workflow=workflow, node_type='pig', params=json.dumps(pig_params), files=json.dumps(files), archives=json.dumps(archives), job_properties=json.dumps(job_properties) ) if pig_script.use_hcatalog and get_oozie(self.user).security_enabled: action.credentials = [{'name': 'hcat', 'value': True}] action.save() action.add_node(workflow.end) start_link = workflow.start.get_link() start_link.child = action start_link.save() return workflow
def _submit_workflow(user, fs, workflow, mapping): try: submission = Submission(user, workflow, fs, mapping) job_id = submission.run() History.objects.create_from_submission(submission) return job_id except RestException, ex: detail = ex._headers.get('oozie-error-message', ex) if 'urlopen error' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.error(smart_str(detail)) raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)
def invalidate_tables(self, database, tables): handle = None for table in tables: try: hql = "INVALIDATE METADATA `%s`.`%s`" % (database, table) query = hql_query(hql, database, query_type=QUERY_TYPES[1]) handle = self.execute_and_wait(query, timeout_sec=10.0) except Exception, e: LOG.warn("Refresh tables cache out of sync: %s" % smart_str(e)) finally:
def _submit_workflow(user, fs, jt, workflow, mapping): try: submission = Submission(user, workflow, fs, jt, mapping) job_id = submission.run() History.objects.create_from_submission(submission) return job_id except RestException as ex: detail = ex._headers.get('oozie-error-message', ex) if 'Max retries exceeded with url' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.error(smart_str(detail)) raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail) return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
def test_connection(self, throw_exception=True): try: self.connect() return True except Exception as e: message = force_unicode(smart_str(e)) if throw_exception: if 'Access denied' in message: raise AuthenticationRequired() raise else: return False finally: self.close()
def decorator(*args, **kwargs): try: return func(*args, **kwargs) except AuthenticationRequired as e: raise e except Exception as e: message = force_unicode(smart_str(e)) if 'error occurred while trying to connect to the Java server' in message: raise QueryError, _('%s: is the DB Proxy server running?' ) % message, sys.exc_info()[2] elif 'Access denied' in message: raise AuthenticationRequired, '', sys.exc_info()[2] else: raise QueryError, message, sys.exc_info()[2]
def _augment_pivot_nd(facet_id, counts, selected_values, fields='', values=''): for c in counts: fq_fields = (fields if fields else []) + [c['field']] fq_values = (values if values else []) + [smart_str(c['value'])] if 'pivot' in c: _augment_pivot_nd(facet_id, c['pivot'], selected_values, fq_fields, fq_values) fq_filter = selected_values.get(facet_id, []) _selected_values = [f['value'] for f in fq_filter] c['selected'] = fq_values in _selected_values c['exclude'] = False c['fq_fields'] = fq_fields c['fq_values'] = fq_values
def _submit_workflow(user, fs, jt, workflow, mapping): try: submission = Submission(user, workflow, fs, jt, mapping) job_id = submission.run() workflow.document.add_to_history(submission.user, {'properties': submission.properties, 'oozie_id': submission.oozie_id}) return job_id except RestException, ex: detail = ex._headers.get('oozie-error-message', ex) if 'Max retries exceeded with url' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.exception('Error submitting workflow: %s' % smart_str(detail)) raise PopupException(_("Error submitting workflow %s: %s") % (workflow, detail))
def process_response(self, request, response): if not _has_tidylib or not self._is_html(request, response): return response html, errors = tidylib.tidy_document(response.content, self._options, keep_doc=True) if not errors: return response # Filter out what we care about err_list = errors.rstrip().split('\n') err_list = self._filter_warnings(err_list) if not err_list: return response try: fn = urlresolvers.resolve(request.path)[0] fn_name = '%s.%s' % (fn.__module__, fn.__name__) except: LOG.exception('failed to resolve url') fn_name = '<unresolved_url>' # Write the two versions of html out for offline debugging filename = os.path.join(self._outdir, fn_name) result = "HTML tidy result: %s [%s]:" \ "\n\t%s" \ "\nPlease see %s.orig %s.tidy\n-------" % \ (request.path, fn_name, '\n\t'.join(err_list), filename, filename) file(filename + '.orig', 'w').write(i18n.smart_str(response.content)) file(filename + '.tidy', 'w').write(i18n.smart_str(html)) file(filename + '.info', 'w').write(i18n.smart_str(result)) self._logger.error(result) return response
def export_documents(request): if request.GET.get('documents'): selection = json.loads(request.GET.get('documents')) else: selection = json.loads(request.POST.get('documents')) # Only export documents the user has permissions to read docs = Document2.objects.documents(user=request.user, perms='both', include_history=True, include_trashed=True).\ filter(id__in=selection).order_by('-id') # Add any dependencies to the set of exported documents export_doc_set = _get_dependencies(docs) # For directories, add any children docs to the set of exported documents export_doc_set.update(_get_dependencies(docs, deps_mode=False)) # Get PKs of documents to export doc_ids = [doc.pk for doc in export_doc_set] num_docs = len(doc_ids) if len(selection) == 1 and num_docs >= len(selection) and docs[0].name: filename = docs[0].name else: filename = 'hue-documents-%s-(%s)' % (datetime.today().strftime('%Y-%m-%d'), num_docs) f = StringIO.StringIO() if doc_ids: doc_ids = ','.join(map(str, doc_ids)) management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_foreign_keys=True, verbosity=2, stdout=f) if request.GET.get('format') == 'json': return JsonResponse(f.getvalue(), safe=False) elif request.GET.get('format') == 'zip': zfile = zipfile.ZipFile(f, 'w') zfile.writestr("hue.json", f.getvalue()) for doc in docs: if doc.type == 'notebook': try: from spark.models import Notebook zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str())) except Exception, e: LOG.exception(e) zfile.close() response = HttpResponse(content_type="application/zip") response["Content-Length"] = len(f.getvalue()) response['Content-Disposition'] = 'attachment; filename="%s".zip' % filename response.write(f.getvalue()) return response
def _create_workflow(self, pig_script, params): workflow = Workflow.objects.new_workflow(self.user) workflow.name = OozieApi.WORKFLOW_NAME workflow.is_history = True if pig_script.use_hcatalog: workflow.add_parameter("oozie.action.sharelib.for.pig", "pig,hcatalog") workflow.save() Workflow.objects.initialize(workflow, self.fs) script_path = workflow.deployment_dir + '/script.pig' if self.fs: # For testing, difficult to mock self.fs.do_as_user(self.user.username, self.fs.create, script_path, data=smart_str(pig_script.dict['script'])) files = [] archives = [] popup_params = json.loads(params) popup_params_names = [param['name'] for param in popup_params] pig_params = self._build_parameters(popup_params) script_params = [param for param in pig_script.dict['parameters'] if param['name'] not in popup_params_names] pig_params += self._build_parameters(script_params) job_properties = [{"name": prop['name'], "value": prop['value']} for prop in pig_script.dict['hadoopProperties']] for resource in pig_script.dict['resources']: if resource['type'] == 'file': files.append(resource['value']) if resource['type'] == 'archive': archives.append({"dummy": "", "name": resource['value']}) action = Pig.objects.create( name='pig', script_path=script_path, workflow=workflow, node_type='pig', params=json.dumps(pig_params), files=json.dumps(files), archives=json.dumps(archives), job_properties=json.dumps(job_properties), ) action.add_node(workflow.end) start_link = workflow.start.get_link() start_link.child = action start_link.save() return workflow
def invalidate_tables(self, database, tables): handle = None for table in tables: try: hql = "INVALIDATE METADATA `%s`.`%s`" % ( database, table, ) query = hql_query(hql, database, query_type=QUERY_TYPES[1]) handle = self.execute_and_wait(query, timeout_sec=10.0) except Exception, e: LOG.warn('Refresh tables cache out of sync: %s' % smart_str(e)) finally:
def get_logs(request): response = {'status': -1} notebook = json.loads(request.POST.get('notebook', '{}')) snippet = json.loads(request.POST.get('snippet', '{}')) startFrom = request.POST.get('from') startFrom = int(startFrom) if startFrom else None size = request.POST.get('size') size = int(size) if size else None full_log = smart_str(request.POST.get('full_log', '')) db = get_api(request, snippet) with opentracing.tracer.start_span('notebook-get_logs') as span: logs = smart_str( db.get_log(notebook, snippet, startFrom=startFrom, size=size)) span.set_tag('user-id', request.user.username) span.set_tag( 'query-id', snippet['result']['handle']['guid'] if snippet['result'].get('handle') and snippet['result']['handle'].get('guid') else None) full_log += logs jobs = db.get_jobs(notebook, snippet, full_log) response['logs'] = logs.strip() response['progress'] = min( db.progress(notebook, snippet, logs=full_log), 99) if snippet['status'] != 'available' and snippet[ 'status'] != 'success' else 100 response['jobs'] = jobs response['isFullLogs'] = db.get_log_is_full_log(notebook, snippet) response['status'] = 0 return JsonResponse(response)
def query_error_handler(func): def decorator(*args, **kwargs): try: return func(*args, **kwargs) except AuthenticationRequired, e: raise e except Exception, e: message = force_unicode(smart_str(e)) if 'error occurred while trying to connect to the Java server' in message: raise QueryError( _('%s: is the DB Proxy server running?') % message) elif 'Access denied' in message: raise AuthenticationRequired() else: raise QueryError(message)
def _fixup(self): """ Fixup: - time fields as struct_time - config dict """ super(BundleAction, self)._fixup() self.type = 'coord-action' self.name = self.coordJobName if self.conf: xml = StringIO(i18n.smart_str(self.conf)) self.conf_dict = hadoop.confparse.ConfParse(xml) else: self.conf_dict = {}
def download_log_view(request): """ Zip up the log buffer and then return as a file attachment. """ if not is_admin(request.user): return HttpResponse(_("You must be a superuser.")) l = logging.getLogger() for h in l.handlers: if isinstance(h, desktop.log.log_buffer.FixedBufferHandler): try: # We want to avoid doing a '\n'.join of the entire log in memory # in case it is rather big. So we write it to a file line by line # and pass that file to zipfile, which might follow a more efficient path. tmp = tempfile.NamedTemporaryFile() log_tmp = tempfile.NamedTemporaryFile( "w+t" ) if sys.version_info[0] == 2 else tempfile.NamedTemporaryFile( "w+t", encoding='utf-8') for l in h.buf: log_tmp.write(smart_str(l, errors='replace') + '\n') # This is not just for show - w/out flush, we often get truncated logs log_tmp.flush() t = time.time() zip = zipfile.ZipFile(tmp, "w", zipfile.ZIP_DEFLATED) zip.write(log_tmp.name, "hue-logs/hue-%s.log" % t) zip.close() length = tmp.tell() # if we don't seek to start of file, no bytes will be written tmp.seek(0) wrapper = FileWrapper(tmp) response = HttpResponse(wrapper, content_type="application/zip") response[ 'Content-Disposition'] = 'attachment; filename=hue-logs-%s.zip' % t response['Content-Length'] = length return response except Exception as e: LOG.exception("Couldn't construct zip file to write logs") return log_view(request) return render_to_response( "logs.mako", dict(log=[_("No logs found.")], is_embeddable=request.GET.get('is_embeddable', False)))
def import_documents(request): if request.FILES.get('documents'): documents = request.FILES['documents'].read() else: documents = json.loads(request.POST.get('documents')) documents = json.loads(documents) docs = [] uuids_map = dict((doc['fields']['uuid'], None) for doc in documents) for doc in documents: # Remove any deprecated fields if 'tags' in doc['fields']: doc['fields'].pop('tags') # If doc is not owned by current user, make a copy of the document with current user as owner if doc['fields']['owner'][0] != request.user.username: doc = _copy_document_with_owner(doc, request.user, uuids_map) else: # Update existing doc or create new doc = _create_or_update_document_with_owner( doc, request.user, uuids_map) # If the doc contains any history dependencies, ignore them # NOTE: this assumes that each dependency is exported as an array using the natural PK [uuid, version, is_history] deps_minus_history = [ dep for dep in doc.get('dependencies', []) if len(dep) >= 3 and not dep[2] ] doc['dependencies'] = deps_minus_history # Set last modified date to now doc['fields']['last_modified'] = datetime.now().replace( microsecond=0).isoformat() docs.append(doc) f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json') f.write(json.dumps(docs)) f.flush() stdout = StringIO.StringIO() try: management.call_command('loaddata', f.name, stdout=stdout) except Exception, e: return JsonResponse({'message': smart_str(e)})
def _submit_workflow_helper(request, workflow, submit_action): ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': cluster = json.loads(request.POST.get('cluster', '{}')) params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' mapping['send_email'] = request.POST.get('email_checkbox') == 'on' if '/submit_single_action/' in submit_action: mapping['submit_single_action'] = True try: job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping) except Exception as e: raise PopupException(_('Workflow submission failed'), detail=smart_str(e), error_code=200) jsonify = request.POST.get('format') == 'json' if jsonify: return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'workflow'}, safe=False) else: request.info(_('Workflow submitted')) return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors)) else: cluster_json = request.GET.get('cluster', '{}') parameters = workflow and workflow.find_all_parameters() or [] initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters])) params_form = ParametersFormSet(initial=initial_params) return render( '/scheduler/submit_job_popup.mako', request, { 'params_form': params_form, 'name': workflow.name, 'action': submit_action, 'show_dryrun': True, 'email_id': request.user.email, 'is_oozie_mail_enabled': _is_oozie_mail_enabled(request.user), 'return_json': request.GET.get('format') == 'json', 'cluster_json': cluster_json }, force_template=True )
def export_documents(request): if request.GET.get('documents'): selection = json.loads(request.GET.get('documents')) else: selection = json.loads(request.POST.get('documents')) # If non admin, only export documents the user owns docs = Document2.objects if not request.user.is_superuser: docs = docs.filter(owner=request.user) docs = docs.filter(id__in=selection).order_by('-id') doc_ids = docs.values_list('id', flat=True) f = StringIO.StringIO() if doc_ids: doc_ids = ','.join(map(str, doc_ids)) management.call_command('dumpdata', 'desktop.Document2', primary_keys=doc_ids, indent=2, use_natural_keys=True, verbosity=2, stdout=f) if request.GET.get('format') == 'json': return JsonResponse(f.getvalue(), safe=False) elif request.GET.get('format') == 'zip': zfile = zipfile.ZipFile(f, 'w') zfile.writestr("hue.json", f.getvalue()) for doc in docs: if doc.type == 'notebook': try: from spark.models import Notebook zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id), smart_str(Notebook(document=doc).get_str())) except Exception, e: print e LOG.exception(e) zfile.close() response = HttpResponse(content_type="application/zip") response["Content-Length"] = len(f.getvalue()) response[ 'Content-Disposition'] = 'attachment; filename="hue-documents.zip"' response.write(f.getvalue()) return response
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) # Automatically create the workflow of a scheduled document # To move to save coordinator document_uuid = request.GET.get('document') if document_uuid: # Has already a workflow managing the query for this user? workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid]) if workflows.exists(): workflow_doc = workflows.get() else: document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid) workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True) if doc: doc.dependencies.add(workflow_doc) workflow_uuid = workflow_doc.uuid coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name elif request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def submit_external_job(request, application_path): ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' application_name = os.path.basename(application_path) application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow() mapping[application_class.get_application_path_key()] = os.path.dirname(application_path) try: submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping) job_id = submission.run(application_path) except RestException as ex: detail = ex._headers.get('oozie-error-message', ex) if 'Max retries exceeded with url' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.exception(smart_str(detail)) raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail) jsonify = request.POST.get('format') == 'json' if jsonify: return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False) else: request.info(_('Oozie job submitted')) view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow' return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors)) else: parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path) initial_params = ParameterForm.get_initial_params(parameters) params_form = ParametersFormSet(initial=initial_params) popup = render('editor/submit_job_popup.mako', request, { 'params_form': params_form, 'name': _('Job'), 'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}), 'show_dryrun': os.path.basename(application_path) != 'bundle.xml', 'return_json': request.GET.get('format') == 'json' }, force_template=True).content return JsonResponse(popup, safe=False)
def kill(self, app_id): data = {'state': 'KILLED'} token = None # Tokens are managed within the kill method but should be moved out when not alpha anymore or we support submitting an app. if self.security_enabled and False: full_token = self.delegation_token() if 'token' not in full_token: raise PopupException(_('YARN did not return any token field.'), detail=smart_str(full_token)) data['X-Hadoop-Delegation-Token'] = token = full_token.pop('token') LOG.debug('Received delegation token %s' % full_token) try: params = self._get_params() return self._execute(self._root.put, 'cluster/apps/%(app_id)s/state' % {'app_id': app_id}, params=params, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) finally: if token: self.cancel_token(token)
def _submit_workflow_helper(request, workflow, submit_action): ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' try: job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping) except Exception, e: raise PopupException(_('Workflow submission failed'), detail=smart_str(e)) request.info(_('Workflow submitted')) return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors))
def query_and_fetch(db, statement, n=None): data = None try: db.connect() curs = db.cursor() try: if curs.execute(statement): data = curs.fetchmany(n) meta = curs.description return data, meta finally: curs.close() except Exception, e: message = force_unicode(smart_str(e)) if 'Access denied' in message: raise AuthenticationRequired() raise
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator') doc = None if coordinator_id: doc = Document2.objects.get(id=coordinator_id) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def submit_workflow(request, doc_id): workflow = Workflow(document=Document2.objects.get(id=doc_id)) ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) try: job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping) except Exception, e: raise PopupException(_('Workflow submission failed'), detail=smart_str(e)) request.info(_('Workflow submitted')) return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors))
def get_indexes(self, include_cores=False): indexes = [] try: if self.is_solr_cloud_mode(): collections = self.api.collections2() for name in collections: indexes.append({ 'name': name, 'type': 'collection', 'collections': [] }) if self.is_solr_cloud_mode(): try: if self.is_solr_six_or_more(): solr_aliases = self.api.list_aliases() else: solr_aliases = self.api.aliases() for name in solr_aliases: collections = solr_aliases[name].split() indexes.append({ 'name': name, 'type': 'alias', 'collections': collections }) except Exception: LOG.exception('Aliases could not be retrieved') if not self.is_solr_cloud_mode() or include_cores: solr_cores = self.api.cores() for name in solr_cores: indexes.append({ 'name': name, 'type': 'core', 'collections': [] }) except Exception as e: msg = _('Solr server could not be contacted properly: %s') % e LOG.warn(msg) raise PopupException(msg, detail=smart_str(e)) return sorted(indexes, key=lambda index: index['name'])
def _fixup(self): """ Fixup: - time fields as struct_time - config dict """ super(BundleAction, self)._fixup() self.type = 'coord-action' self.name = self.coordJobName if self.conf: conf_data = i18n.smart_str(self.conf) if not isinstance(conf_data, bytes): conf_data = conf_data.encode('utf-8') xml = string_io(conf_data) self.conf_dict = hadoop.confparse.ConfParse(xml) else: self.conf_dict = {}
def force_list_to_strings(lst): if not lst: return lst new_list = [] for item in lst: if isinstance(item, string_types): # Strings should not be unicode. new_list.append(smart_str(item)) elif isinstance(item, dict): # Recursively force dicts to strings. new_list.append(force_dict_to_strings(item)) elif isinstance(item, list): new_list.append(force_list_to_strings(item)) else: # Normal objects, or other literals, should not be converted. new_list.append(item) return new_list
def _submit_workflow_helper(request, workflow, submit_action): ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': cluster = json.loads(request.POST.get('cluster', '{}')) params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' mapping['send_email'] = request.POST.get('email_checkbox') == 'on' if '/submit_single_action/' in submit_action: mapping['submit_single_action'] = True if 'altus' in cluster.get('type', ''): mapping['cluster'] = cluster.get('id') try: job_id = _submit_workflow(request.user, request.fs, request.jt, workflow, mapping) except Exception, e: raise PopupException(_('Workflow submission failed'), detail=smart_str(e), error_code=200) jsonify = request.POST.get('format') == 'json' if jsonify: return JsonResponse( { 'status': 0, 'job_id': job_id, 'type': 'workflow' }, safe=False) else: request.info(_('Workflow submitted')) return redirect( reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors))
def invalidate_tables(self, database, tables=None): handle = None try: if tables: for table in tables: hql = "INVALIDATE METADATA `%s`.`%s`" % ( database, table, ) print hql query = hql_query(hql, database, query_type=QUERY_TYPES[1]) handle = self.execute_and_wait(query, timeout_sec=10.0) else: # call INVALIDATE on entire DB to pick up newly created tables hql = "INVALIDATE METADATA `%s`" % database print hql query = hql_query(hql, database, query_type=QUERY_TYPES[1]) handle = self.execute_and_wait(query, timeout_sec=10.0) except Exception, e: LOG.warn('Refresh tables cache out of sync: %s' % smart_str(e))
def _fixup(self): """ Fixup: - time fields as struct_time - config dict """ super(CoordinatorAction, self)._fixup() if self.createdTime: self.createdTime = parse_timestamp(self.createdTime) if self.nominalTime: self.nominalTime = parse_timestamp(self.nominalTime) if self.lastModifiedTime: self.lastModifiedTime = parse_timestamp(self.lastModifiedTime) if self.runConf: xml = StringIO(i18n.smart_str(self.runConf)) self.conf_dict = hadoop.confparse.ConfParse(xml) else: self.conf_dict = {}
def _fixup(self): """ Fixup: - time fields as struct_time - config dict """ super(WorkflowAction, self)._fixup() if self.startTime: self.startTime = parse_timestamp(self.startTime) if self.endTime: self.endTime = parse_timestamp(self.endTime) if self.retries: self.retries = int(self.retries) if self.conf: xml = StringIO(i18n.smart_str(self.conf)) self.conf_dict = hadoop.confparse.ConfParse(xml) else: self.conf_dict = {}