def _get_params(self): if self.security_enabled: return {'doAs': self.user, 'timezone': TIME_ZONE.get()} return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) params = None if action == 'change': pause_time_val = request.POST.get('pause_time') if request.POST.get('clear_pause_time') == 'true': pause_time_val = '' end_time_val = request.POST.get('end_time') if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = {'value': 'endtime=%s' % (end_time_val) + ';' 'pausetime=%s' % (pause_time_val) + ';' 'concurrency=%s' % (request.POST.get('concurrency'))} elif action == 'ignore': oozie_api = get_oozie(request.user, api_version="v2") params = { 'type': 'action', 'scope': ','.join(job.aggreate(request.POST.get('actions').split())), } response['data'] = oozie_api.job_control(job_id, action, parameters=params) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException as ex: ex_message = ex.message if ex._headers.get('oozie-error-message'): ex_message = ex._headers.get('oozie-error-message') msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response['data'] = _(msg) return JsonResponse(response)
def manage_oozie_jobs(request, job_id, action): if request.method != 'POST': raise PopupException(_('Use a POST request to manage an Oozie job.')) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {'status': -1, 'data': ''} try: oozie_api = get_oozie(request.user) params = None if action == 'change': pause_time_val = request.POST.get('pause_time') if request.POST.get('clear_pause_time') == 'true': pause_time_val = '' end_time_val = request.POST.get('end_time') if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = {'value': 'endtime=%s' % (end_time_val) + ';' 'pausetime=%s' % (pause_time_val) + ';' 'concurrency=%s' % (request.POST.get('concurrency'))} elif action == 'ignore': oozie_api = get_oozie(request.user, api_version="v2") params = { 'type': 'action', 'scope': ','.join(job.aggreate(request.POST.get('actions').split())), } response['data'] = oozie_api.job_control(job_id, action, parameters=params) response['status'] = 0 if 'notification' in request.POST: request.info(_(request.POST.get('notification'))) except RestException, ex: ex_message = ex.message if ex._headers.get('oozie-error-message'): ex_message = ex._headers.get('oozie-error-message') msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response['data'] = _(msg)
def manage_oozie_jobs(request, job_id, action): if request.method != "POST": raise PopupException(_("Use a POST request to manage an Oozie job.")) job = check_job_access_permission(request, job_id) check_job_edition_permission(job, request.user) response = {"status": -1, "data": ""} try: oozie_api = get_oozie(request.user) params = None if action == "change": pause_time_val = request.POST.get("pause_time") if request.POST.get("clear_pause_time") == "true": pause_time_val = "" end_time_val = request.POST.get("end_time") if end_time_val: end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get()) if pause_time_val: pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get()) params = { "value": "endtime=%s" % (end_time_val) + ";" "pausetime=%s" % (pause_time_val) + ";" "concurrency=%s" % (request.POST.get("concurrency")) } elif action == "ignore": oozie_api = get_oozie(request.user, api_version="v2") params = {"type": "action", "scope": ",".join(job.aggreate(request.POST.get("actions").split()))} response["data"] = oozie_api.job_control(job_id, action, parameters=params) response["status"] = 0 if "notification" in request.POST: request.info(_(request.POST.get("notification"))) except RestException, ex: ex_message = ex.message if ex._headers.get("oozie-error-message"): ex_message = ex._headers.get("oozie-error-message") msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message) LOG.exception(msg) response["data"] = _(msg)
def _get_params(self): if self.security_enabled: return { 'doAs': self.user, 'timezone': TIME_ZONE.get() } return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }
def _get_params(self): if self.security_enabled: return {"doAs": self.user, "timezone": TIME_ZONE.get()} return {"user.name": DEFAULT_USER, "doAs": self.user, "timezone": TIME_ZONE.get()}
def handle(self, *args, **options): test_options = {} test_options['TIME_ZONE'] = TIME_ZONE.get() test_options['DOAS'] = options['username'] test_options['NOW'] = current_milli_time() test_options['NOWLESSMIN'] = test_options['NOW'] - 60000 if options['testoptions'] is not None: for test_option in options['testoptions'].split(','): option, option_value = test_option.split('=') test_options[option.upper()] = option_value test_services = options['service'].split(',') supported_services = [ 'all', 'httpfs', 'solr', 'oozie', 'rm', 'jhs', 'sparkhs' ] allowed_tests = {} allowed_tests['httpfs'] = {} allowed_tests['httpfs']['USERHOME'] = None allowed_tests['jhs'] = {} allowed_tests['jhs']['FINISHED'] = None allowed_tests['oozie'] = {} allowed_tests['oozie']['STATUS'] = None allowed_tests['oozie']['CONFIGURATION'] = None allowed_tests['oozie']['WORKFLOWS'] = None allowed_tests['oozie']['COORDS'] = None allowed_tests['oozie'][ 'WORKFLOW'] = "oozie_id=0000001-190820133637006-oozie-oozi-W" allowed_tests['oozie'][ 'WORKFLOWLOG'] = "oozie_id=0000001-190820133637006-oozie-oozi-W" allowed_tests['oozie'][ 'WORKFLOWDEF'] = "oozie_id=0000001-190820133637006-oozie-oozi-W" allowed_tests['oozie'][ 'COORD'] = "oozie_id=0000001-190820133637006-oozie-oozi-C" allowed_tests['rm'] = {} allowed_tests['rm']['CLUSTERINFO'] = None allowed_tests['solr'] = {} allowed_tests['solr']['JMX'] = None if options['testname'] is not None: if len(test_services) > 1 or "all" in test_services: logging.warn( "When using --testname you must only submit one service name and you must not use all" ) sys.exit(1) if options['testname'] not in allowed_tests[ options['service'].lower()].keys(): logging.warn( "--testname %s not found in allowed_tests for service %s" % (options['testname'], options['service'])) logging.warn("Allowed tests for service:") for test in allowed_tests[options['service'].lower()].keys(): if allowed_tests[options['service'].lower()][test] is None: testoptions = "NONE" else: testoptions = allowed_tests[ options['service'].lower()][test] logging.warn("testname: %s : testoptions: %s" % (test, testoptions)) sys.exit(1) if not any(elem in test_services for elem in supported_services): logging.warn( "Your service list does not contain a supported service: %s" % options['service']) logging.warn( "Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs" ) logging.warn("Format: httpfs,solr,oozie") sys.exit(1) if not all(elem in supported_services for elem in test_services): logging.warn( "Your service list contains an unsupported service: %s" % options['service']) logging.warn( "Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs" ) logging.warn("Format: httpfs,solr,oozie") sys.exit(1) if options['service'] == 'sparkhs': logging.warn("Spark History Server not supported yet") sys.exit(1) logging.info("TEST: %s" % str(test_options['NOW'])) logging.info("Running REST API Tests on Services: %s" % options['service']) curl = Curl(verbose=options['verbose']) available_services = {} #Add Solr add_service_test(available_services, options=options, service_name="Solr", testname="JMX", suburl='jmx', method='GET', teststring='solr.solrxml.location', test_options=test_options) #Add Oozie if options['testname'] is None or options['testname'].upper( ) == "STATUS": add_service_test( available_services, options=options, service_name="Oozie", testname="STATUS", suburl= 'v1/admin/status?timezone=TIME_ZONE&user.name=hue&doAs=DOAS', method='GET', teststring='{"systemMode":"NORMAL"}', test_options=test_options) elif options['testname'].upper() == 'CONFIGURATION': add_service_test( available_services, options=options, service_name="Oozie", testname="CONFIGURATION", suburl= 'v2/admin/configuration?timezone=TIME_ZONE&user.name=hue&doAs=DOAS', method='GET', teststring='{"oozie.email.smtp.auth', test_options=test_options) elif options['testname'].upper() == 'WORKFLOWS': add_service_test( available_services, options=options, service_name="Oozie", testname="WORKFLOWS", suburl= 'v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset=1&timezone=TIME_ZONE&jobtype=wf', method='GET', teststring='"workflows":[', test_options=test_options) elif options['testname'].upper() == 'WORKFLOW': add_service_test( available_services, options=options, service_name="Oozie", testname="WORKFLOW", suburl= 'v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS', method='GET', teststring='{"appName":', test_options=test_options) elif options['testname'].upper() == 'WORKFLOWLOG': add_service_test( available_services, options=options, service_name="Oozie", testname="WORKFLOWLOG", suburl= 'v2/job/OOZIE_ID?timezone=TIME_ZONE&show=log&user.name=hue&logfilter=&doAs=DOAS', method='GET', teststring='org.apache.oozie.service.JPAService: SERVER', test_options=test_options) elif options['testname'].upper() == 'WORKFLOWDEF': add_service_test( available_services, options=options, service_name="Oozie", testname="WORKFLOWDEF", suburl= 'v2/job/OOZIE_ID?timezone=TIME_ZONE&show=definition&user.name=hue&logfilter=&doAs=DOAS', method='GET', teststring='xmlns="uri', test_options=test_options) elif options['testname'].upper() == 'COORDS': add_service_test( available_services, options=options, service_name="Oozie", testname="COORDS", suburl= 'v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset=1&timezone=TIME_ZONE&jobtype=coord', method='GET', teststring='"coordinatorjobs":[', test_options=test_options) elif options['testname'].upper() == 'COORD': add_service_test( available_services, options=options, service_name="Oozie", testname="COORD", suburl= 'v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS', method='GET', teststring='{"appName":', test_options=test_options) #Add HTTPFS add_service_test( available_services, options=options, service_name="Httpfs", testname="USERHOME", suburl='user/DOAS?op=GETFILESTATUS&user.name=hue&DOAS=%s', method='GET', teststring='"type":"DIRECTORY"', test_options=test_options) #Add RM add_service_test(available_services, options=options, service_name="RM", testname="CLUSTERINFO", suburl='ws/v1/cluster/info', method='GET', teststring='"clusterInfo"', test_options=test_options) #Add JHS add_service_test( available_services, options=options, service_name="JHS", testname="FINISHED", suburl= 'ws/v1/history/mapreduce/jobs?finishedTimeBegin=NOWLESSMIN&finishedTimeEnd=NOW', method='GET', teststring='{"jobs"', test_options=test_options) for service in available_services: for service_test in available_services[service]['tests']: logging.info("Running %s %s Test:" % (service, service_test)) start_time = time.time() response = curl.do_curl_available_services( available_services[service]['tests'][service_test]) returned_in = (time.time() - start_time) * 1000 if available_services[service]['tests'][service_test][ 'test'] in response: logging.info( "TEST: %s %s: Passed in %dms: %s found in response" % (service, service_test, returned_in, available_services[service]['tests'][service_test] ['test'])) if options['entireresponse']: logging.info("TEST: %s %s: Response: %s" % (service, service_test, response)) else: logging.info( "TEST: %s %s: Failed in %dms: Response: %s" % (service, service_test, returned_in, response)) log_file = log_dir + '/backend_test_curl.log' print "" print "Tests completed, view logs here: %s" % log_file print "Report:" cmd = 'grep -A1000 "%s" %s | grep "TEST:" | sed "s/.*INFO.*TEST:/ TEST:/g"' % ( str(test_options['NOW']), log_file) grep_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) grep_response = grep_process.communicate()[0] print "%s" % grep_response print "" print "OS Repro Commands are:" cmd = 'grep -A1000 "%s" %s | grep "OSRUN:" | sed "s/.*INFO.*OSRUN:/ /g"' % ( str(test_options['NOW']), log_file) grep_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) grep_response = grep_process.communicate()[0] print "%s" % grep_response
def _get_params(self): return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }