def manage_oozie_jobs(request, job_id, action):
if request.method != 'POST':
raise PopupException(_('Use a POST request to manage an Oozie job.'))
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
response = {'status': -1, 'data': ''}
try:
oozie_api = get_oozie(request.user)
params = None
if action == 'change':
pause_time_val = request.POST.get('pause_time')
if request.POST.get('clear_pause_time') == 'true':
pause_time_val = ''
end_time_val = request.POST.get('end_time')
if end_time_val:
end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
if pause_time_val:
pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
params = {'value': 'endtime=%s' % (end_time_val) + ';'
'pausetime=%s' % (pause_time_val) + ';'
'concurrency=%s' % (request.POST.get('concurrency'))}
elif action == 'ignore':
oozie_api = get_oozie(request.user, api_version="v2")
params = {
'type': 'action',
'scope': ','.join(job.aggreate(request.POST.get('actions').split())),
}
response['data'] = oozie_api.job_control(job_id, action, parameters=params)
response['status'] = 0
if 'notification' in request.POST:
request.info(_(request.POST.get('notification')))
except RestException, ex:
ex_message = ex.message
if ex._headers.get('oozie-error-message'):
ex_message = ex._headers.get('oozie-error-message')
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response['data'] = _(msg)
def manage_oozie_jobs(request, job_id, action):
if request.method != 'POST':
raise PopupException(_('Use a POST request to manage an Oozie job.'))
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
response = {'status': -1, 'data': ''}
try:
oozie_api = get_oozie(request.user)
params = None
if action == 'change':
pause_time_val = request.POST.get('pause_time')
if request.POST.get('clear_pause_time') == 'true':
pause_time_val = ''
end_time_val = request.POST.get('end_time')
if end_time_val:
end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
if pause_time_val:
pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
params = {'value': 'endtime=%s' % (end_time_val) + ';'
'pausetime=%s' % (pause_time_val) + ';'
'concurrency=%s' % (request.POST.get('concurrency'))}
elif action == 'ignore':
oozie_api = get_oozie(request.user, api_version="v2")
params = {
'type': 'action',
'scope': ','.join(job.aggreate(request.POST.get('actions').split())),
}
response['data'] = oozie_api.job_control(job_id, action, parameters=params)
response['status'] = 0
if 'notification' in request.POST:
request.info(_(request.POST.get('notification')))
except RestException, ex:
ex_message = ex.message
if ex._headers.get('oozie-error-message'):
ex_message = ex._headers.get('oozie-error-message')
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response['data'] = _(msg)
def manage_oozie_jobs(request, job_id, action):
if request.method != "POST":
raise PopupException(_("Use a POST request to manage an Oozie job."))
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
response = {"status": -1, "data": ""}
try:
oozie_api = get_oozie(request.user)
params = None
if action == "change":
pause_time_val = request.POST.get("pause_time")
if request.POST.get("clear_pause_time") == "true":
pause_time_val = ""
end_time_val = request.POST.get("end_time")
if end_time_val:
end_time_val = convert_to_server_timezone(end_time_val, TIME_ZONE.get())
if pause_time_val:
pause_time_val = convert_to_server_timezone(pause_time_val, TIME_ZONE.get())
params = {
"value": "endtime=%s" % (end_time_val) + ";"
"pausetime=%s" % (pause_time_val) + ";"
"concurrency=%s" % (request.POST.get("concurrency"))
}
elif action == "ignore":
oozie_api = get_oozie(request.user, api_version="v2")
params = {"type": "action", "scope": ",".join(job.aggreate(request.POST.get("actions").split()))}
response["data"] = oozie_api.job_control(job_id, action, parameters=params)
response["status"] = 0
if "notification" in request.POST:
request.info(_(request.POST.get("notification")))
except RestException, ex:
ex_message = ex.message
if ex._headers.get("oozie-error-message"):
ex_message = ex._headers.get("oozie-error-message")
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response["data"] = _(msg)
def _get_params(self):
if self.security_enabled:
return { 'doAs': self.user, 'timezone': TIME_ZONE.get() }
return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }
def _get_params(self):
if self.security_enabled:
return { 'doAs': self.user, 'timezone': TIME_ZONE.get() }
return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }
def _get_params(self):
if self.security_enabled:
return {"doAs": self.user, "timezone": TIME_ZONE.get()}
return {"user.name": DEFAULT_USER, "doAs": self.user, "timezone": TIME_ZONE.get()}
def handle(self, *args, **options):
test_options = {}
test_options['TIME_ZONE'] = TIME_ZONE.get()
test_options['DOAS'] = options['username']
test_options['NOW'] = current_milli_time()
test_options['NOWLESSMIN'] = test_options['NOW'] - 60000
if options['testoptions'] is not None:
for test_option in options['testoptions'].split(','):
option, option_value = test_option.split('=')
test_options[option.upper()] = option_value
test_services = options['service'].split(',')
supported_services = [
'all', 'httpfs', 'solr', 'oozie', 'rm', 'jhs', 'sparkhs'
]
allowed_tests = {}
allowed_tests['httpfs'] = {}
allowed_tests['httpfs']['USERHOME'] = None
allowed_tests['jhs'] = {}
allowed_tests['jhs']['FINISHED'] = None
allowed_tests['oozie'] = {}
allowed_tests['oozie']['STATUS'] = None
allowed_tests['oozie']['CONFIGURATION'] = None
allowed_tests['oozie']['WORKFLOWS'] = None
allowed_tests['oozie']['COORDS'] = None
allowed_tests['oozie'][
'WORKFLOW'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
allowed_tests['oozie'][
'WORKFLOWLOG'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
allowed_tests['oozie'][
'WORKFLOWDEF'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
allowed_tests['oozie'][
'COORD'] = "oozie_id=0000001-190820133637006-oozie-oozi-C"
allowed_tests['rm'] = {}
allowed_tests['rm']['CLUSTERINFO'] = None
allowed_tests['solr'] = {}
allowed_tests['solr']['JMX'] = None
if options['testname'] is not None:
if len(test_services) > 1 or "all" in test_services:
logging.warn(
"When using --testname you must only submit one service name and you must not use all"
)
sys.exit(1)
if options['testname'] not in allowed_tests[
options['service'].lower()].keys():
logging.warn(
"--testname %s not found in allowed_tests for service %s" %
(options['testname'], options['service']))
logging.warn("Allowed tests for service:")
for test in allowed_tests[options['service'].lower()].keys():
if allowed_tests[options['service'].lower()][test] is None:
testoptions = "NONE"
else:
testoptions = allowed_tests[
options['service'].lower()][test]
logging.warn("testname: %s : testoptions: %s" %
(test, testoptions))
sys.exit(1)
if not any(elem in test_services for elem in supported_services):
logging.warn(
"Your service list does not contain a supported service: %s" %
options['service'])
logging.warn(
"Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs"
)
logging.warn("Format: httpfs,solr,oozie")
sys.exit(1)
if not all(elem in supported_services for elem in test_services):
logging.warn(
"Your service list contains an unsupported service: %s" %
options['service'])
logging.warn(
"Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs"
)
logging.warn("Format: httpfs,solr,oozie")
sys.exit(1)
if options['service'] == 'sparkhs':
logging.warn("Spark History Server not supported yet")
sys.exit(1)
logging.info("TEST: %s" % str(test_options['NOW']))
logging.info("Running REST API Tests on Services: %s" %
options['service'])
curl = Curl(verbose=options['verbose'])
available_services = {}
#Add Solr
add_service_test(available_services,
options=options,
service_name="Solr",
testname="JMX",
suburl='jmx',
method='GET',
teststring='solr.solrxml.location',
test_options=test_options)
#Add Oozie
if options['testname'] is None or options['testname'].upper(
) == "STATUS":
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="STATUS",
suburl=
'v1/admin/status?timezone=TIME_ZONE&user.name=hue&doAs=DOAS',
method='GET',
teststring='{"systemMode":"NORMAL"}',
test_options=test_options)
elif options['testname'].upper() == 'CONFIGURATION':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="CONFIGURATION",
suburl=
'v2/admin/configuration?timezone=TIME_ZONE&user.name=hue&doAs=DOAS',
method='GET',
teststring='{"oozie.email.smtp.auth',
test_options=test_options)
elif options['testname'].upper() == 'WORKFLOWS':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="WORKFLOWS",
suburl=
'v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset=1&timezone=TIME_ZONE&jobtype=wf',
method='GET',
teststring='"workflows":[',
test_options=test_options)
elif options['testname'].upper() == 'WORKFLOW':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="WORKFLOW",
suburl=
'v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS',
method='GET',
teststring='{"appName":',
test_options=test_options)
elif options['testname'].upper() == 'WORKFLOWLOG':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="WORKFLOWLOG",
suburl=
'v2/job/OOZIE_ID?timezone=TIME_ZONE&show=log&user.name=hue&logfilter=&doAs=DOAS',
method='GET',
teststring='org.apache.oozie.service.JPAService: SERVER',
test_options=test_options)
elif options['testname'].upper() == 'WORKFLOWDEF':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="WORKFLOWDEF",
suburl=
'v2/job/OOZIE_ID?timezone=TIME_ZONE&show=definition&user.name=hue&logfilter=&doAs=DOAS',
method='GET',
teststring='xmlns="uri',
test_options=test_options)
elif options['testname'].upper() == 'COORDS':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="COORDS",
suburl=
'v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset=1&timezone=TIME_ZONE&jobtype=coord',
method='GET',
teststring='"coordinatorjobs":[',
test_options=test_options)
elif options['testname'].upper() == 'COORD':
add_service_test(
available_services,
options=options,
service_name="Oozie",
testname="COORD",
suburl=
'v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS',
method='GET',
teststring='{"appName":',
test_options=test_options)
#Add HTTPFS
add_service_test(
available_services,
options=options,
service_name="Httpfs",
testname="USERHOME",
suburl='user/DOAS?op=GETFILESTATUS&user.name=hue&DOAS=%s',
method='GET',
teststring='"type":"DIRECTORY"',
test_options=test_options)
#Add RM
add_service_test(available_services,
options=options,
service_name="RM",
testname="CLUSTERINFO",
suburl='ws/v1/cluster/info',
method='GET',
teststring='"clusterInfo"',
test_options=test_options)
#Add JHS
add_service_test(
available_services,
options=options,
service_name="JHS",
testname="FINISHED",
suburl=
'ws/v1/history/mapreduce/jobs?finishedTimeBegin=NOWLESSMIN&finishedTimeEnd=NOW',
method='GET',
teststring='{"jobs"',
test_options=test_options)
for service in available_services:
for service_test in available_services[service]['tests']:
logging.info("Running %s %s Test:" % (service, service_test))
start_time = time.time()
response = curl.do_curl_available_services(
available_services[service]['tests'][service_test])
returned_in = (time.time() - start_time) * 1000
if available_services[service]['tests'][service_test][
'test'] in response:
logging.info(
"TEST: %s %s: Passed in %dms: %s found in response" %
(service, service_test, returned_in,
available_services[service]['tests'][service_test]
['test']))
if options['entireresponse']:
logging.info("TEST: %s %s: Response: %s" %
(service, service_test, response))
else:
logging.info(
"TEST: %s %s: Failed in %dms: Response: %s" %
(service, service_test, returned_in, response))
log_file = log_dir + '/backend_test_curl.log'
print ""
print "Tests completed, view logs here: %s" % log_file
print "Report:"
cmd = 'grep -A1000 "%s" %s | grep "TEST:" | sed "s/.*INFO.*TEST:/ TEST:/g"' % (
str(test_options['NOW']), log_file)
grep_process = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE)
grep_response = grep_process.communicate()[0]
print "%s" % grep_response
print ""
print "OS Repro Commands are:"
cmd = 'grep -A1000 "%s" %s | grep "OSRUN:" | sed "s/.*INFO.*OSRUN:/ /g"' % (
str(test_options['NOW']), log_file)
grep_process = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE)
grep_response = grep_process.communicate()[0]
print "%s" % grep_response
def _get_params(self):
return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }