def test_jobsub_setup(self):
# User 'test' triggers the setup of the examples.
# 'hue' home will be deleted, the examples installed in the new one
# and 'test' will try to access them.
self.cluster.fs.setuser('jobsub_test')
username = '******'
home_dir = '/user/%s/' % username
finish = conf.REMOTE_DATA_DIR.set_for_testing('%s/jobsub' % home_dir)
try:
data_dir = conf.REMOTE_DATA_DIR.get()
if not jobsub_setup.Command().has_been_setup():
self.cluster.fs.setuser(self.cluster.fs.superuser)
if self.cluster.fs.exists(home_dir):
self.cluster.fs.rmtree(home_dir)
jobsub_setup.Command().handle()
self.cluster.fs.setuser('jobsub_test')
stats = self.cluster.fs.stats(home_dir)
assert_equal(stats['user'], username)
assert_equal(oct(stats['mode']), '040755') #04 because is a dir
stats = self.cluster.fs.stats(data_dir)
assert_equal(stats['user'], username)
assert_equal(oct(stats['mode']), '041777')
# Only examples should have been created by 'hue'
stats = self.cluster.fs.listdir_stats(data_dir)
sample_stats = filter(lambda stat: stat.user == username, stats)
assert_equal(len(sample_stats), 2)
finally:
finish()
def setup_class(cls):
OozieServerProvider.setup_class()
if not cls.cluster.fs.exists("/tmp"):
cls.cluster.fs.do_as_superuser(cls.cluster.fs.mkdir, "/tmp")
cls.cluster.fs.do_as_superuser(cls.cluster.fs.chmod, "/tmp", 0777)
# Install examples
import jobsub.management.commands.jobsub_setup as jobsub_setup
if not jobsub_setup.Command().has_been_setup():
jobsub_setup.Command().handle()
cls.sleep_design_id = OozieDesign.objects.get(name='sleep_job').id
def list_designs(request, saved=None):
"""
Lists extant job designs.
Filters can be specified for owners.
Note: the URL is named "list", but since list is a built-in,
better to name the method somethign else.
"""
show_install_examples = request.user.is_superuser and not jobsub_setup.Command(
).has_been_setup()
data = JobDesign.objects.order_by('-last_modified')
owner = request.GET.get("owner", "")
name = request.GET.get('name', "")
if owner:
data = data.filter(owner__username__icontains=owner)
if name:
data = data.filter(name__icontains=name)
newlinks = [(type,
urlresolvers.reverse("jobsub.new", kwargs=dict(type=type)))
for type in interface.registry]
return render(
"list.html", request, {
'jobdesigns': list(data),
'currentuser': request.user,
'newlinks': newlinks,
'owner': owner,
'name': name,
'saved': saved,
'show_install_examples': show_install_examples,
})
def setup(request):
"""Installs jobsub examples."""
if request.method == "GET":
return render(
"confirm.html", request,
dict(url=request.path, title="Install job design examples?"))
else:
jobsub_setup.Command().handle_noargs()
return format_preserving_redirect(request, "/jobsub")
def setup(request):
"""Installs jobsub examples."""
if request.method != "POST":
raise PopupException(_('Use a POST request to install the examples.'))
try:
# Warning: below will modify fs.user
jobsub_setup.Command().handle_noargs()
except WebHdfsException, e:
raise PopupException(_('The examples could not be installed.'),
detail=e)
def test_jobsub_setup_and_run_samples(self):
"""
Merely exercises jobsub_setup, and then runs the sleep example.
"""
if not jobsub_setup.Command().has_been_setup():
jobsub_setup.Command().handle()
self.cluster.fs.setuser('jobsub_test')
assert_equal(
3,
OozieDesign.objects.filter(owner__username='******').count())
assert_equal(
2,
OozieMapreduceAction.objects.filter(
ooziedesign__owner__username='******').count())
assert_equal(
1,
OozieStreamingAction.objects.filter(
ooziedesign__owner__username='******').count())
# Make sure sample user got created.
assert_equal(1, User.objects.filter(username='******').count())
# Clone design
assert_equal(
0,
OozieDesign.objects.filter(owner__username='******').count())
jobid = OozieDesign.objects.get(name='sleep_job',
owner__username='******').id
self.client.post('/jobsub/clone_design/%d' % jobid)
assert_equal(
1,
OozieDesign.objects.filter(owner__username='******').count())
jobid = OozieDesign.objects.get(owner__username='******').id
# And now submit and run the sleep sample
response = self.client.post('/jobsub/submit_design/%d' % jobid, {
'num_reduces': 1,
'num_maps': 1,
'map_sleep_time': 1,
'reduce_sleep_time': 1
},
follow=True)
assert_true(
sum([
status in response.content for status in ('PREP', 'OK', 'DONE')
]) > 0)
assert_true(str(jobid) in response.content)
oozie_job_id = response.context['jobid']
job = OozieServerProvider.wait_until_completion(oozie_job_id,
timeout=120,
step=1)
logs = OozieServerProvider.oozie.get_job_log(oozie_job_id)
assert_equal('SUCCEEDED', job.status, logs)
# Grep
n = OozieDesign.objects.filter(owner__username='******').count()
jobid = OozieDesign.objects.get(name='grep_example').id
self.client.post('/jobsub/clone_design/%d' % jobid)
assert_equal(
n + 1,
OozieDesign.objects.filter(owner__username='******').count())
jobid = OozieDesign.objects.get(owner__username='******',
name__contains='sleep_job').id
# And now submit and run the sleep sample
response = self.client.post('/jobsub/submit_design/%d' % jobid, {
'num_reduces': 1,
'num_maps': 1,
'map_sleep_time': 1,
'reduce_sleep_time': 1
},
follow=True)
assert_true(
sum([
status in response.content for status in ('PREP', 'OK', 'DONE')
]) > 0)
assert_true(str(jobid) in response.content)
oozie_job_id = response.context['jobid']
job = OozieServerProvider.wait_until_completion(oozie_job_id,
timeout=60,
step=1)
logs = OozieServerProvider.oozie.get_job_log(oozie_job_id)
assert_equal('SUCCEEDED', job.status, logs)
def test_new_jobs(self):
"""
Submit jobs. Let them succeed or fail and view them.
"""
# Install examples
import jobsub.management.commands.jobsub_setup as jobsub_setup
if not jobsub_setup.Command().has_been_setup():
jobsub_setup.Command().handle()
# Run the sleep example, since it doesn't require user home directory
design_id = JobDesign.objects.get(name__contains="Example: Sleep").id
response = self.client.post(
"/jobsub/submit/%d" % (design_id, ),
dict(map_sleep_time_millis=1,
num_mappers=1,
num_reducers=1,
reduce_sleep_time_millis=1))
watch_id = parse_out_id(response)
response = watch_till_complete(self.client, watch_id)
job_id = Submission.objects.get(id=watch_id).submission_handle.id
hadoop_job_id = get_hadoop_job_id(self.jobsubd, job_id)
# All jobs page
response = self.client.get('/jobbrowser/jobs/')
assert_true(hadoop_job_id.lstrip('job_') in response.content)
# Single job page
response = self.client.get('/jobbrowser/jobs/%s' % hadoop_job_id)
# Check some counters for single job.
counters = response.context['job'].counters
counters_file_bytes_written = counters['FileSystemCounters'][
'counters']['FILE_BYTES_WRITTEN']
assert_true(counters_file_bytes_written['map'] > 0)
assert_true(counters_file_bytes_written['reduce'] > 0)
assert_equal(counters_file_bytes_written['displayName'],
'FILE_BYTES_WRITTEN')
assert_equal(counters_file_bytes_written['displayName'],
'FILE_BYTES_WRITTEN')
# We can't just check the complete contents of the python map because the
# SLOTS_MILLIS_* entries have a variable number of milliseconds from
# run-to-run.
assert_equal(
response.context['job'].
counters['org.apache.hadoop.mapred.JobInProgress$Counter']
['counters']['TOTAL_LAUNCHED_MAPS']['total'], 1)
assert_equal(
response.context['job'].
counters['org.apache.hadoop.mapred.JobInProgress$Counter']
['counters']['TOTAL_LAUNCHED_REDUCES']['total'], 1)
assert_equal(
response.context['job'].
counters['org.apache.hadoop.mapred.JobInProgress$Counter']
['counters']['FALLOW_SLOTS_MILLIS_MAPS']['total'], 0)
assert_equal(
response.context['job'].
counters['org.apache.hadoop.mapred.JobInProgress$Counter']
['counters']['FALLOW_SLOTS_MILLIS_REDUCES']['total'], 0)
assert_true(response.context['job'].
counters['org.apache.hadoop.mapred.JobInProgress$Counter']
['counters']['SLOTS_MILLIS_MAPS']['total'] > 0)
assert_true(response.context['job'].
counters['org.apache.hadoop.mapred.JobInProgress$Counter']
['counters']['SLOTS_MILLIS_REDUCES']['total'] > 0)
# Check conf keys made it
assert_equal(response.context['job'].conf_keys['mapredReducerClass'],
'org.apache.hadoop.examples.SleepJob')
# There should be 4 tasks for this job: cleanup, setup, map, reduce
response = self.client.get('/jobbrowser/jobs/%s/tasks' %
(hadoop_job_id, ))
assert_true('Showing 1 to 4 of 4 tasks' in response.content)
# Select by tasktype
response = self.client.get(
'/jobbrowser/jobs/%s/tasks?tasktype=reduce' % (hadoop_job_id, ))
assert_true('Showing 1 to 1 of 1 tasks' in response.content)
# Select by taskstate
response = self.client.get(
'/jobbrowser/jobs/%s/tasks?taskstate=succeeded' %
(hadoop_job_id, ))
assert_true('Showing 1 to 4 of 4 tasks' in response.content)
# Select by text
response = self.client.get('/jobbrowser/jobs/%s/tasks?tasktext=clean' %
(hadoop_job_id, ))
assert_true('Showing 1 to 1 of 1 tasks' in response.content)
# Run another sleep job but kill it
response = self.client.post(
"/jobsub/submit/%d" % (design_id, ),
dict(map_sleep_time_millis=1,
num_mappers=2000,
num_reducers=2000,
reduce_sleep_time_millis=1))
job_id = parse_out_id(response)
time.sleep(15) # 15 seconds should be enough to start the job
hadoop_job_id = get_hadoop_job_id(self.jobsubd, job_id)
client2 = make_logged_in_client('test_non_superuser',
is_superuser=False)
response = client2.post('/jobbrowser/jobs/%s/kill' % (hadoop_job_id, ))
assert_equal(
"Permission denied. User test_non_superuser cannot delete user test's job.",
response.context["error"])
self.client.post('/jobbrowser/jobs/%s/kill' % (hadoop_job_id, ))
# It should say killed
response = self.client.get('/jobbrowser/jobs/%s' % (hadoop_job_id, ))
html = response.content.lower()
assert_true(hadoop_job_id in html)
assert_true('killed' in html)
# Exercise select by taskstate
self.client.get('/jobbrowser/jobs/%s/tasks?taskstate=failed' %
(hadoop_job_id, ))
self.client.get('/jobbrowser/jobs/%s/tasks?taskstate=pending' %
(hadoop_job_id, ))
self.client.get('/jobbrowser/jobs/%s/tasks?taskstate=succeeded' %
(hadoop_job_id, ))
self.client.get('/jobbrowser/jobs/%s/tasks?taskstate=running' %
(hadoop_job_id, ))
self.client.get('/jobbrowser/jobs/%s/tasks?taskstate=killed' %
(hadoop_job_id, ))
# Test single task page
late_task_id = hadoop_job_id.replace('job', 'task') + '_r_001999'
response = self.client.get('/jobbrowser/jobs/%s/tasks/%s' %
(hadoop_job_id, late_task_id))
assert_false('succeed' in response.content)
assert_true('killed' in response.content)
# The first task should've succeeded
early_task_id = hadoop_job_id.replace('job', 'task') + '_m_000000'
response = self.client.get('/jobbrowser/jobs/%s/tasks/%s' %
(hadoop_job_id, early_task_id))
assert_true('succeed' in response.content)
assert_false('failed' in response.content)
# Test single attempt page
attempt_id = early_task_id.replace('task', 'attempt') + '_0'
response = self.client.get('/jobbrowser/jobs/%s/tasks/%s/attempts/%s' %
(hadoop_job_id, early_task_id, attempt_id))
assert_true('syslog' in response.content)
# Test dock jobs
response = self.client.get('/jobbrowser/dock_jobs/')
assert_true('completed' in response.content)
# TODO(atm): I'm pretty sure the following test only passes because of
# failed jobs which are run in test_failed_jobs
assert_true('failed' in response.content)
def list_designs(request):
'''
List all workflow designs. Result sorted by last modification time.
Query params:
owner - Substring filter by owner field
name - Substring filter by design name field
'''
data = models.OozieDesign.objects
owner = request.GET.get('owner', '')
name = request.GET.get('name', '')
if owner:
data = data.filter(owner__username__icontains=owner)
if name:
data = data.filter(name__icontains=name)
data = data.order_by('-last_modified')
show_install_examples = \
request.user.is_superuser and not jobsub_setup.Command().has_been_setup()
designs = []
for design in data:
ko_design = {
'id':
design.id,
'owner':
design.owner.username,
'name':
design.name,
'description':
design.description,
'type':
design.root_action.action_type,
'last_modified':
py_time.mktime(design.last_modified.timetuple()),
'url_params':
urlresolvers.reverse(jobsub.views.get_design_params,
kwargs={'design_id': design.id}),
'url_submit':
urlresolvers.reverse(jobsub.views.submit_design,
kwargs={'design_id': design.id}),
'url_edit':
urlresolvers.reverse(jobsub.views.edit_design,
kwargs={'design_id': design.id}),
'url_delete':
urlresolvers.reverse(jobsub.views.delete_design,
kwargs={'design_id': design.id}),
'url_clone':
urlresolvers.reverse(jobsub.views.clone_design,
kwargs={'design_id': design.id}),
'can_submit':
request.user.username == design.owner.username,
'can_delete':
request.user.is_superuser
or request.user.username == design.owner.username
}
designs.append(ko_design)
return render(
"list_designs.mako", request, {
'currentuser': request.user,
'owner': owner,
'name': name,
'designs': json.dumps(designs),
'show_install_examples': show_install_examples,
})
def test_jobsub_setup_and_samples():
"""
Merely exercises jobsub_setup, and then runs
all the examples.
"""
cluster = mini_cluster.shared_cluster(conf=True)
jobsubd = in_process_jobsubd(cluster.config_dir)
try:
c = make_logged_in_client()
# Create a job, to make sure that it sticks around
response = c.post("/jobsub/new/jar", dict(
name="should_stick_around",
jarfile="foo",
arguments="foo", submit="Save"))
design_id = response.context["saved"]
import jobsub.management.commands.jobsub_setup as jobsub_setup
if not jobsub_setup.Command().has_been_setup():
jobsub_setup.Command().handle()
# Make sure we have three job designs now.
assert_equal(3, JobDesign.objects.filter(name__startswith="Example: ").count())
# Make sure "should_stick_around" is still there
assert_equal(1, JobDesign.objects.filter(name="should_stick_around").count())
# Make sure sample user got created.
assert_equal(1, User.objects.filter(username="******").count())
assert_equal(1, User.objects.filter(username="******").count())
# And now submit and run the samples
# pi Example
# Irritatingly, /user/test needs to exist first
setup_cluster_fs(cluster)
id = JobDesign.objects.get(name__contains="Example: Pi").id
response = c.get("/jobsub/submit/%d" % id)
assert_true("Iterations per mapper" in response.content)
assert_true("Num of mappers" in response.content)
response = c.post("/jobsub/submit/%d" % id, dict(
iterations_per_mapper=10,
num_of_mappers=1))
response = watch_till_complete(c, parse_out_id(response))
assert_true("Estimated value of Pi is" in response.context["job_data"].stdout_tail)
assert_true("bin/hadoop returned 0" in response.content)
# Wordcount example
id = JobDesign.objects.get(name__contains="Example: Streaming Wordcount").id
response = c.get("/jobsub/submit/%d" % id)
response = c.post("/jobsub/submit/%d" % id, dict(
output="/user/test/jobsub-streaming-test"))
response = watch_till_complete(c, parse_out_id(response))
assert_true("streaming.StreamJob: Job complete:" in response.context["job_data"].stderr_tail)
assert_true(cluster.fs.exists("/user/test/jobsub-streaming-test/part-00000"))
# Not running sleep example, since it adds little.
finally:
jobsubd.exit()
cluster.shutdown()