def test_convert_pig_script(self):
attrs = {
'user': self.user,
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'parameters': [],
'resources': [],
'hadoopProperties': []
}
pig_script = create_or_update_script(**attrs)
try:
# Test that corresponding doc2 is created after convert
assert_false(
Document2.objects.filter(owner=self.user,
type='link-pigscript').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user,
type='link-pigscript')
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
finally:
pig_script.delete()
def test_convert_pig_script(self):
attrs = {
'user': self.user,
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'parameters': [],
'resources': [],
'hadoopProperties': []
}
pig_script = create_or_update_script(**attrs)
try:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='link-pigscript').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='link-pigscript')
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
finally:
pig_script.delete()
def run(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
attrs = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'script': request.POST.get('script'),
'user': request.user,
'parameters': json.loads(request.POST.get('parameters')),
'resources': json.loads(request.POST.get('resources')),
'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
'is_design': False
}
pig_script = create_or_update_script(**attrs)
params = request.POST.get('submissionVariables')
oozie_id = api.get(request.fs, request.jt, request.user).submit(pig_script, params)
pig_script.update_from_dict({'job_id': oozie_id})
pig_script.save()
response = {
'id': pig_script.id,
'watchUrl': reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python'
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def run(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
attrs = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'script': request.POST.get('script'),
'user': request.user,
'parameters': json.loads(request.POST.get('parameters')),
'resources': json.loads(request.POST.get('resources')),
'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
'is_design': False
}
pig_script = create_or_update_script(**attrs)
params = request.POST.get('submissionVariables')
oozie_id = api.get(request.fs, request.jt,
request.user).submit(pig_script, params)
pig_script.update_from_dict({'job_id': oozie_id})
pig_script.save()
response = {
'id':
pig_script.id,
'watchUrl':
reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python'
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def run(request):
if request.method != "POST":
raise PopupException(_("POST request required."))
attrs = {
"id": request.POST.get("id"),
"name": request.POST.get("name"),
"script": request.POST.get("script"),
"user": request.user,
"parameters": json.loads(request.POST.get("parameters")),
"resources": json.loads(request.POST.get("resources")),
"hadoopProperties": json.loads(request.POST.get("hadoopProperties")),
"is_design": False,
}
pig_script = create_or_update_script(**attrs)
params = request.POST.get("submissionVariables")
oozie_id = api.get(request.fs, request.jt, request.user).submit(pig_script, params)
pig_script.update_from_dict({"job_id": oozie_id})
pig_script.save()
response = {"id": pig_script.id, "watchUrl": reverse("pig:watch", kwargs={"job_id": oozie_id}) + "?format=python"}
return HttpResponse(json.dumps(response), content_type="text/plain")
def test_convert_pig_script(self):
attrs = {
"user": self.user,
"id": 1000,
"name": "Test",
"script": 'A = LOAD "$data"; STORE A INTO "$output";',
"parameters": [],
"resources": [],
"hadoopProperties": [],
}
pig_script = create_or_update_script(**attrs)
try:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type="link-pigscript").exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type="link-pigscript")
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
finally:
pig_script.delete()
def test_convert_pig_script(self):
attrs = {
'user': self.user,
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'hadoopProperties': [
{u'name': u'mapred.job.queue.name', u'value': u'pig'},
{u'name': u'mapreduce.task.profile', u'value': u'true'}
],
'parameters': [
{u'name': u'input', u'value': u'/user/test/data'},
{u'name': u'verbose', u'value': u'true'}
],
'resources': [
{u'type': u'file', u'value': u'/user/test/test.txt'},
{u'type': u'archive', u'value': u'/user/test/test.jar'}
],
}
pig_script = create_or_update_script(**attrs)
pig_script.save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=pig_script.doc.get().id)
Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
if IS_HUE_4.get():
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-pig').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-pig')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement'], doc2.data_dict)
assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement_raw'])
assert_equal(['mapred.job.queue.name=pig', 'mapreduce.task.profile=true'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
assert_equal(['input=/user/test/data', 'verbose=true'], doc2.data_dict['snippets'][0]['properties']['parameters'])
assert_equal(['/user/test/test.txt', '/user/test/test.jar'], doc2.data_dict['snippets'][0]['properties']['resources'])
else:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='link-pigscript').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='link-pigscript')
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
finally:
pig_script.delete()
def create_script(self):
attrs = {
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'user': self.user,
'parameters': [],
'resources': [],
}
return create_or_update_script(**attrs)
def create_script(self):
attrs = {
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'user': self.user,
'parameters': [],
'resources': [],
}
return create_or_update_script(**attrs)
def save(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user)
pig_script.is_design = True
pig_script.save()
response = {
'id': pig_script.id,
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def create_script(user, xattrs=None):
attrs = {'user': user}
attrs.update(TestPigBase.SCRIPT_ATTRS)
if xattrs is not None:
attrs.update(xattrs)
return create_or_update_script(**attrs)
def make_log_links(self):
# FileBrowser
assert_equal(
"""<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a> <dir>""",
OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp <dir>')
)
assert_equal(
"""<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a><dir>""",
OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp<dir>')
)
assert_equal(
"""output: <a href="/filebrowser/view/user/romain/tmp" target="_blank">/user/romain/tmp</a> <dir>""",
OozieApi._make_links('output: /user/romain/tmp <dir>')
)
assert_equal(
'Successfully read 3760 records (112648 bytes) from: "<a href="/filebrowser/view/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>"',
OozieApi._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"')
)
assert_equal(
'data,upper_case MAP_ONLY <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,',
OozieApi._make_links('data,upper_case MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff,')
)
assert_equal(
'MAP_ONLY <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013',
OozieApi._make_links('MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff\n2013')
)
# JobBrowser
assert_equal(
"""<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
OozieApi._make_links('job_201306261521_0058')
)
assert_equal(
"""Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
OozieApi._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058')
)
assert_equal(
"""MapReduceLauncher - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
OozieApi._make_links('MapReduceLauncher - HadoopJobId: job_201306261521_0058')
)
assert_equal(
"""- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""",
OozieApi._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058')
)
def run(request):
pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_design=False)
params = {}
oozie_id = api.get(request.fs, request.user).submit(pig_script, params)
pig_script.update_from_dict({'job_id': oozie_id})
pig_script.save()
response = {
'id': pig_script.id,
'watchUrl': reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python'
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def test_convert_pig_script(self):
attrs = {
'user': self.user,
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'hadoopProperties': [
{u'name': u'mapred.job.queue.name', u'value': u'pig'},
{u'name': u'mapreduce.task.profile', u'value': u'true'}
],
'parameters': [
{u'name': u'input', u'value': u'/user/test/data'},
{u'name': u'verbose', u'value': u'true'}
],
'resources': [
{u'type': u'file', u'value': u'/user/test/test.txt'},
{u'type': u'archive', u'value': u'/user/test/test.jar'}
],
}
pig_script = create_or_update_script(**attrs)
pig_script.save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=pig_script.doc.get().id)
Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-pig').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-pig')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement'], doc2.data_dict)
assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement_raw'])
assert_equal(['mapred.job.queue.name=pig', 'mapreduce.task.profile=true'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
assert_equal(['input=/user/test/data', 'verbose=true'], doc2.data_dict['snippets'][0]['properties']['parameters'])
assert_equal(['/user/test/test.txt', '/user/test/test.jar'], doc2.data_dict['snippets'][0]['properties']['resources'])
finally:
pig_script.delete()
def save(request):
if request.method != "POST":
raise PopupException(_("POST request required."))
attrs = {
"id": request.POST.get("id"),
"name": request.POST.get("name"),
"script": request.POST.get("script"),
"user": request.user,
"parameters": json.loads(request.POST.get("parameters")),
"resources": json.loads(request.POST.get("resources")),
}
pig_script = create_or_update_script(**attrs)
pig_script.is_design = True
pig_script.save()
response = {"id": pig_script.id}
return HttpResponse(json.dumps(response), content_type="text/plain")
def save(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
attrs = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'script': request.POST.get('script'),
'user': request.user,
'parameters': json.loads(request.POST.get('parameters')),
'resources': json.loads(request.POST.get('resources')),
'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
}
pig_script = create_or_update_script(**attrs)
pig_script.is_design = True
pig_script.save()
response = {'id': pig_script.id, 'docId': pig_script.doc.get().id}
return HttpResponse(json.dumps(response), content_type="text/plain")
def save(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
attrs = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'script': request.POST.get('script'),
'user': request.user,
'parameters': json.loads(request.POST.get('parameters')),
'resources': json.loads(request.POST.get('resources')),
}
pig_script = create_or_update_script(**attrs)
pig_script.is_design = True
pig_script.save()
response = {
'id': pig_script.id,
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def test_convert_pig_script(self):
attrs = {
'user': self.user,
'id': 1000,
'name': 'Test',
'script': 'A = LOAD "$data"; STORE A INTO "$output";',
'parameters': [],
'resources': [],
'hadoopProperties': []
}
pig_script = create_or_update_script(**attrs)
# Setting doc.last_modified to older date
doc = Document.objects.get(id=pig_script.doc.get().id)
Document.objects.filter(
id=doc.id).update(last_modified=datetime.strptime(
'2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
# Test that corresponding doc2 is created after convert
assert_false(
Document2.objects.filter(owner=self.user,
type='link-pigscript').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user,
type='link-pigscript')
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'),
doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
finally:
pig_script.delete()
def create_script(user, xattrs=None):
attrs = {'user': user}
attrs.update(TestPigBase.SCRIPT_ATTRS)
if xattrs is not None:
attrs.update(xattrs)
return create_or_update_script(**attrs)
def create_script(self):
attrs = {"user": self.user}
attrs.update(TestPigBase.SCRIPT_ATTRS)
return create_or_update_script(**attrs)
def create_script(user, xattrs=None):
attrs = {'user': user}
attrs.update(TestPigBase.SCRIPT_ATTRS)
if xattrs is not None:
attrs.update(xattrs)
return create_or_update_script(**attrs)
def create_script(self):
attrs = {'user': self.user,}
attrs.update(TestPigBase.SCRIPT_ATTRS)
return create_or_update_script(**attrs)
