def test_convert_pig_script(self): attrs = { 'user': self.user, 'id': 1000, 'name': 'Test', 'script': 'A = LOAD "$data"; STORE A INTO "$output";', 'parameters': [], 'resources': [], 'hadoopProperties': [] } pig_script = create_or_update_script(**attrs) try: # Test that corresponding doc2 is created after convert assert_false( Document2.objects.filter(owner=self.user, type='link-pigscript').exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type='link-pigscript') # Verify absolute_url response = self.client.get(doc2.get_absolute_url()) assert_equal(200, response.status_code) finally: pig_script.delete()
def test_convert_pig_script(self): attrs = { 'user': self.user, 'id': 1000, 'name': 'Test', 'script': 'A = LOAD "$data"; STORE A INTO "$output";', 'parameters': [], 'resources': [], 'hadoopProperties': [] } pig_script = create_or_update_script(**attrs) try: # Test that corresponding doc2 is created after convert assert_false(Document2.objects.filter(owner=self.user, type='link-pigscript').exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type='link-pigscript') # Verify absolute_url response = self.client.get(doc2.get_absolute_url()) assert_equal(200, response.status_code) finally: pig_script.delete()
def run(request): if request.method != 'POST': raise PopupException(_('POST request required.')) attrs = { 'id': request.POST.get('id'), 'name': request.POST.get('name'), 'script': request.POST.get('script'), 'user': request.user, 'parameters': json.loads(request.POST.get('parameters')), 'resources': json.loads(request.POST.get('resources')), 'hadoopProperties': json.loads(request.POST.get('hadoopProperties')), 'is_design': False } pig_script = create_or_update_script(**attrs) params = request.POST.get('submissionVariables') oozie_id = api.get(request.fs, request.jt, request.user).submit(pig_script, params) pig_script.update_from_dict({'job_id': oozie_id}) pig_script.save() response = { 'id': pig_script.id, 'watchUrl': reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python' } return HttpResponse(json.dumps(response), content_type="text/plain")
def run(request): if request.method != "POST": raise PopupException(_("POST request required.")) attrs = { "id": request.POST.get("id"), "name": request.POST.get("name"), "script": request.POST.get("script"), "user": request.user, "parameters": json.loads(request.POST.get("parameters")), "resources": json.loads(request.POST.get("resources")), "hadoopProperties": json.loads(request.POST.get("hadoopProperties")), "is_design": False, } pig_script = create_or_update_script(**attrs) params = request.POST.get("submissionVariables") oozie_id = api.get(request.fs, request.jt, request.user).submit(pig_script, params) pig_script.update_from_dict({"job_id": oozie_id}) pig_script.save() response = {"id": pig_script.id, "watchUrl": reverse("pig:watch", kwargs={"job_id": oozie_id}) + "?format=python"} return HttpResponse(json.dumps(response), content_type="text/plain")
def test_convert_pig_script(self): attrs = { "user": self.user, "id": 1000, "name": "Test", "script": 'A = LOAD "$data"; STORE A INTO "$output";', "parameters": [], "resources": [], "hadoopProperties": [], } pig_script = create_or_update_script(**attrs) try: # Test that corresponding doc2 is created after convert assert_false(Document2.objects.filter(owner=self.user, type="link-pigscript").exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type="link-pigscript") # Verify absolute_url response = self.client.get(doc2.get_absolute_url()) assert_equal(200, response.status_code) finally: pig_script.delete()
def test_convert_pig_script(self): attrs = { 'user': self.user, 'id': 1000, 'name': 'Test', 'script': 'A = LOAD "$data"; STORE A INTO "$output";', 'hadoopProperties': [ {u'name': u'mapred.job.queue.name', u'value': u'pig'}, {u'name': u'mapreduce.task.profile', u'value': u'true'} ], 'parameters': [ {u'name': u'input', u'value': u'/user/test/data'}, {u'name': u'verbose', u'value': u'true'} ], 'resources': [ {u'type': u'file', u'value': u'/user/test/test.txt'}, {u'type': u'archive', u'value': u'/user/test/test.jar'} ], } pig_script = create_or_update_script(**attrs) pig_script.save() # Setting doc.last_modified to older date doc = Document.objects.get(id=pig_script.doc.get().id) Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ')) doc = Document.objects.get(id=doc.id) try: if IS_HUE_4.get(): # Test that corresponding doc2 is created after convert assert_false(Document2.objects.filter(owner=self.user, type='query-pig').exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type='query-pig') # Verify snippet values assert_equal('ready', doc2.data_dict['snippets'][0]['status']) assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement'], doc2.data_dict) assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement_raw']) assert_equal(['mapred.job.queue.name=pig', 'mapreduce.task.profile=true'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties']) assert_equal(['input=/user/test/data', 'verbose=true'], doc2.data_dict['snippets'][0]['properties']['parameters']) assert_equal(['/user/test/test.txt', '/user/test/test.jar'], doc2.data_dict['snippets'][0]['properties']['resources']) else: # Test that corresponding doc2 is created after convert assert_false(Document2.objects.filter(owner=self.user, type='link-pigscript').exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type='link-pigscript') # Verify absolute_url response = self.client.get(doc2.get_absolute_url()) assert_equal(200, response.status_code) assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S')) finally: pig_script.delete()
def create_script(self): attrs = { 'id': 1000, 'name': 'Test', 'script': 'A = LOAD "$data"; STORE A INTO "$output";', 'user': self.user, 'parameters': [], 'resources': [], } return create_or_update_script(**attrs)
def save(request): if request.method != 'POST': raise PopupException(_('POST request required.')) pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user) pig_script.is_design = True pig_script.save() response = { 'id': pig_script.id, } return HttpResponse(json.dumps(response), content_type="text/plain")
def create_script(user, xattrs=None): attrs = {'user': user} attrs.update(TestPigBase.SCRIPT_ATTRS) if xattrs is not None: attrs.update(xattrs) return create_or_update_script(**attrs) def make_log_links(self): # FileBrowser assert_equal( """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a> <dir>""", OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp <dir>') ) assert_equal( """<a href="/filebrowser/view/user/romain/tmp" target="_blank">hdfs://localhost:8020/user/romain/tmp</a><dir>""", OozieApi._make_links('hdfs://localhost:8020/user/romain/tmp<dir>') ) assert_equal( """output: <a href="/filebrowser/view/user/romain/tmp" target="_blank">/user/romain/tmp</a> <dir>""", OozieApi._make_links('output: /user/romain/tmp <dir>') ) assert_equal( 'Successfully read 3760 records (112648 bytes) from: "<a href="/filebrowser/view/user/hue/pig/examples/data/midsummer.txt" target="_blank">/user/hue/pig/examples/data/midsummer.txt</a>"', OozieApi._make_links('Successfully read 3760 records (112648 bytes) from: "/user/hue/pig/examples/data/midsummer.txt"') ) assert_equal( 'data,upper_case MAP_ONLY <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>,', OozieApi._make_links('data,upper_case MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff,') ) assert_equal( 'MAP_ONLY <a href="/filebrowser/view/user/romain/out/fffff" target="_blank">hdfs://localhost:8020/user/romain/out/fffff</a>\n2013', OozieApi._make_links('MAP_ONLY hdfs://localhost:8020/user/romain/out/fffff\n2013') ) # JobBrowser assert_equal( """<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""", OozieApi._make_links('job_201306261521_0058') ) assert_equal( """Hadoop Job IDs executed by Pig: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""", OozieApi._make_links('Hadoop Job IDs executed by Pig: job_201306261521_0058') ) assert_equal( """MapReduceLauncher - HadoopJobId: <a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""", OozieApi._make_links('MapReduceLauncher - HadoopJobId: job_201306261521_0058') ) assert_equal( """- More information at: http://localhost:50030/jobdetails.jsp?jobid=<a href="/jobbrowser/jobs/job_201306261521_0058" target="_blank">job_201306261521_0058</a>""", OozieApi._make_links('- More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201306261521_0058') )
def run(request): pig_script = create_or_update_script(request.POST.get('id'), request.POST.get('name'), request.POST.get('script'), request.user, is_design=False) params = {} oozie_id = api.get(request.fs, request.user).submit(pig_script, params) pig_script.update_from_dict({'job_id': oozie_id}) pig_script.save() response = { 'id': pig_script.id, 'watchUrl': reverse('pig:watch', kwargs={'job_id': oozie_id}) + '?format=python' } return HttpResponse(json.dumps(response), content_type="text/plain")
def test_convert_pig_script(self): attrs = { 'user': self.user, 'id': 1000, 'name': 'Test', 'script': 'A = LOAD "$data"; STORE A INTO "$output";', 'hadoopProperties': [ {u'name': u'mapred.job.queue.name', u'value': u'pig'}, {u'name': u'mapreduce.task.profile', u'value': u'true'} ], 'parameters': [ {u'name': u'input', u'value': u'/user/test/data'}, {u'name': u'verbose', u'value': u'true'} ], 'resources': [ {u'type': u'file', u'value': u'/user/test/test.txt'}, {u'type': u'archive', u'value': u'/user/test/test.jar'} ], } pig_script = create_or_update_script(**attrs) pig_script.save() # Setting doc.last_modified to older date doc = Document.objects.get(id=pig_script.doc.get().id) Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ')) doc = Document.objects.get(id=doc.id) try: # Test that corresponding doc2 is created after convert assert_false(Document2.objects.filter(owner=self.user, type='query-pig').exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type='query-pig') # Verify snippet values assert_equal('ready', doc2.data_dict['snippets'][0]['status']) assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement'], doc2.data_dict) assert_equal(attrs['script'], doc2.data_dict['snippets'][0]['statement_raw']) assert_equal(['mapred.job.queue.name=pig', 'mapreduce.task.profile=true'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties']) assert_equal(['input=/user/test/data', 'verbose=true'], doc2.data_dict['snippets'][0]['properties']['parameters']) assert_equal(['/user/test/test.txt', '/user/test/test.jar'], doc2.data_dict['snippets'][0]['properties']['resources']) finally: pig_script.delete()
def save(request): if request.method != "POST": raise PopupException(_("POST request required.")) attrs = { "id": request.POST.get("id"), "name": request.POST.get("name"), "script": request.POST.get("script"), "user": request.user, "parameters": json.loads(request.POST.get("parameters")), "resources": json.loads(request.POST.get("resources")), } pig_script = create_or_update_script(**attrs) pig_script.is_design = True pig_script.save() response = {"id": pig_script.id} return HttpResponse(json.dumps(response), content_type="text/plain")
def save(request): if request.method != 'POST': raise PopupException(_('POST request required.')) attrs = { 'id': request.POST.get('id'), 'name': request.POST.get('name'), 'script': request.POST.get('script'), 'user': request.user, 'parameters': json.loads(request.POST.get('parameters')), 'resources': json.loads(request.POST.get('resources')), 'hadoopProperties': json.loads(request.POST.get('hadoopProperties')), } pig_script = create_or_update_script(**attrs) pig_script.is_design = True pig_script.save() response = {'id': pig_script.id, 'docId': pig_script.doc.get().id} return HttpResponse(json.dumps(response), content_type="text/plain")
def save(request): if request.method != 'POST': raise PopupException(_('POST request required.')) attrs = { 'id': request.POST.get('id'), 'name': request.POST.get('name'), 'script': request.POST.get('script'), 'user': request.user, 'parameters': json.loads(request.POST.get('parameters')), 'resources': json.loads(request.POST.get('resources')), } pig_script = create_or_update_script(**attrs) pig_script.is_design = True pig_script.save() response = { 'id': pig_script.id, } return HttpResponse(json.dumps(response), content_type="text/plain")
def test_convert_pig_script(self): attrs = { 'user': self.user, 'id': 1000, 'name': 'Test', 'script': 'A = LOAD "$data"; STORE A INTO "$output";', 'parameters': [], 'resources': [], 'hadoopProperties': [] } pig_script = create_or_update_script(**attrs) # Setting doc.last_modified to older date doc = Document.objects.get(id=pig_script.doc.get().id) Document.objects.filter( id=doc.id).update(last_modified=datetime.strptime( '2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ')) doc = Document.objects.get(id=doc.id) try: # Test that corresponding doc2 is created after convert assert_false( Document2.objects.filter(owner=self.user, type='link-pigscript').exists()) converter = DocumentConverter(self.user) converter.convert() doc2 = Document2.objects.get(owner=self.user, type='link-pigscript') # Verify absolute_url response = self.client.get(doc2.get_absolute_url()) assert_equal(200, response.status_code) assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S')) finally: pig_script.delete()
def create_script(user, xattrs=None): attrs = {'user': user} attrs.update(TestPigBase.SCRIPT_ATTRS) if xattrs is not None: attrs.update(xattrs) return create_or_update_script(**attrs)
def create_script(self): attrs = {"user": self.user} attrs.update(TestPigBase.SCRIPT_ATTRS) return create_or_update_script(**attrs)
def create_script(self): attrs = {'user': self.user,} attrs.update(TestPigBase.SCRIPT_ATTRS) return create_or_update_script(**attrs)