def _update_properties(self, jobtracker_addr, deployment_dir=None): LOG.info('Using FS %s and JT %s' % (self.fs, self.jt)) if self.jt and self.jt.logical_name: jobtracker_addr = self.jt.logical_name if self.fs.logical_name: fs_defaultfs = self.fs.logical_name else: fs_defaultfs = self.fs.fs_defaultfs self.properties.update({ 'jobTracker': jobtracker_addr, 'nameNode': fs_defaultfs, }) if self.job and deployment_dir: self.properties.update({ self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id }) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties['credentials'] = credentials.get_properties()
def _update_properties(self, jobtracker_addr, deployment_dir=None): LOG.info('Using FS %s and JT %s' % (self.fs, self.jt)) if self.jt and self.jt.logical_name: jobtracker_addr = self.jt.logical_name if self.fs.logical_name: fs_defaultfs = self.fs.logical_name else: fs_defaultfs = self.fs.fs_defaultfs self.properties.update({ 'jobTracker': jobtracker_addr, 'nameNode': fs_defaultfs, }) if self.job and deployment_dir: self.properties.update({ self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id }) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties['credentials'] = credentials.get_properties() self._update_credentials_from_hive_action(credentials)
def test_gen_properties(self): creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy()) metastore = { 'thrift_uri': 'thrift://hue-koh-chang:9999', 'kerberos_principal': 'hive', } finish = ( beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'), beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345), ) try: assert_equal({ 'hcat': { 'xml_name': 'hcat', 'properties': [ ('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'), ('hcat.metastore.principal', 'hive') ]}, 'hive2': { 'xml_name': 'hive2', 'properties': [ ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'), ('hive2.server.principal', 'hive') ]}, 'hbase': { 'xml_name': 'hbase', 'properties': [] } }, creds.get_properties(metastore)) finally: for f in finish: f()
def _update_properties(self, jobtracker_addr, deployment_dir=None): LOG.info("Using FS %s and JT %s" % (self.fs, self.jt)) if self.jt and self.jt.logical_name: jobtracker_addr = self.jt.logical_name if self.fs.logical_name: fs_defaultfs = self.fs.logical_name else: fs_defaultfs = self.fs.fs_defaultfs self.properties.update({"jobTracker": jobtracker_addr, "nameNode": fs_defaultfs}) if self.job and deployment_dir: self.properties.update( { self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id, } ) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties["credentials"] = credentials.get_properties()
def test_update_credentials_from_hive_action_when_jdbc_url_is_variable( self): class TestJob(object): XML_FILE_NAME = 'workflow.xml' def __init__(self): self.deployment_dir = '/tmp/test' self.nodes = [ Node({ 'id': '1', 'type': 'hive-document', 'properties': { 'jdbc_url': u"${wf:actionData('shell-31b5')['hiveserver']}", 'password': u'test' } }) ] user = User.objects.get(username='******') submission = Submission(user, job=TestJob(), fs=MockFs(logical_name='fsname'), jt=MockJt(logical_name='jtname')) finish = ( beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'), beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345), ) try: creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy()) hive_properties = { 'thrift_uri': 'thrift://*****:*****@test-realm.com', } submission.properties['credentials'] = creds.get_properties( hive_properties) submission._update_credentials_from_hive_action(creds) assert_equal( submission.properties['credentials'][ creds.hiveserver2_name]['properties'], [('hive2.jdbc.url', u'jdbc:hive2://hue-koh-chang:12345/default'), ('hive2.server.principal', u'hive/[email protected]') ]) finally: for f in finish: f()
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def test_gen_properties(self): creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy()) hive_properties = { 'thrift_uri': 'thrift://hue-koh-chang:9999', 'kerberos_principal': 'hive', 'hive2.server.principal': 'hive', } finish = ( beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'), beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345), ) try: assert_equal( { 'hcat': { 'xml_name': 'hcat', 'properties': [('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'), ('hcat.metastore.principal', 'hive')] }, 'hive2': { 'xml_name': 'hive2', 'properties': [('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'), ('hive2.server.principal', 'hive')] }, 'hbase': { 'xml_name': 'hbase', 'properties': [] } }, creds.get_properties(hive_properties)) finally: for f in finish: f()
def _update_properties(self, jobtracker_addr, deployment_dir): if self.fs and self.jt: self.properties.update( { "jobTracker": self.jt.logical_name or jobtracker_addr, "nameNode": self.fs.logical_name or self.fs.fs_defaultfs, } ) if self.job: self.properties.update( { self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id, } ) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties["credentials"] = credentials.get_properties()