def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({
'jobTracker': jobtracker_addr,
'nameNode': fs_defaultfs,
})
if self.job and deployment_dir:
self.properties.update({
self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties['credentials'] = credentials.get_properties()
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({
'jobTracker': jobtracker_addr,
'nameNode': fs_defaultfs,
})
if self.job and deployment_dir:
self.properties.update({
self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties['credentials'] = credentials.get_properties()
self._update_credentials_from_hive_action(credentials)
def test_gen_properties(self):
creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
metastore = {
'thrift_uri': 'thrift://hue-koh-chang:9999',
'kerberos_principal': 'hive',
}
finish = (
beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
)
try:
assert_equal({
'hcat': {
'xml_name': 'hcat',
'properties': [
('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
('hcat.metastore.principal', 'hive')
]},
'hive2': {
'xml_name': 'hive2',
'properties': [
('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'),
('hive2.server.principal', 'hive')
]},
'hbase': {
'xml_name': 'hbase',
'properties': []
}
}, creds.get_properties(metastore))
finally:
for f in finish:
f()
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info("Using FS %s and JT %s" % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({"jobTracker": jobtracker_addr, "nameNode": fs_defaultfs})
if self.job and deployment_dir:
self.properties.update(
{
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id,
}
)
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties["credentials"] = credentials.get_properties()
def test_update_credentials_from_hive_action_when_jdbc_url_is_variable(
self):
class TestJob(object):
XML_FILE_NAME = 'workflow.xml'
def __init__(self):
self.deployment_dir = '/tmp/test'
self.nodes = [
Node({
'id': '1',
'type': 'hive-document',
'properties': {
'jdbc_url':
u"${wf:actionData('shell-31b5')['hiveserver']}",
'password': u'test'
}
})
]
user = User.objects.get(username='******')
submission = Submission(user,
job=TestJob(),
fs=MockFs(logical_name='fsname'),
jt=MockJt(logical_name='jtname'))
finish = (
beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
)
try:
creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
hive_properties = {
'thrift_uri': 'thrift://*****:*****@test-realm.com',
}
submission.properties['credentials'] = creds.get_properties(
hive_properties)
submission._update_credentials_from_hive_action(creds)
assert_equal(
submission.properties['credentials'][
creds.hiveserver2_name]['properties'],
[('hive2.jdbc.url',
u'jdbc:hive2://hue-koh-chang:12345/default'),
('hive2.server.principal', u'hive/[email protected]')
])
finally:
for f in finish:
f()
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def test_gen_properties(self):
creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
hive_properties = {
'thrift_uri': 'thrift://hue-koh-chang:9999',
'kerberos_principal': 'hive',
'hive2.server.principal': 'hive',
}
finish = (
beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
)
try:
assert_equal(
{
'hcat': {
'xml_name':
'hcat',
'properties':
[('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
('hcat.metastore.principal', 'hive')]
},
'hive2': {
'xml_name':
'hive2',
'properties':
[('hive2.jdbc.url',
'jdbc:hive2://hue-koh-chang:12345/default'),
('hive2.server.principal', 'hive')]
},
'hbase': {
'xml_name': 'hbase',
'properties': []
}
}, creds.get_properties(hive_properties))
finally:
for f in finish:
f()
def _update_properties(self, jobtracker_addr, deployment_dir):
if self.fs and self.jt:
self.properties.update(
{
"jobTracker": self.jt.logical_name or jobtracker_addr,
"nameNode": self.fs.logical_name or self.fs.fs_defaultfs,
}
)
if self.job:
self.properties.update(
{
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id,
}
)
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties["credentials"] = credentials.get_properties()