def get_properties(self, hive_properties=None):
credentials = {}
from beeswax import hive_site, conf
if hive_properties is None:
hive_properties = hive_site.get_metastore()
if hive_properties:
hive_properties['hive2.server.principal'] = hive_site.get_hiveserver2_kerberos_principal(conf.HIVE_SERVER_HOST.get())
if not hive_properties:
hive_properties = {}
LOG.warn('Could not get all the Oozie credentials: hive-site.xml required on the Hue host.')
credentials[self.hive_name] = {
'xml_name': self.hive_name,
'properties': [
('hcat.metastore.uri', hive_properties.get('thrift_uri')),
('hcat.metastore.principal', hive_properties.get('kerberos_principal')),
]
}
credentials[self.hiveserver2_name] = {
'xml_name': self.hiveserver2_name,
'properties': [
('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()),
('hive2.server.principal', hive_properties.get('hive2.server.principal')),
]
}
credentials[self.hbase_name] = {
'xml_name': self.hbase_name,
'properties': []
}
return credentials
def _update_properties(self, jobtracker_addr, deployment_dir):
if self.fs and self.jt:
self.properties.update({
'jobTracker':
self.jt.logical_name or jobtracker_addr,
'nameNode':
self.fs.logical_name or self.fs.fs_defaultfs,
})
if self.job:
self.properties.update({
self.job.get_application_path_key():
self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID:
self.job.id
})
# Even if no Hive action for now
from beeswax.hive_site import get_metastore
metastore = get_metastore()
if metastore and metastore.get(
'use_sasl') and False: # Disabled for now
self.properties.update({
'is_kerberized_hive':
True,
'credential_type':
'hcat',
'thrift_server':
metastore.get('thrift_uri'),
'hive_principal':
metastore.get('kerberos_principal')
})
def get_properties(self, metastore=None):
credentials = {}
from beeswax import hive_site
if metastore is None:
metastore = hive_site.get_metastore()
if not metastore:
metastore = {}
LOG.warn('Could not get all the Oozie credentials: hive-site.xml required on the Hue host.')
credentials[self.hive_name] = {
'xml_name': self.hive_name,
'properties': [
('hcat.metastore.uri', metastore.get('thrift_uri')),
('hcat.metastore.principal', metastore.get('kerberos_principal')),
]
}
credentials[self.hiveserver2_name] = {
'xml_name': self.hiveserver2_name,
'properties': [
('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()),
('hive2.server.principal', metastore.get('kerberos_principal')),
]
}
credentials[self.hbase_name] = {
'xml_name': self.hbase_name,
'properties': []
}
LOG.info('Using credentials: %s' % credentials)
return credentials
def get_properties(self, hive_properties=None):
credentials = {}
from beeswax import hive_site, conf
if not hasattr(conf.HIVE_SERVER_HOST,
'get') or not conf.HIVE_SERVER_HOST.get():
LOG.warn(
'Could not get all the Oozie credentials: beeswax app is blacklisted.'
)
else:
if hive_properties is None:
hive_properties = hive_site.get_metastore()
if hive_properties:
hive_properties[
'hive2.server.principal'] = hive_site.get_hiveserver2_kerberos_principal(
conf.HIVE_SERVER_HOST.get())
if not hive_properties:
hive_properties = {}
LOG.warn(
'Could not get all the Oozie credentials: hive-site.xml required on the Hue host.'
)
credentials[self.hive_name] = {
'xml_name':
self.hive_name,
'properties': [
('hcat.metastore.uri', hive_properties.get('thrift_uri')),
('hcat.metastore.principal',
hive_properties.get('kerberos_principal')),
]
}
credentials[self.hiveserver2_name] = {
'xml_name':
self.hiveserver2_name,
'properties': [
('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()),
('hive2.server.principal',
hive_properties.get('hive2.server.principal')),
]
}
credentials[self.hbase_name] = {
'xml_name': self.hbase_name,
'properties': []
}
return credentials
def get_properties(self, metastore=None):
credentials = {}
from beeswax import hive_site
if metastore is None:
metastore = hive_site.get_metastore()
if not metastore:
metastore = {}
LOG.warn(
'Could not get all the Oozie credentials: hive-site.xml required on the Hue host.'
)
credentials[self.hive_name] = {
'xml_name':
self.hive_name,
'properties': [
('hcat.metastore.uri', metastore.get('thrift_uri')),
('hcat.metastore.principal',
metastore.get('kerberos_principal')),
]
}
credentials[self.hiveserver2_name] = {
'xml_name':
self.hiveserver2_name,
'properties': [
('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()),
('hive2.server.principal',
metastore.get('kerberos_principal')),
]
}
credentials[self.hbase_name] = {
'xml_name': self.hbase_name,
'properties': []
}
LOG.info('Using credentials: %s' % credentials)
return credentials
def _update_properties(self, jobtracker_addr, deployment_dir):
if self.fs and self.jt:
self.properties.update({
'jobTracker': self.jt.logical_name or jobtracker_addr,
'nameNode': self.fs.logical_name or self.fs.fs_defaultfs,
})
if self.job:
self.properties.update({
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Even if no Hive action for now
from beeswax.hive_site import get_metastore
metastore = get_metastore()
if metastore and metastore.get('use_sasl') and False: # Disabled for now
self.properties.update({
'is_kerberized_hive': True,
'credential_type': 'hcat',
'thrift_server': metastore.get('thrift_uri'),
'hive_principal': metastore.get('kerberos_principal')
})
def meta_client():
"""Get the Thrift client to talk to the metastore"""
class UnicodeMetastoreClient(object):
"""Wrap the thrift client to take and return Unicode."""
def __init__(self, client):
self._client = client
def __getattr__(self, attr):
if attr in self.__dict__:
return self.__dict__[attr]
return getattr(self._client, attr)
def _encode_storage_descriptor(self, sd):
_encode_struct_attr(sd, 'location')
for col in sd.cols:
_encode_struct_attr(col, 'comment')
self._encode_map(sd.parameters)
def _decode_storage_descriptor(self, sd):
_decode_struct_attr(sd, 'location')
for col in sd.cols:
_decode_struct_attr(col, 'comment')
self._decode_map(sd.parameters)
def _encode_map(self, mapp):
for key, value in mapp.iteritems():
mapp[key] = smart_str(value, strings_only=True)
def _decode_map(self, mapp):
for key, value in mapp.iteritems():
mapp[key] = force_unicode(value, strings_only=True, errors='replace')
def create_database(self, name, description):
description = smart_str(description)
return self._client.create_database(name, description)
def get_database(self, *args, **kwargs):
db = self._client.get_database(*args, **kwargs)
return _decode_struct_attr(db, 'description')
def get_fields(self, *args, **kwargs):
res = self._client.get_fields(*args, **kwargs)
for fschema in res:
_decode_struct_attr(fschema, 'comment')
return res
def get_table(self, *args, **kwargs):
res = self._client.get_table(*args, **kwargs)
self._decode_storage_descriptor(res.sd)
self._decode_map(res.parameters)
return res
def alter_table(self, dbname, tbl_name, new_tbl):
self._encode_storage_descriptor(new_tbl.sd)
self._encode_map(new_tbl.parameters)
return self._client.alter_table(dbname, tbl_name, new_tbl)
def _encode_partition(self, part):
self._encode_storage_descriptor(part.sd)
self._encode_map(part.parameters)
return part
def _decode_partition(self, part):
self._decode_storage_descriptor(part.sd)
self._decode_map(part.parameters)
return part
def add_partition(self, new_part):
self._encode_partition(new_part)
part = self._client.add_partition(new_part)
return self._decode_partition(part)
def get_partition(self, *args, **kwargs):
part = self._client.get_partition(*args, **kwargs)
return self._decode_partition(part)
def get_partitions(self, *args, **kwargs):
part_list = self._client.get_partitions(*args, **kwargs)
for part in part_list:
self._decode_partition(part)
return part_list
def alter_partition(self, db_name, tbl_name, new_part):
self._encode_partition(new_part)
return self._client.alter_partition(db_name, tbl_name, new_part)
_, host, port = hive_site.get_metastore()
client = thrift_util.get_client(ThriftHiveMetastore.Client,
host,
port,
service_name="Hive Metastore Server",
timeout_seconds=conf.METASTORE_CONN_TIMEOUT.get())
return UnicodeMetastoreClient(client)
def meta_client(self):
"""Get the Thrift client to talk to the metastore"""
class UnicodeMetastoreClient(object):
"""Wrap the thrift client to take and return Unicode."""
def __init__(self, client):
self._client = client
def __getattr__(self, attr):
if attr in self.__dict__:
return self.__dict__[attr]
return getattr(self._client, attr)
def _encode_storage_descriptor(self, sd):
_encode_struct_attr(sd, 'location')
for col in sd.cols:
_encode_struct_attr(col, 'comment')
self._encode_map(sd.parameters)
def _decode_storage_descriptor(self, sd):
_decode_struct_attr(sd, 'location')
for col in sd.cols:
_decode_struct_attr(col, 'comment')
self._decode_map(sd.parameters)
def _encode_map(self, mapp):
for key, value in mapp.iteritems():
mapp[key] = smart_str(value, strings_only=True)
def _decode_map(self, mapp):
for key, value in mapp.iteritems():
mapp[key] = force_unicode(value, strings_only=True, errors='replace')
def create_database(self, name, description):
description = smart_str(description)
return self._client.create_database(name, description)
def get_database(self, *args, **kwargs):
db = self._client.get_database(*args, **kwargs)
return _decode_struct_attr(db, 'description')
def get_fields(self, *args, **kwargs):
res = self._client.get_fields(*args, **kwargs)
for fschema in res:
_decode_struct_attr(fschema, 'comment')
return res
def get_table(self, *args, **kwargs):
res = self._client.get_table(*args, **kwargs)
self._decode_storage_descriptor(res.sd)
self._decode_map(res.parameters)
return res
def alter_table(self, dbname, tbl_name, new_tbl):
self._encode_storage_descriptor(new_tbl.sd)
self._encode_map(new_tbl.parameters)
return self._client.alter_table(dbname, tbl_name, new_tbl)
def _encode_partition(self, part):
self._encode_storage_descriptor(part.sd)
self._encode_map(part.parameters)
return part
def _decode_partition(self, part):
self._decode_storage_descriptor(part.sd)
self._decode_map(part.parameters)
return part
def add_partition(self, new_part):
self._encode_partition(new_part)
part = self._client.add_partition(new_part)
return self._decode_partition(part)
def get_partition(self, *args, **kwargs):
part = self._client.get_partition(*args, **kwargs)
return self._decode_partition(part)
def get_partitions(self, *args, **kwargs):
part_list = self._client.get_partitions(*args, **kwargs)
for part in part_list:
self._decode_partition(part)
return part_list
def alter_partition(self, db_name, tbl_name, new_part):
self._encode_partition(new_part)
return self._client.alter_partition(db_name, tbl_name, new_part)
# Use service name from kerberos principal set in hive-site.xml
_, host, port, metastore_kerberos_principal = hive_site.get_metastore()
use_sasl, kerberos_principal_short_name = BeeswaxClient.get_security()
kerberos_principal_short_name = metastore_kerberos_principal and metastore_kerberos_principal.split('/', 1)[0] or None
client = thrift_util.get_client(ThriftHiveMetastore.Client,
host,
port,
service_name="Hive Metastore Server",
kerberos_principal=kerberos_principal_short_name,
use_sasl=use_sasl,
timeout_seconds=conf.METASTORE_CONN_TIMEOUT.get())
return UnicodeMetastoreClient(client)