def get_properties(self, hive_properties=None): credentials = {} from beeswax import hive_site, conf if hive_properties is None: hive_properties = hive_site.get_metastore() if hive_properties: hive_properties['hive2.server.principal'] = hive_site.get_hiveserver2_kerberos_principal(conf.HIVE_SERVER_HOST.get()) if not hive_properties: hive_properties = {} LOG.warn('Could not get all the Oozie credentials: hive-site.xml required on the Hue host.') credentials[self.hive_name] = { 'xml_name': self.hive_name, 'properties': [ ('hcat.metastore.uri', hive_properties.get('thrift_uri')), ('hcat.metastore.principal', hive_properties.get('kerberos_principal')), ] } credentials[self.hiveserver2_name] = { 'xml_name': self.hiveserver2_name, 'properties': [ ('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()), ('hive2.server.principal', hive_properties.get('hive2.server.principal')), ] } credentials[self.hbase_name] = { 'xml_name': self.hbase_name, 'properties': [] } return credentials
def _update_properties(self, jobtracker_addr, deployment_dir): if self.fs and self.jt: self.properties.update({ 'jobTracker': self.jt.logical_name or jobtracker_addr, 'nameNode': self.fs.logical_name or self.fs.fs_defaultfs, }) if self.job: self.properties.update({ self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id }) # Even if no Hive action for now from beeswax.hive_site import get_metastore metastore = get_metastore() if metastore and metastore.get( 'use_sasl') and False: # Disabled for now self.properties.update({ 'is_kerberized_hive': True, 'credential_type': 'hcat', 'thrift_server': metastore.get('thrift_uri'), 'hive_principal': metastore.get('kerberos_principal') })
def get_properties(self, metastore=None): credentials = {} from beeswax import hive_site if metastore is None: metastore = hive_site.get_metastore() if not metastore: metastore = {} LOG.warn('Could not get all the Oozie credentials: hive-site.xml required on the Hue host.') credentials[self.hive_name] = { 'xml_name': self.hive_name, 'properties': [ ('hcat.metastore.uri', metastore.get('thrift_uri')), ('hcat.metastore.principal', metastore.get('kerberos_principal')), ] } credentials[self.hiveserver2_name] = { 'xml_name': self.hiveserver2_name, 'properties': [ ('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()), ('hive2.server.principal', metastore.get('kerberos_principal')), ] } credentials[self.hbase_name] = { 'xml_name': self.hbase_name, 'properties': [] } LOG.info('Using credentials: %s' % credentials) return credentials
def get_properties(self, hive_properties=None): credentials = {} from beeswax import hive_site, conf if not hasattr(conf.HIVE_SERVER_HOST, 'get') or not conf.HIVE_SERVER_HOST.get(): LOG.warn( 'Could not get all the Oozie credentials: beeswax app is blacklisted.' ) else: if hive_properties is None: hive_properties = hive_site.get_metastore() if hive_properties: hive_properties[ 'hive2.server.principal'] = hive_site.get_hiveserver2_kerberos_principal( conf.HIVE_SERVER_HOST.get()) if not hive_properties: hive_properties = {} LOG.warn( 'Could not get all the Oozie credentials: hive-site.xml required on the Hue host.' ) credentials[self.hive_name] = { 'xml_name': self.hive_name, 'properties': [ ('hcat.metastore.uri', hive_properties.get('thrift_uri')), ('hcat.metastore.principal', hive_properties.get('kerberos_principal')), ] } credentials[self.hiveserver2_name] = { 'xml_name': self.hiveserver2_name, 'properties': [ ('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()), ('hive2.server.principal', hive_properties.get('hive2.server.principal')), ] } credentials[self.hbase_name] = { 'xml_name': self.hbase_name, 'properties': [] } return credentials
def get_properties(self, metastore=None): credentials = {} from beeswax import hive_site if metastore is None: metastore = hive_site.get_metastore() if not metastore: metastore = {} LOG.warn( 'Could not get all the Oozie credentials: hive-site.xml required on the Hue host.' ) credentials[self.hive_name] = { 'xml_name': self.hive_name, 'properties': [ ('hcat.metastore.uri', metastore.get('thrift_uri')), ('hcat.metastore.principal', metastore.get('kerberos_principal')), ] } credentials[self.hiveserver2_name] = { 'xml_name': self.hiveserver2_name, 'properties': [ ('hive2.jdbc.url', hive_site.hiveserver2_jdbc_url()), ('hive2.server.principal', metastore.get('kerberos_principal')), ] } credentials[self.hbase_name] = { 'xml_name': self.hbase_name, 'properties': [] } LOG.info('Using credentials: %s' % credentials) return credentials
def _update_properties(self, jobtracker_addr, deployment_dir): if self.fs and self.jt: self.properties.update({ 'jobTracker': self.jt.logical_name or jobtracker_addr, 'nameNode': self.fs.logical_name or self.fs.fs_defaultfs, }) if self.job: self.properties.update({ self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id }) # Even if no Hive action for now from beeswax.hive_site import get_metastore metastore = get_metastore() if metastore and metastore.get('use_sasl') and False: # Disabled for now self.properties.update({ 'is_kerberized_hive': True, 'credential_type': 'hcat', 'thrift_server': metastore.get('thrift_uri'), 'hive_principal': metastore.get('kerberos_principal') })
def meta_client(): """Get the Thrift client to talk to the metastore""" class UnicodeMetastoreClient(object): """Wrap the thrift client to take and return Unicode.""" def __init__(self, client): self._client = client def __getattr__(self, attr): if attr in self.__dict__: return self.__dict__[attr] return getattr(self._client, attr) def _encode_storage_descriptor(self, sd): _encode_struct_attr(sd, 'location') for col in sd.cols: _encode_struct_attr(col, 'comment') self._encode_map(sd.parameters) def _decode_storage_descriptor(self, sd): _decode_struct_attr(sd, 'location') for col in sd.cols: _decode_struct_attr(col, 'comment') self._decode_map(sd.parameters) def _encode_map(self, mapp): for key, value in mapp.iteritems(): mapp[key] = smart_str(value, strings_only=True) def _decode_map(self, mapp): for key, value in mapp.iteritems(): mapp[key] = force_unicode(value, strings_only=True, errors='replace') def create_database(self, name, description): description = smart_str(description) return self._client.create_database(name, description) def get_database(self, *args, **kwargs): db = self._client.get_database(*args, **kwargs) return _decode_struct_attr(db, 'description') def get_fields(self, *args, **kwargs): res = self._client.get_fields(*args, **kwargs) for fschema in res: _decode_struct_attr(fschema, 'comment') return res def get_table(self, *args, **kwargs): res = self._client.get_table(*args, **kwargs) self._decode_storage_descriptor(res.sd) self._decode_map(res.parameters) return res def alter_table(self, dbname, tbl_name, new_tbl): self._encode_storage_descriptor(new_tbl.sd) self._encode_map(new_tbl.parameters) return self._client.alter_table(dbname, tbl_name, new_tbl) def _encode_partition(self, part): self._encode_storage_descriptor(part.sd) self._encode_map(part.parameters) return part def _decode_partition(self, part): self._decode_storage_descriptor(part.sd) self._decode_map(part.parameters) return part def add_partition(self, new_part): self._encode_partition(new_part) part = self._client.add_partition(new_part) return self._decode_partition(part) def get_partition(self, *args, **kwargs): part = self._client.get_partition(*args, **kwargs) return self._decode_partition(part) def get_partitions(self, *args, **kwargs): part_list = self._client.get_partitions(*args, **kwargs) for part in part_list: self._decode_partition(part) return part_list def alter_partition(self, db_name, tbl_name, new_part): self._encode_partition(new_part) return self._client.alter_partition(db_name, tbl_name, new_part) _, host, port = hive_site.get_metastore() client = thrift_util.get_client(ThriftHiveMetastore.Client, host, port, service_name="Hive Metastore Server", timeout_seconds=conf.METASTORE_CONN_TIMEOUT.get()) return UnicodeMetastoreClient(client)
def meta_client(self): """Get the Thrift client to talk to the metastore""" class UnicodeMetastoreClient(object): """Wrap the thrift client to take and return Unicode.""" def __init__(self, client): self._client = client def __getattr__(self, attr): if attr in self.__dict__: return self.__dict__[attr] return getattr(self._client, attr) def _encode_storage_descriptor(self, sd): _encode_struct_attr(sd, 'location') for col in sd.cols: _encode_struct_attr(col, 'comment') self._encode_map(sd.parameters) def _decode_storage_descriptor(self, sd): _decode_struct_attr(sd, 'location') for col in sd.cols: _decode_struct_attr(col, 'comment') self._decode_map(sd.parameters) def _encode_map(self, mapp): for key, value in mapp.iteritems(): mapp[key] = smart_str(value, strings_only=True) def _decode_map(self, mapp): for key, value in mapp.iteritems(): mapp[key] = force_unicode(value, strings_only=True, errors='replace') def create_database(self, name, description): description = smart_str(description) return self._client.create_database(name, description) def get_database(self, *args, **kwargs): db = self._client.get_database(*args, **kwargs) return _decode_struct_attr(db, 'description') def get_fields(self, *args, **kwargs): res = self._client.get_fields(*args, **kwargs) for fschema in res: _decode_struct_attr(fschema, 'comment') return res def get_table(self, *args, **kwargs): res = self._client.get_table(*args, **kwargs) self._decode_storage_descriptor(res.sd) self._decode_map(res.parameters) return res def alter_table(self, dbname, tbl_name, new_tbl): self._encode_storage_descriptor(new_tbl.sd) self._encode_map(new_tbl.parameters) return self._client.alter_table(dbname, tbl_name, new_tbl) def _encode_partition(self, part): self._encode_storage_descriptor(part.sd) self._encode_map(part.parameters) return part def _decode_partition(self, part): self._decode_storage_descriptor(part.sd) self._decode_map(part.parameters) return part def add_partition(self, new_part): self._encode_partition(new_part) part = self._client.add_partition(new_part) return self._decode_partition(part) def get_partition(self, *args, **kwargs): part = self._client.get_partition(*args, **kwargs) return self._decode_partition(part) def get_partitions(self, *args, **kwargs): part_list = self._client.get_partitions(*args, **kwargs) for part in part_list: self._decode_partition(part) return part_list def alter_partition(self, db_name, tbl_name, new_part): self._encode_partition(new_part) return self._client.alter_partition(db_name, tbl_name, new_part) # Use service name from kerberos principal set in hive-site.xml _, host, port, metastore_kerberos_principal = hive_site.get_metastore() use_sasl, kerberos_principal_short_name = BeeswaxClient.get_security() kerberos_principal_short_name = metastore_kerberos_principal and metastore_kerberos_principal.split('/', 1)[0] or None client = thrift_util.get_client(ThriftHiveMetastore.Client, host, port, service_name="Hive Metastore Server", kerberos_principal=kerberos_principal_short_name, use_sasl=use_sasl, timeout_seconds=conf.METASTORE_CONN_TIMEOUT.get()) return UnicodeMetastoreClient(client)