def test_check_automated_hcatalogs_credentials(self):
reset = SECURITY_ENABLED.set_for_testing(True)
try:
api = get(None, None, self.user)
pig_script = self.create_script()
pig_script.update_from_dict({
'script':"""
a = LOAD 'sample_07' USING org.apache.hcatalog.pig.HCatLoader();
dump a;
STORE raw_data INTO 'students' USING
org.apache.pig.backend.hadoop.hbase.HBaseStorage
org.apache.pig.backend.hadoop.hbase.HBaseStorage (
'info:first_name info:last_name info:age info:gpa info:part');
raw_data = LOAD 'students' USING PigStorage( ' ' ) AS (
id: chararray,
first_name: chararray,
last_name: chararray,
age: int,
gpa: float,
part: int );
"""})
pig_script.save()
wf = api._create_workflow(pig_script, '[]')
start_link = wf.start.get_link()
pig_action = start_link.child
assert_equal([{u'name': u'hcat', u'value': True}, {u'name': u'hbase', u'value': True}], pig_action.credentials)
finally:
reset()
def test_check_automated_hcatalogs_credentials(self):
reset = SECURITY_ENABLED.set_for_testing(True)
try:
api = get(None, None, self.user)
pig_script = self.create_script()
pig_script.update_from_dict({
'script':"""
a = LOAD 'sample_07' USING org.apache.hcatalog.pig.HCatLoader();
dump a;
STORE raw_data INTO 'students' USING
org.apache.pig.backend.hadoop.hbase.HBaseStorage
org.apache.pig.backend.hadoop.hbase.HBaseStorage (
'info:first_name info:last_name info:age info:gpa info:part');
raw_data = LOAD 'students' USING PigStorage( ' ' ) AS (
id: chararray,
first_name: chararray,
last_name: chararray,
age: int,
gpa: float,
part: int );
"""})
pig_script.save()
wf = api._create_workflow(pig_script, '[]')
start_link = wf.start.get_link()
pig_action = start_link.child
assert_equal([{u'name': u'hcat', u'value': True}, {u'name': u'hbase', u'value': True}], pig_action.credentials)
finally:
reset()
def get_oozie(user, api_version=API_VERSION):
oozie_url = OOZIE_URL.get()
secure = SECURITY_ENABLED.get()
ssl_cert_ca_verify = SSL_CERT_CA_VERIFY.get()
return OozieApi(
oozie_url, user, security_enabled=secure, api_version=api_version, ssl_cert_ca_verify=ssl_cert_ca_verify
)
def get_oozie(user, api_version=API_VERSION):
oozie_url = OOZIE_URL.get()
secure = SECURITY_ENABLED.get()
ssl_cert_ca_verify = SSL_CERT_CA_VERIFY.get()
return OozieApi(oozie_url,
user,
security_enabled=secure,
api_version=api_version,
ssl_cert_ca_verify=ssl_cert_ca_verify)
def get_oozie(user, api_version=API_VERSION):
global _api_cache
if _api_cache is None or _api_cache.api_version != api_version:
_api_cache_lock.acquire()
try:
if _api_cache is None or _api_cache.api_version != api_version:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure, api_version)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_oozie():
"""Return a cached OozieApi"""
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure)
finally:
_api_cache_lock.release()
return _api_cache
def get_oozie(user, api_version=API_VERSION):
global _api_cache
if _api_cache is None or _api_cache.api_version != api_version:
_api_cache_lock.acquire()
try:
if _api_cache is None or _api_cache.api_version != api_version:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure, api_version)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_oozie(user):
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_oozie(user):
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_service_info(service):
service_info = {}
if service.lower() == 'solr':
service_info['url'] = SOLR_URL.get()
service_info['security_enabled'] = SOLR_SECURITY_ENABLED.get()
if service.lower() == 'oozie':
service_info['url'] = OOZIE_URL.get()
service_info['security_enabled'] = OOZIE_SECURITY_ENABLED.get()
if service.lower() == 'httpfs':
hdfs_config = hdfs_conf.HDFS_CLUSTERS['default']
service_info['url'] = hdfs_config.WEBHDFS_URL.get()
service_info['security_enabled'] = hdfs_config.SECURITY_ENABLED.get()
if service.lower() == 'rm':
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
service_info['url'] = yarn_cluster.RESOURCE_MANAGER_API_URL.get()
service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
if service.lower() == 'jhs':
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
service_info['url'] = yarn_cluster.HISTORY_SERVER_API_URL.get()
service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
if service.lower() == 'sparkhs':
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
service_info['url'] = yarn_cluster.SPARK_HISTORY_SERVER_URL.get()
service_info[
'security_enabled'] = yarn_cluster.SPARK_HISTORY_SERVER_SECURITY_ENABLED.get(
)
if 'url' not in service_info:
logging.info("Hue does not have %s configured, cannot test %s" %
(service, service))
elif service_info['url'] is None:
logging.info("Hue does not have %s configured, cannot test %s" %
(service, service))
if service_info['url'].endswith('/'):
service_info['url'] = service_info['url'][:-1]
return service_info
def get_oozie(user, api_version=API_VERSION): oozie_url = OOZIE_URL.get() secure = SECURITY_ENABLED.get() return OozieApi(oozie_url, user, security_enabled=secure, api_version=api_version)
