def _get_shared_oozie_server(cls):
callback = lambda: None
_oozie_lock.acquire()
if not OozieServerProvider.is_oozie_running:
LOG.info(
'\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n'
)
LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
finish = (OOZIE_URL.set_for_testing(
"http://localhost:%s/oozie" %
OozieServerProvider.OOZIE_TEST_PORT), )
# Setup
cluster = pseudo_hdfs4.shared_cluster()
cls._setup_sharelib()
cls._reset_oozie()
p = cls._start_oozie(cluster)
def kill():
LOG.info("Killing Oozie server (pid %d)." % p.pid)
os.kill(p.pid, 9)
p.wait()
atexit.register(kill)
start = time.time()
started = False
sleep = 0.01
while not started and time.time() - start < 30.0:
status = None
try:
LOG.info('Check Oozie status...')
status = get_oozie().get_oozie_status()
if status['systemMode'] == 'NORMAL':
started = True
break
time.sleep(sleep)
sleep *= 2
except Exception, e:
LOG.info('Oozie server status not NORMAL yet: %s - %s' %
(status, e))
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Oozie server took too long to come up.")
OozieServerProvider.is_oozie_running = True
def shutdown():
for f in finish:
f()
cluster.stop()
callback = shutdown
def get_oozie(user, api_version=API_VERSION):
oozie_url = OOZIE_URL.get()
secure = SECURITY_ENABLED.get()
ssl_cert_ca_verify = SSL_CERT_CA_VERIFY.get()
return OozieApi(
oozie_url, user, security_enabled=secure, api_version=api_version, ssl_cert_ca_verify=ssl_cert_ca_verify
)
def get_oozie(user, api_version=API_VERSION):
oozie_url = OOZIE_URL.get()
secure = SECURITY_ENABLED.get()
ssl_cert_ca_verify = SSL_CERT_CA_VERIFY.get()
return OozieApi(oozie_url,
user,
security_enabled=secure,
api_version=api_version,
ssl_cert_ca_verify=ssl_cert_ca_verify)
def _get_shared_oozie_server(cls):
callback = lambda: None
_oozie_lock.acquire()
if not OozieServerProvider.is_oozie_running:
LOG.info('\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n')
LOG.info('See https://issues.cloudera.org/browse/HUE-861\n')
finish = (
OOZIE_URL.set_for_testing("http://%s:%s/oozie" % (socket.getfqdn(), OozieServerProvider.OOZIE_TEST_PORT)),
)
# Setup
cluster = pseudo_hdfs4.shared_cluster()
cls._setup_sharelib()
cls._reset_oozie(cluster)
p = cls._start_oozie(cluster)
def kill():
LOG.info("Killing Oozie server (pid %d)." % p.pid)
os.kill(p.pid, 9)
p.wait()
atexit.register(kill)
start = time.time()
started = False
sleep = 0.01
while not started and time.time() - start < 30.0:
status = None
try:
LOG.info('Check Oozie status...')
status = get_oozie().get_oozie_status()
if status['systemMode'] == 'NORMAL':
started = True
break
time.sleep(sleep)
sleep *= 2
except Exception, e:
LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e))
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Oozie server took too long to come up.")
OozieServerProvider.is_oozie_running = True
def shutdown():
for f in finish:
f()
cluster.stop()
callback = shutdown
def get_oozie(user, api_version=API_VERSION):
global _api_cache
if _api_cache is None or _api_cache.api_version != api_version:
_api_cache_lock.acquire()
try:
if _api_cache is None or _api_cache.api_version != api_version:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure, api_version)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_oozie():
"""Return a cached OozieApi"""
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure)
finally:
_api_cache_lock.release()
return _api_cache
def get_oozie(user, api_version=API_VERSION):
global _api_cache
if _api_cache is None or _api_cache.api_version != api_version:
_api_cache_lock.acquire()
try:
if _api_cache is None or _api_cache.api_version != api_version:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure, api_version)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_oozie(user):
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def get_oozie(user):
global _api_cache
if _api_cache is None:
_api_cache_lock.acquire()
try:
if _api_cache is None:
secure = SECURITY_ENABLED.get()
_api_cache = OozieApi(OOZIE_URL.get(), secure)
finally:
_api_cache_lock.release()
_api_cache.setuser(user)
return _api_cache
def _get_shared_oozie_server(cls):
global _oozie_running
callback = lambda: None
_oozie_lock.acquire()
if not _oozie_running:
finish = (
OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT),
)
cluster = pseudo_hdfs4.shared_cluster()
start = time.time()
started = False
sleep = 0.01
while not started and time.time() - start < 20.0:
try:
LOG.info('Check Oozie status...')
status = get_oozie().get_oozie_status()
if status['systemMode'] == 'NORMAL':
started = True
break
time.sleep(sleep)
sleep *= 2
except:
LOG.info('Oozie server status not NORMAL yet.')
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Oozie server took too long to come up.")
_oozie_running = True
def shutdown():
for f in finish:
f()
cluster.stop()
callback = shutdown
_oozie_lock.release()
return get_oozie(), callback
def get_service_info(service):
service_info = {}
if service.lower() == 'solr':
service_info['url'] = SOLR_URL.get()
service_info['security_enabled'] = SOLR_SECURITY_ENABLED.get()
if service.lower() == 'oozie':
service_info['url'] = OOZIE_URL.get()
service_info['security_enabled'] = OOZIE_SECURITY_ENABLED.get()
if service.lower() == 'httpfs':
hdfs_config = hdfs_conf.HDFS_CLUSTERS['default']
service_info['url'] = hdfs_config.WEBHDFS_URL.get()
service_info['security_enabled'] = hdfs_config.SECURITY_ENABLED.get()
if service.lower() == 'rm':
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
service_info['url'] = yarn_cluster.RESOURCE_MANAGER_API_URL.get()
service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
if service.lower() == 'jhs':
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
service_info['url'] = yarn_cluster.HISTORY_SERVER_API_URL.get()
service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
if service.lower() == 'sparkhs':
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
service_info['url'] = yarn_cluster.SPARK_HISTORY_SERVER_URL.get()
service_info[
'security_enabled'] = yarn_cluster.SPARK_HISTORY_SERVER_SECURITY_ENABLED.get(
)
if 'url' not in service_info:
logging.info("Hue does not have %s configured, cannot test %s" %
(service, service))
elif service_info['url'] is None:
logging.info("Hue does not have %s configured, cannot test %s" %
(service, service))
if service_info['url'].endswith('/'):
service_info['url'] = service_info['url'][:-1]
return service_info
def get_oozie(user, api_version=API_VERSION): oozie_url = OOZIE_URL.get() secure = SECURITY_ENABLED.get() return OozieApi(oozie_url, user, security_enabled=secure, api_version=api_version)
