def _get_shared_oozie_server(cls): callback = lambda: None _oozie_lock.acquire() if not OozieServerProvider.is_oozie_running: LOG.info( '\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n' ) LOG.info('See https://issues.cloudera.org/browse/HUE-861\n') finish = (OOZIE_URL.set_for_testing( "http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT), ) # Setup cluster = pseudo_hdfs4.shared_cluster() cls._setup_sharelib() cls._reset_oozie() p = cls._start_oozie(cluster) def kill(): LOG.info("Killing Oozie server (pid %d)." % p.pid) os.kill(p.pid, 9) p.wait() atexit.register(kill) start = time.time() started = False sleep = 0.01 while not started and time.time() - start < 30.0: status = None try: LOG.info('Check Oozie status...') status = get_oozie().get_oozie_status() if status['systemMode'] == 'NORMAL': started = True break time.sleep(sleep) sleep *= 2 except Exception, e: LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e)) time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Oozie server took too long to come up.") OozieServerProvider.is_oozie_running = True def shutdown(): for f in finish: f() cluster.stop() callback = shutdown
def get_oozie(user, api_version=API_VERSION): oozie_url = OOZIE_URL.get() secure = SECURITY_ENABLED.get() ssl_cert_ca_verify = SSL_CERT_CA_VERIFY.get() return OozieApi( oozie_url, user, security_enabled=secure, api_version=api_version, ssl_cert_ca_verify=ssl_cert_ca_verify )
def get_oozie(user, api_version=API_VERSION): oozie_url = OOZIE_URL.get() secure = SECURITY_ENABLED.get() ssl_cert_ca_verify = SSL_CERT_CA_VERIFY.get() return OozieApi(oozie_url, user, security_enabled=secure, api_version=api_version, ssl_cert_ca_verify=ssl_cert_ca_verify)
def _get_shared_oozie_server(cls): callback = lambda: None _oozie_lock.acquire() if not OozieServerProvider.is_oozie_running: LOG.info('\nStarting a Mini Oozie. Requires "tools/jenkins/jenkins.sh" to be previously ran.\n') LOG.info('See https://issues.cloudera.org/browse/HUE-861\n') finish = ( OOZIE_URL.set_for_testing("http://%s:%s/oozie" % (socket.getfqdn(), OozieServerProvider.OOZIE_TEST_PORT)), ) # Setup cluster = pseudo_hdfs4.shared_cluster() cls._setup_sharelib() cls._reset_oozie(cluster) p = cls._start_oozie(cluster) def kill(): LOG.info("Killing Oozie server (pid %d)." % p.pid) os.kill(p.pid, 9) p.wait() atexit.register(kill) start = time.time() started = False sleep = 0.01 while not started and time.time() - start < 30.0: status = None try: LOG.info('Check Oozie status...') status = get_oozie().get_oozie_status() if status['systemMode'] == 'NORMAL': started = True break time.sleep(sleep) sleep *= 2 except Exception, e: LOG.info('Oozie server status not NORMAL yet: %s - %s' % (status, e)) time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Oozie server took too long to come up.") OozieServerProvider.is_oozie_running = True def shutdown(): for f in finish: f() cluster.stop() callback = shutdown
def get_oozie(user, api_version=API_VERSION): global _api_cache if _api_cache is None or _api_cache.api_version != api_version: _api_cache_lock.acquire() try: if _api_cache is None or _api_cache.api_version != api_version: secure = SECURITY_ENABLED.get() _api_cache = OozieApi(OOZIE_URL.get(), secure, api_version) finally: _api_cache_lock.release() _api_cache.setuser(user) return _api_cache
def get_oozie(): """Return a cached OozieApi""" global _api_cache if _api_cache is None: _api_cache_lock.acquire() try: if _api_cache is None: secure = SECURITY_ENABLED.get() _api_cache = OozieApi(OOZIE_URL.get(), secure) finally: _api_cache_lock.release() return _api_cache
def get_oozie(user, api_version=API_VERSION): global _api_cache if _api_cache is None or _api_cache.api_version != api_version: _api_cache_lock.acquire() try: if _api_cache is None or _api_cache.api_version != api_version: secure = SECURITY_ENABLED.get() _api_cache = OozieApi(OOZIE_URL.get(), secure, api_version) finally: _api_cache_lock.release() _api_cache.setuser(user) return _api_cache
def get_oozie(user): global _api_cache if _api_cache is None: _api_cache_lock.acquire() try: if _api_cache is None: secure = SECURITY_ENABLED.get() _api_cache = OozieApi(OOZIE_URL.get(), secure) finally: _api_cache_lock.release() _api_cache.setuser(user) return _api_cache
def get_oozie(user): global _api_cache if _api_cache is None: _api_cache_lock.acquire() try: if _api_cache is None: secure = SECURITY_ENABLED.get() _api_cache = OozieApi(OOZIE_URL.get(), secure) finally: _api_cache_lock.release() _api_cache.setuser(user) return _api_cache
def _get_shared_oozie_server(cls): global _oozie_running callback = lambda: None _oozie_lock.acquire() if not _oozie_running: finish = ( OOZIE_URL.set_for_testing("http://localhost:%s/oozie" % OozieServerProvider.OOZIE_TEST_PORT), ) cluster = pseudo_hdfs4.shared_cluster() start = time.time() started = False sleep = 0.01 while not started and time.time() - start < 20.0: try: LOG.info('Check Oozie status...') status = get_oozie().get_oozie_status() if status['systemMode'] == 'NORMAL': started = True break time.sleep(sleep) sleep *= 2 except: LOG.info('Oozie server status not NORMAL yet.') time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Oozie server took too long to come up.") _oozie_running = True def shutdown(): for f in finish: f() cluster.stop() callback = shutdown _oozie_lock.release() return get_oozie(), callback
def get_service_info(service): service_info = {} if service.lower() == 'solr': service_info['url'] = SOLR_URL.get() service_info['security_enabled'] = SOLR_SECURITY_ENABLED.get() if service.lower() == 'oozie': service_info['url'] = OOZIE_URL.get() service_info['security_enabled'] = OOZIE_SECURITY_ENABLED.get() if service.lower() == 'httpfs': hdfs_config = hdfs_conf.HDFS_CLUSTERS['default'] service_info['url'] = hdfs_config.WEBHDFS_URL.get() service_info['security_enabled'] = hdfs_config.SECURITY_ENABLED.get() if service.lower() == 'rm': yarn_cluster = cluster.get_cluster_conf_for_job_submission() service_info['url'] = yarn_cluster.RESOURCE_MANAGER_API_URL.get() service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get() if service.lower() == 'jhs': yarn_cluster = cluster.get_cluster_conf_for_job_submission() service_info['url'] = yarn_cluster.HISTORY_SERVER_API_URL.get() service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get() if service.lower() == 'sparkhs': yarn_cluster = cluster.get_cluster_conf_for_job_submission() service_info['url'] = yarn_cluster.SPARK_HISTORY_SERVER_URL.get() service_info[ 'security_enabled'] = yarn_cluster.SPARK_HISTORY_SERVER_SECURITY_ENABLED.get( ) if 'url' not in service_info: logging.info("Hue does not have %s configured, cannot test %s" % (service, service)) elif service_info['url'] is None: logging.info("Hue does not have %s configured, cannot test %s" % (service, service)) if service_info['url'].endswith('/'): service_info['url'] = service_info['url'][:-1] return service_info
def get_oozie(user, api_version=API_VERSION): oozie_url = OOZIE_URL.get() secure = SECURITY_ENABLED.get() return OozieApi(oozie_url, user, security_enabled=secure, api_version=api_version)