def shared_cluster(): global _shared_cluster if _shared_cluster is None: if is_live_cluster(): cluster = LiveHdfs() else: cluster = PseudoHdfs4() atexit.register(cluster.stop) cluster.start() fqdn = socket.getfqdn() webhdfs_url = "http://%s:%s/webhdfs/v1" % ( fqdn, cluster.dfs_http_port, ) closers = [ hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS. set_for_testing(cluster.fs_default_name), hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL. set_for_testing(webhdfs_url), hadoop.conf.YARN_CLUSTERS['default'].HOST.set_for_testing( fqdn), hadoop.conf.YARN_CLUSTERS['default'].PORT.set_for_testing( cluster._rm_port), hadoop.conf.YARN_CLUSTERS['default'].RESOURCE_MANAGER_API_URL. set_for_testing('http://%s:%s' % ( cluster._fqdn, cluster._rm_webapp_port, )), hadoop.conf.YARN_CLUSTERS['default'].PROXY_API_URL. set_for_testing('http://%s:%s' % ( cluster._fqdn, cluster._rm_webapp_port, )), hadoop.conf.YARN_CLUSTERS['default'].HISTORY_SERVER_API_URL. set_for_testing('%s:%s' % ( cluster._fqdn, cluster._jh_web_port, )), ] old = hadoop.cluster.clear_caches() def restore_config(): hadoop.cluster.restore_caches(old) for x in closers: x() cluster.shutdown_hook = restore_config _shared_cluster = cluster return _shared_cluster
def main(): logging.basicConfig(level=logging.DEBUG) cluster = PseudoHdfs4() cluster.start() print "%s running" % (cluster, ) print "fs.default.name=%s" % (cluster.fs_default_name, ) print "dfs.http.address=%s" % (cluster.dfs_http_address, ) print "jobtracker.thrift.port=%s" % (cluster.jt_thrift_port, ) print "mapred.job.tracker=%s" % (cluster.mapred_job_tracker, ) from IPython.Shell import IPShellEmbed IPShellEmbed()() cluster.stop()
def main(): logging.basicConfig(level=logging.DEBUG) cluster = PseudoHdfs4() cluster.start() print "%s running" % (cluster,) print "fs.default.name=%s" % (cluster.fs_default_name,) print "dfs.http.address=%s" % (cluster.dfs_http_address,) print "jobtracker.thrift.port=%s" % (cluster.jt_thrift_port,) print "mapred.job.tracker=%s" % (cluster.mapred_job_tracker,) from IPython.Shell import IPShellEmbed IPShellEmbed()() cluster.stop()
def shared_cluster(): global _shared_cluster if _shared_cluster is None: if is_live_cluster(): cluster = LiveHdfs() else: cluster = PseudoHdfs4() atexit.register(cluster.stop) try: cluster.start() except Exception, ex: LOG.exception("Failed to fully bring up test cluster: %s" % (ex,)) fqdn = socket.getfqdn() webhdfs_url = "http://%s:%s/webhdfs/v1" % (fqdn, cluster.dfs_http_port) closers = [ hadoop.conf.HDFS_CLUSTERS["default"].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name), hadoop.conf.HDFS_CLUSTERS["default"].WEBHDFS_URL.set_for_testing(webhdfs_url), hadoop.conf.YARN_CLUSTERS["default"].HOST.set_for_testing(fqdn), hadoop.conf.YARN_CLUSTERS["default"].PORT.set_for_testing(cluster._rm_port), hadoop.conf.YARN_CLUSTERS["default"].RESOURCE_MANAGER_API_URL.set_for_testing( "http://%s:%s" % (cluster._fqdn, cluster._rm_webapp_port) ), hadoop.conf.YARN_CLUSTERS["default"].PROXY_API_URL.set_for_testing( "http://%s:%s" % (cluster._fqdn, cluster._rm_webapp_port) ), hadoop.conf.YARN_CLUSTERS["default"].HISTORY_SERVER_API_URL.set_for_testing( "%s:%s" % (cluster._fqdn, cluster._jh_web_port) ), ] old = hadoop.cluster.clear_caches() def restore_config(): hadoop.cluster.restore_caches(old) for x in closers: x() cluster.shutdown_hook = restore_config _shared_cluster = cluster
def shared_cluster(): global _shared_cluster if _shared_cluster is None: if is_live_cluster(): cluster = LiveHdfs() else: cluster = PseudoHdfs4() atexit.register(cluster.stop) cluster.start() fqdn = socket.getfqdn() webhdfs_url = "http://%s:%s/webhdfs/v1" % (fqdn, cluster.dfs_http_port,) closers = [ hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name), hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url), hadoop.conf.YARN_CLUSTERS['default'].HOST.set_for_testing(fqdn), hadoop.conf.YARN_CLUSTERS['default'].PORT.set_for_testing(cluster._rm_port), hadoop.conf.YARN_CLUSTERS['default'].RESOURCE_MANAGER_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._rm_webapp_port,)), hadoop.conf.YARN_CLUSTERS['default'].PROXY_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._rm_webapp_port,)), hadoop.conf.YARN_CLUSTERS['default'].HISTORY_SERVER_API_URL.set_for_testing('%s:%s' % (cluster._fqdn, cluster._jh_web_port,)), ] old_caches = clear_sys_caches() def restore_config(): restore_sys_caches(old_caches) for x in closers: x() cluster.shutdown_hook = restore_config _shared_cluster = cluster return _shared_cluster
""" Manages _shared_cluster. """ global _shared_cluster if _shared_cluster is None: _shared_cluster = MiniHadoopCluster() _shared_cluster.start() atexit.register(_shared_cluster.stop) return _shared_cluster if __name__ == '__main__': """ It's poor form to write tests for tests (the world-wide stack overflow exception), so this merely tries the code. """ logging.basicConfig(level=logging.DEBUG) import desktop desktop.lib.conf.initialize([hadoop.conf]) if True: cluster = MiniHadoopCluster(num_datanodes=5, num_tasktrackers=5) cluster.start() print cluster.namenode_port print cluster.jobtracker_port print cluster.config.get("dfs.thrift.address") cluster.dump_ini(sys.stdout) from IPython.Shell import IPShellEmbed IPShellEmbed()() cluster.stop()