Python start Examples

Example #1

File: pseudo_hdfs4.py Project: tempbottle/hue

def shared_cluster():
    global _shared_cluster

    if _shared_cluster is None:
        if is_live_cluster():
            cluster = LiveHdfs()
        else:
            cluster = PseudoHdfs4()
            atexit.register(cluster.stop)

            cluster.start()

            fqdn = socket.getfqdn()
            webhdfs_url = "http://%s:%s/webhdfs/v1" % (
                fqdn,
                cluster.dfs_http_port,
            )

            closers = [
                hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.
                set_for_testing(cluster.fs_default_name),
                hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.
                set_for_testing(webhdfs_url),
                hadoop.conf.YARN_CLUSTERS['default'].HOST.set_for_testing(
                    fqdn),
                hadoop.conf.YARN_CLUSTERS['default'].PORT.set_for_testing(
                    cluster._rm_port),
                hadoop.conf.YARN_CLUSTERS['default'].RESOURCE_MANAGER_API_URL.
                set_for_testing('http://%s:%s' % (
                    cluster._fqdn,
                    cluster._rm_webapp_port,
                )),
                hadoop.conf.YARN_CLUSTERS['default'].PROXY_API_URL.
                set_for_testing('http://%s:%s' % (
                    cluster._fqdn,
                    cluster._rm_webapp_port,
                )),
                hadoop.conf.YARN_CLUSTERS['default'].HISTORY_SERVER_API_URL.
                set_for_testing('%s:%s' % (
                    cluster._fqdn,
                    cluster._jh_web_port,
                )),
            ]

            old = hadoop.cluster.clear_caches()

            def restore_config():
                hadoop.cluster.restore_caches(old)
                for x in closers:
                    x()

            cluster.shutdown_hook = restore_config

        _shared_cluster = cluster

    return _shared_cluster

Example #2

File: pseudo_hdfs4.py Project: tempbottle/hue

def main():
    logging.basicConfig(level=logging.DEBUG)

    cluster = PseudoHdfs4()
    cluster.start()

    print "%s running" % (cluster, )
    print "fs.default.name=%s" % (cluster.fs_default_name, )
    print "dfs.http.address=%s" % (cluster.dfs_http_address, )
    print "jobtracker.thrift.port=%s" % (cluster.jt_thrift_port, )
    print "mapred.job.tracker=%s" % (cluster.mapred_job_tracker, )

    from IPython.Shell import IPShellEmbed
    IPShellEmbed()()

    cluster.stop()

Example #3

File: pseudo_hdfs4.py Project: neiodavince/hue

def main():
  logging.basicConfig(level=logging.DEBUG)

  cluster = PseudoHdfs4()
  cluster.start()

  print "%s running" % (cluster,)
  print "fs.default.name=%s" % (cluster.fs_default_name,)
  print "dfs.http.address=%s" % (cluster.dfs_http_address,)
  print "jobtracker.thrift.port=%s" % (cluster.jt_thrift_port,)
  print "mapred.job.tracker=%s" % (cluster.mapred_job_tracker,)

  from IPython.Shell import IPShellEmbed
  IPShellEmbed()()

  cluster.stop()

Example #4

def shared_cluster():
    global _shared_cluster

    if _shared_cluster is None:
        if is_live_cluster():
            cluster = LiveHdfs()
        else:
            cluster = PseudoHdfs4()
            atexit.register(cluster.stop)

            try:
                cluster.start()
            except Exception, ex:
                LOG.exception("Failed to fully bring up test cluster: %s" % (ex,))

            fqdn = socket.getfqdn()
            webhdfs_url = "http://%s:%s/webhdfs/v1" % (fqdn, cluster.dfs_http_port)

            closers = [
                hadoop.conf.HDFS_CLUSTERS["default"].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
                hadoop.conf.HDFS_CLUSTERS["default"].WEBHDFS_URL.set_for_testing(webhdfs_url),
                hadoop.conf.YARN_CLUSTERS["default"].HOST.set_for_testing(fqdn),
                hadoop.conf.YARN_CLUSTERS["default"].PORT.set_for_testing(cluster._rm_port),
                hadoop.conf.YARN_CLUSTERS["default"].RESOURCE_MANAGER_API_URL.set_for_testing(
                    "http://%s:%s" % (cluster._fqdn, cluster._rm_webapp_port)
                ),
                hadoop.conf.YARN_CLUSTERS["default"].PROXY_API_URL.set_for_testing(
                    "http://%s:%s" % (cluster._fqdn, cluster._rm_webapp_port)
                ),
                hadoop.conf.YARN_CLUSTERS["default"].HISTORY_SERVER_API_URL.set_for_testing(
                    "%s:%s" % (cluster._fqdn, cluster._jh_web_port)
                ),
            ]

            old = hadoop.cluster.clear_caches()

            def restore_config():
                hadoop.cluster.restore_caches(old)
                for x in closers:
                    x()

            cluster.shutdown_hook = restore_config

        _shared_cluster = cluster

Example #5

File: pseudo_hdfs4.py Project: 277800076/hue

def shared_cluster():
  global _shared_cluster

  if _shared_cluster is None:
    if is_live_cluster():
      cluster = LiveHdfs()
    else:
      cluster = PseudoHdfs4()
      atexit.register(cluster.stop)

      cluster.start()

      fqdn = socket.getfqdn()
      webhdfs_url = "http://%s:%s/webhdfs/v1" % (fqdn, cluster.dfs_http_port,)

      closers = [
        hadoop.conf.HDFS_CLUSTERS['default'].FS_DEFAULTFS.set_for_testing(cluster.fs_default_name),
        hadoop.conf.HDFS_CLUSTERS['default'].WEBHDFS_URL.set_for_testing(webhdfs_url),

        hadoop.conf.YARN_CLUSTERS['default'].HOST.set_for_testing(fqdn),
        hadoop.conf.YARN_CLUSTERS['default'].PORT.set_for_testing(cluster._rm_port),

        hadoop.conf.YARN_CLUSTERS['default'].RESOURCE_MANAGER_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._rm_webapp_port,)),
        hadoop.conf.YARN_CLUSTERS['default'].PROXY_API_URL.set_for_testing('http://%s:%s' % (cluster._fqdn, cluster._rm_webapp_port,)),
        hadoop.conf.YARN_CLUSTERS['default'].HISTORY_SERVER_API_URL.set_for_testing('%s:%s' % (cluster._fqdn, cluster._jh_web_port,)),
      ]

      old_caches = clear_sys_caches()

      def restore_config():
        restore_sys_caches(old_caches)
        for x in closers:
          x()

      cluster.shutdown_hook = restore_config

    _shared_cluster = cluster

  return _shared_cluster

Example #6

File: mini_cluster.py Project: ycaihua/hue

  """
  Manages _shared_cluster.
  """
  global _shared_cluster
  if _shared_cluster is None:
    _shared_cluster = MiniHadoopCluster()
    _shared_cluster.start()
    atexit.register(_shared_cluster.stop)
  return _shared_cluster

if __name__ == '__main__':
  """
  It's poor form to write tests for tests (the world-wide stack
  overflow exception), so this merely tries the code.
  """
  logging.basicConfig(level=logging.DEBUG)
  import desktop
  desktop.lib.conf.initialize([hadoop.conf])

  if True:
    cluster = MiniHadoopCluster(num_datanodes=5, num_tasktrackers=5)
    cluster.start()
    print cluster.namenode_port
    print cluster.jobtracker_port
    print cluster.config.get("dfs.thrift.address")
    cluster.dump_ini(sys.stdout)

    from IPython.Shell import IPShellEmbed
    IPShellEmbed()()
    cluster.stop()