Python read_data_from_hdfs примеры использования

Пример #1

def test_integrity_on_data_node_failure():
    """
    Verifies proper data replication among data nodes.
    """
    # An HDFS write will only successfully return when the data replication has taken place
    config.write_data_to_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)

    sdk_tasks.kill_task_with_pattern("DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-0-node'))
    sdk_tasks.kill_task_with_pattern("DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-1-node'))

    config.read_data_from_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)

    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #2

Файл: test_overlay.py Проект: keithchambers/dcos-commons

def test_integrity_on_data_node_failure():
    """
    Verifies proper data replication among data nodes.
    """
    test_filename = get_unique_filename("test_datanode_fail")

    # An HDFS write will only successfully return when the data replication has taken place
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)

    sdk_cmd.kill_task_with_pattern("DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-0-node'))
    sdk_cmd.kill_task_with_pattern("DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-1-node'))

    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)

    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #3

def test_integrity_on_data_node_failure():
    """
    Verifies proper data replication among data nodes.
    """
    test_filename = "test_datanode_fail"  # must be unique among tests in this suite

    # An HDFS write will only successfully return when the data replication has taken place
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)

    sdk_cmd.kill_task_with_pattern(
        "DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-0-node'))
    sdk_cmd.kill_task_with_pattern(
        "DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-1-node'))

    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)

    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #4

def test_integrity_on_name_node_failure():
    """
    The first name node (name-0-node) is the active name node by default when HDFS gets installed.
    This test checks that it is possible to write and read data after the active name node fails
    so as to verify a failover sustains expected functionality.
    """
    active_name_node = config.get_active_name_node(config.SERVICE_NAME)
    sdk_tasks.kill_task_with_pattern("NameNode", sdk_hosts.system_host(config.SERVICE_NAME, active_name_node))

    predicted_active_name_node = "name-1-node"
    if active_name_node == "name-1-node":
        predicted_active_name_node = "name-0-node"

    wait_for_failover_to_complete(predicted_active_name_node)

    config.write_data_to_hdfs(config.SERVICE_NAME, config.TEST_FILE_2_NAME)
    config.read_data_from_hdfs(config.SERVICE_NAME, config.TEST_FILE_2_NAME)

    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #5

Файл: test_overlay.py Проект: keithchambers/dcos-commons

def test_integrity_on_name_node_failure():
    """
    The first name node (name-0-node) is the active name node by default when HDFS gets installed.
    This test checks that it is possible to write and read data after the active name node fails
    so as to verify a failover sustains expected functionality.
    """
    active_name_node = config.get_active_name_node(config.SERVICE_NAME)
    sdk_cmd.kill_task_with_pattern("NameNode", sdk_hosts.system_host(config.SERVICE_NAME, active_name_node))

    predicted_active_name_node = "name-1-node"
    if active_name_node == "name-1-node":
        predicted_active_name_node = "name-0-node"

    wait_for_failover_to_complete(predicted_active_name_node)

    test_filename = get_unique_filename("test_namenode_fail")
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)
    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)

    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #6

Файл: test_tls.py Проект: mesosphere-backup/mesosphere_development

def test_write_and_read_data_over_tls(hdfs_service_tls):
    config.write_data_to_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)
    config.read_data_from_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)

Пример #7

def test_write_and_read_data_over_tls(hdfs_service):
    test_filename = "test_data_tls"  # must be unique among tests in this suite
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)
    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)

Пример #8

def test_write_and_read_data_on_overlay():
    config.write_data_to_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)
    config.read_data_from_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)
    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #9

def test_write_and_read_data_on_overlay():
    test_filename = get_unique_filename("test_overlay_data")
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)
    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)
    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #10

def test_write_and_read_data_on_overlay():
    test_filename = "test_overlay_data"  # must be unique among tests in this suite
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)
    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)
    config.check_healthy(service_name=config.SERVICE_NAME)

Пример #11

Файл: test_tls.py Проект: keithchambers/dcos-commons

def test_write_and_read_data_over_tls(hdfs_service_tls):
    test_filename = "test_data_tls"  # must be unique among tests in this suite
    config.write_data_to_hdfs(config.SERVICE_NAME, test_filename)
    config.read_data_from_hdfs(config.SERVICE_NAME, test_filename)