Exemplo n.º 1
0
def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname, oozie_hostname):
    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context
    cfg = {
        "fs.default.name": "hdfs://%s:8020" % nn_hostname,
        "dfs.name.dir": extract_hadoop_path(storage_path, "/lib/hadoop/hdfs/namenode"),
        "dfs.data.dir": extract_hadoop_path(storage_path, "/lib/hadoop/hdfs/datanode"),
        "dfs.hosts": "/etc/hadoop/dn.incl",
        "dfs.hosts.exclude": "/etc/hadoop/dn.excl",
    }

    if jt_hostname:
        mr_cfg = {
            "mapred.job.tracker": "%s:8021" % jt_hostname,
            "mapred.system.dir": extract_hadoop_path(storage_path, "/mapred/mapredsystem"),
            "mapred.local.dir": extract_hadoop_path(storage_path, "/lib/hadoop/mapred"),
            "mapred.hosts": "/etc/hadoop/tt.incl",
            "mapred.hosts.exclude": "/etc/hadoop/tt.excl",
        }
        cfg.update(mr_cfg)

    if oozie_hostname:
        o_cfg = {
            "hadoop.proxyuser.hadoop.hosts": "localhost," + oozie_hostname,
            "hadoop.proxyuser.hadoop.groups": "hadoop",
        }
        cfg.update(o_cfg)
        LOG.debug("Applied Oozie configs for core-site.xml")
        cfg.update(o_h.get_oozie_required_xml_configs())
        LOG.debug("Applied Oozie configs for oozie-site.xml")

    # inserting user-defined configs
    for key, value in extract_xml_confs(configs):
        cfg[key] = value

    # applying swift configs if user enabled it
    swift_xml_confs = []
    # TODO(aignatov): should be changed. General configs not only Swift
    swift_in_config = False
    if "general" in configs and ENABLE_SWIFT.name in configs["general"]:
        swift_in_config = True
    if (swift_in_config and configs["general"][ENABLE_SWIFT.name]) or (
        not swift_in_config and ENABLE_SWIFT.default_value
    ):
        swift_xml_confs = swift.get_swift_configs()
        cfg.update(extract_name_values(swift_xml_confs))
        LOG.info("Swift integration is enabled")

    # invoking applied configs to appropriate xml files
    xml_configs = {
        "core-site": x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
        "mapred-site": x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
        "hdfs-site": x.create_hadoop_xml(cfg, HDFS_DEFAULT),
    }

    if oozie_hostname:
        xml_configs.update({"oozie-site": x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
        LOG.debug("Generated oozie-site.xml for oozie % s", oozie_hostname)

    return xml_configs
Exemplo n.º 2
0
def generate_savanna_configs(cluster, node_group=None):
    nn_hostname = _get_hostname(utils.get_namenode(cluster))
    jt_hostname = _get_hostname(utils.get_jobtracker(cluster))
    oozie_hostname = _get_hostname(utils.get_oozie(cluster))
    hive_hostname = _get_hostname(utils.get_hiveserver(cluster))

    storage_path = node_group.storage_paths() if node_group else None

    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context

    cfg = {
        'fs.default.name': 'hdfs://%s:8020' % nn_hostname,
        'dfs.name.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/namenode'),
        'dfs.data.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/datanode'),
        'dfs.hosts': '/etc/hadoop/dn.incl',
        'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
    }

    if jt_hostname:
        mr_cfg = {
            'mapred.job.tracker': '%s:8021' % jt_hostname,
            'mapred.system.dir': extract_hadoop_path(storage_path,
                                                     '/mapred/mapredsystem'),
            'mapred.local.dir': extract_hadoop_path(storage_path,
                                                    '/lib/hadoop/mapred'),
            'mapred.hosts': '/etc/hadoop/tt.incl',
            'mapred.hosts.exclude': '/etc/hadoop/tt.excl',
        }
        cfg.update(mr_cfg)

    if oozie_hostname:
        o_cfg = {
            'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
            'hadoop.proxyuser.hadoop.groups': 'hadoop',
        }
        cfg.update(o_cfg)
        LOG.debug('Applied Oozie configs for core-site.xml')
        cfg.update(o_h.get_oozie_required_xml_configs())
        LOG.debug('Applied Oozie configs for oozie-site.xml')

    if hive_hostname:
        h_cfg = {
            'hive.warehouse.subdir.inherit.perms': True,
            'javax.jdo.option.ConnectionURL':
            'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
        }
        cfg.update(h_cfg)
        LOG.debug('Applied Hive config for hive metastore server')

    return cfg
Exemplo n.º 3
0
def generate_xml_configs(configs, storage_path, nn_hostname,
                         jt_hostname, oozie_hostname, hive_hostname):
    set_general_configs(hive_hostname)
    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context
    cfg = {
        'fs.default.name': 'hdfs://%s:8020' % nn_hostname,
        'dfs.name.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/namenode'),
        'dfs.data.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/datanode'),
        'dfs.hosts': '/etc/hadoop/dn.incl',
        'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
    }

    if jt_hostname:
        mr_cfg = {
            'mapred.job.tracker': '%s:8021' % jt_hostname,
            'mapred.system.dir': extract_hadoop_path(storage_path,
                                                     '/mapred/mapredsystem'),
            'mapred.local.dir': extract_hadoop_path(storage_path,
                                                    '/lib/hadoop/mapred'),
            'mapred.hosts': '/etc/hadoop/tt.incl',
            'mapred.hosts.exclude': '/etc/hadoop/tt.excl',
        }
        cfg.update(mr_cfg)

    if oozie_hostname:
        o_cfg = {
            'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
            'hadoop.proxyuser.hadoop.groups': 'hadoop',
        }
        cfg.update(o_cfg)
        LOG.debug('Applied Oozie configs for core-site.xml')
        cfg.update(o_h.get_oozie_required_xml_configs())
        LOG.debug('Applied Oozie configs for oozie-site.xml')

    if hive_hostname:
        h_cfg = {
            'hive.warehouse.subdir.inherit.perms': True,
            'javax.jdo.option.ConnectionURL':
            'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
        }
        cfg.update(h_cfg)
        LOG.debug('Applied Hive config for hive metastore server')

    # inserting user-defined configs
    for key, value in extract_xml_confs(configs):
        cfg[key] = value

    # applying swift configs if user enabled it
    swift_xml_confs = swift.get_swift_configs()
    cfg = generate_cfg_from_general(cfg, configs, GENERAL_CONFS)
    # invoking applied configs to appropriate xml files
    xml_configs = {
        'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
        'mapred-site': x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
        'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
    }

    if hive_hostname:
        xml_configs.update({'hive-site':
                            x.create_hadoop_xml(cfg, HIVE_DEFAULT)})
        LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)

    if oozie_hostname:
        xml_configs.update({'oozie-site':
                            x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
        LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)

    return xml_configs
Exemplo n.º 4
0
def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname,
                         oozie_hostname):
    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context
    cfg = {
        'fs.default.name':
        'hdfs://%s:8020' % nn_hostname,
        'dfs.name.dir':
        extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/namenode'),
        'dfs.data.dir':
        extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/datanode'),
        'dfs.hosts':
        '/etc/hadoop/dn.incl',
        'dfs.hosts.exclude':
        '/etc/hadoop/dn.excl',
    }

    if jt_hostname:
        mr_cfg = {
            'mapred.job.tracker':
            '%s:8021' % jt_hostname,
            'mapred.system.dir':
            extract_hadoop_path(storage_path, '/mapred/mapredsystem'),
            'mapred.local.dir':
            extract_hadoop_path(storage_path, '/lib/hadoop/mapred'),
            'mapred.hosts':
            '/etc/hadoop/tt.incl',
            'mapred.hosts.exclude':
            '/etc/hadoop/tt.excl',
        }
        cfg.update(mr_cfg)

    if oozie_hostname:
        o_cfg = {
            'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
            'hadoop.proxyuser.hadoop.groups': 'hadoop',
        }
        cfg.update(o_cfg)
        LOG.debug('Applied Oozie configs for core-site.xml')
        cfg.update(o_h.get_oozie_required_xml_configs())
        LOG.debug('Applied Oozie configs for oozie-site.xml')

    # inserting user-defined configs
    for key, value in extract_xml_confs(configs):
        cfg[key] = value

    # applying swift configs if user enabled it
    swift_xml_confs = []
    #TODO(aignatov): should be changed. General configs not only Swift
    swift_in_config = False
    if ('general' in configs and ENABLE_SWIFT.name in configs['general']):
        swift_in_config = True
    if ((swift_in_config and configs['general'][ENABLE_SWIFT.name])
            or (not swift_in_config and ENABLE_SWIFT.default_value)):
        swift_xml_confs = swift.get_swift_configs()
        cfg.update(extract_name_values(swift_xml_confs))
        LOG.info("Swift integration is enabled")

    # invoking applied configs to appropriate xml files
    xml_configs = {
        'core-site': x.create_hadoop_xml(cfg, CORE_DEFAULT + swift_xml_confs),
        'mapred-site': x.create_hadoop_xml(cfg, MAPRED_DEFAULT),
        'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
    }

    if oozie_hostname:
        xml_configs.update(
            {'oozie-site': x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
        LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)

    return xml_configs
Exemplo n.º 5
0
def generate_xml_configs(configs, storage_path, nn_hostname, jt_hostname,
                         oozie_hostname, hive_hostname, passwd_hive_mysql):
    general_cfg = get_general_configs(hive_hostname, passwd_hive_mysql)
    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context
    cfg = {
        'fs.default.name':
        'hdfs://%s:8020' % nn_hostname,
        'dfs.name.dir':
        extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/namenode'),
        'dfs.data.dir':
        extract_hadoop_path(storage_path, '/lib/hadoop/hdfs/datanode'),
        'dfs.hosts':
        '/etc/hadoop/dn.incl',
        'dfs.hosts.exclude':
        '/etc/hadoop/dn.excl',
    }

    if jt_hostname:
        mr_cfg = {
            'mapred.job.tracker':
            '%s:8021' % jt_hostname,
            'mapred.system.dir':
            extract_hadoop_path(storage_path, '/mapred/mapredsystem'),
            'mapred.local.dir':
            extract_hadoop_path(storage_path, '/lib/hadoop/mapred'),
            'mapred.hosts':
            '/etc/hadoop/tt.incl',
            'mapred.hosts.exclude':
            '/etc/hadoop/tt.excl',
        }
        cfg.update(mr_cfg)

    if oozie_hostname:
        o_cfg = {
            'hadoop.proxyuser.hadoop.hosts': "localhost," + oozie_hostname,
            'hadoop.proxyuser.hadoop.groups': 'hadoop',
        }
        cfg.update(o_cfg)
        LOG.debug('Applied Oozie configs for core-site.xml')
        cfg.update(o_h.get_oozie_required_xml_configs())
        LOG.debug('Applied Oozie configs for oozie-site.xml')

    if hive_hostname:
        h_cfg = {
            'hive.warehouse.subdir.inherit.perms':
            True,
            'javax.jdo.option.ConnectionURL':
            'jdbc:derby:;databaseName=/opt/hive/metastore_db;create=true'
        }
        cfg.update(h_cfg)
        LOG.debug('Applied Hive config for hive metastore server')

    # inserting user-defined configs
    for key, value in extract_xml_confs(configs):
        cfg[key] = value

    # applying swift configs if user enabled it
    swift_xml_confs = swift.get_swift_configs()
    cfg = generate_cfg_from_general(cfg, configs, general_cfg)

    # invoking applied configs to appropriate xml files
    core_all = CORE_DEFAULT + swift_xml_confs
    mapred_all = MAPRED_DEFAULT

    if CONF.enable_data_locality:
        cfg.update(topology.TOPOLOGY_CONFIG)

        # applying vm awareness configs
        core_all += topology.vm_awareness_core_config()
        mapred_all += topology.vm_awareness_mapred_config()

    xml_configs = {
        'core-site': x.create_hadoop_xml(cfg, core_all),
        'mapred-site': x.create_hadoop_xml(cfg, mapred_all),
        'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
    }

    if hive_hostname:
        xml_configs.update(
            {'hive-site': x.create_hadoop_xml(cfg, HIVE_DEFAULT)})
        LOG.debug('Generated hive-site.xml for hive % s', hive_hostname)

    if oozie_hostname:
        xml_configs.update(
            {'oozie-site': x.create_hadoop_xml(cfg, o_h.OOZIE_DEFAULT)})
        LOG.debug('Generated oozie-site.xml for oozie % s', oozie_hostname)

    return xml_configs