def configure(ctx, config, hadoops):
tempdir = teuthology.get_testdir(ctx)
log.info("Writing Hadoop slaves file...")
for remote in hadoops.remotes:
path, data = get_slaves_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop masters file...")
for remote in hadoops.remotes:
path, data = get_masters_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop core-site.xml file...")
for remote in hadoops.remotes:
path, data = get_core_site_data(ctx, config)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop yarn-site.xml file...")
for remote in hadoops.remotes:
path, data = get_yarn_site_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop hdfs-site.xml file...")
for remote in hadoops.remotes:
path, data = get_hdfs_site_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop mapred-site.xml file...")
for remote in hadoops.remotes:
path, data = get_mapred_site_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Setting JAVA_HOME in hadoop-env.sh...")
for remote in hadoops.remotes:
path = "{tdir}/hadoop/etc/hadoop/hadoop-env.sh".format(tdir=tempdir)
if remote.os.package_type == 'rpm':
data = "JAVA_HOME=/usr/lib/jvm/java\n"
elif remote.os.package_type == 'deb':
data = "JAVA_HOME=/usr/lib/jvm/default-java\n"
else:
raise UnsupportedPackageTypeError(remote)
teuthology.prepend_lines_to_file(remote, path, data)
if config.get('hdfs', False):
log.info("Formatting HDFS...")
testdir = teuthology.get_testdir(ctx)
hadoop_dir = "{tdir}/hadoop/".format(tdir=testdir)
masters = ctx.cluster.only(teuthology.is_type('hadoop.master'))
assert len(masters.remotes) == 1
master = masters.remotes.keys()[0]
master.run(
args = [
hadoop_dir + "bin/hadoop",
"namenode",
"-format"
],
wait = True,
)
def configure(ctx, config, hadoops):
tempdir = teuthology.get_testdir(ctx)
log.info("Writing Hadoop slaves file...")
for remote in hadoops.remotes:
path, data = get_slaves_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop masters file...")
for remote in hadoops.remotes:
path, data = get_masters_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop core-site.xml file...")
for remote in hadoops.remotes:
path, data = get_core_site_data(ctx, config)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop yarn-site.xml file...")
for remote in hadoops.remotes:
path, data = get_yarn_site_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop hdfs-site.xml file...")
for remote in hadoops.remotes:
path, data = get_hdfs_site_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Writing Hadoop mapred-site.xml file...")
for remote in hadoops.remotes:
path, data = get_mapred_site_data(ctx)
teuthology.write_file(remote, path, StringIO(data))
log.info("Setting JAVA_HOME in hadoop-env.sh...")
for remote in hadoops.remotes:
path = "{tdir}/hadoop/etc/hadoop/hadoop-env.sh".format(tdir=tempdir)
if remote.os.package_type == 'rpm':
data = "JAVA_HOME=/usr/lib/jvm/java\n"
elif remote.os.package_type == 'deb':
data = "JAVA_HOME=/usr/lib/jvm/default-java\n"
else:
raise UnsupportedPackageTypeError(remote)
teuthology.prepend_lines_to_file(remote, path, data)
if config.get('hdfs', False):
log.info("Formatting HDFS...")
testdir = teuthology.get_testdir(ctx)
hadoop_dir = "{tdir}/hadoop/".format(tdir=testdir)
masters = ctx.cluster.only(teuthology.is_type('hadoop.master'))
assert len(masters.remotes) == 1
master = masters.remotes.keys()[0]
master.run(
args=[hadoop_dir + "bin/hadoop", "namenode", "-format"],
wait=True,
)