def get_config_files(self, cluster_context, configs, instance=None):
hbase_version = self._get_hbase_version(cluster_context)
hive_version = self._get_hive_version(cluster_context)
# spark-env-sh
template = 'plugins/mapr/services/' \
'spark/resources/spark-env.template'
env_sh = bcf.TemplateFile('spark-env.sh')
env_sh.remote_path = self.conf_dir(cluster_context)
env_sh.parse(files.get_file_text(template))
env_sh.add_property('version', self.version)
# spark-defaults
conf = bcf.PropertiesFile('spark-defaults.conf', separator=' ')
conf.remote_path = self.conf_dir(cluster_context)
if instance:
conf.fetch(instance)
# compatibility.version
versions = bcf.PropertiesFile('compatibility.version')
versions.remote_path = self.home_dir(cluster_context) + '/mapr-util'
if instance:
versions.fetch(instance)
if hive_version:
versions.add_property('hive_versions', hive_version + '.0')
conf.add_properties(self._hive_properties(cluster_context))
if hbase_version:
versions.add_property('hbase_versions', hbase_version)
conf.add_property(
'spark.executor.extraClassPath', '%s/lib/*' %
self._hbase(cluster_context).home_dir(cluster_context))
return [conf, versions, env_sh]
def test_render(self):
foo = conf_f.PropertiesFile('foo')
expected = {'ke1': 'value1', 'key2': 'value2'}
foo._config_dict = expected
actual = foo.render()
bar = conf_f.PropertiesFile('bar')
bar.parse(actual)
self.assertDictEqual(expected, bar._config_dict)
def get_config_files(self, cluster_context, configs, instance=None):
default_path = 'plugins/mapr/services/maprfs/resources/cldb.conf'
cldb_conf = bcf.PropertiesFile("cldb.conf")
cldb_conf.remote_path = self.conf_dir(cluster_context)
if instance:
cldb_conf.fetch(instance)
cldb_conf.parse(files.get_file_text(default_path))
cldb_conf.add_properties(self._get_cldb_conf_props(cluster_context))
warden_conf = bcf.PropertiesFile("warden.conf")
warden_conf.remote_path = "/opt/mapr/conf/"
if instance:
warden_conf.fetch(instance)
warden_conf.add_properties(
{'service.command.mfs.heapsize.percent': configs[
self.HEAP_SIZE_PERCENT_NAME]})
return [cldb_conf, warden_conf]
def get_config_files(self, cluster_context, configs, instance=None):
default_path = 'plugins/mapr/services/maprfs/resources/cldb.conf'
cldb_conf = bcf.PropertiesFile("cldb.conf")
cldb_conf.remote_path = self.conf_dir(cluster_context)
if instance:
cldb_conf.fetch(instance)
cldb_conf.parse(files.get_file_text(default_path))
cldb_conf.add_properties(self._get_cldb_conf_props(cluster_context))
return [cldb_conf]
def test_add_property(self):
foo = conf_f.PropertiesFile('foo')
expected = {}
self.assertDictEqual(expected, foo._config_dict)
foo.add_property('key1', 'value1')
expected = {'key1': 'value1'}
self.assertDictEqual(expected, foo._config_dict)
foo.add_property('key2', 'value2')
expected = {'key1': 'value1', 'key2': 'value2'}
self.assertDictEqual(expected, foo._config_dict)
def get_config_files(self, cluster_context, configs, instance=None):
hbase_version = self._get_hbase_version(cluster_context)
hive_version = self._get_hive_version(cluster_context)
# spark-env-sh
template = 'plugins/mapr/services/' \
'spark/resources/spark-env.template'
env_sh = bcf.TemplateFile('spark-env.sh')
env_sh.remote_path = self.conf_dir(cluster_context)
env_sh.parse(files.get_file_text(template))
env_sh.add_property('version', self.version)
env_sh.add_property('servlet_api_jar', self.SERVLET_JAR)
# spark-defaults
conf = bcf.PropertiesFile('spark-defaults.conf', separator=' ')
conf.remote_path = self.conf_dir(cluster_context)
if instance:
conf.fetch(instance)
conf.add_property(
'spark.yarn.jar', 'maprfs://%s/%s' %
(self.JAR_FILE_TARGET,
self._assembly_jar_path(cluster_context).rsplit('/', 1)[1]))
# compatibility.version
versions = bcf.PropertiesFile('compatibility.version')
versions.remote_path = self.home_dir(cluster_context) + '/mapr-util'
if instance:
versions.fetch(instance)
if hive_version:
versions.add_property('hive_versions',
self._format_hive_version(hive_version))
conf.add_properties(self._hive_properties(cluster_context))
if hbase_version:
versions.add_property('hbase_versions', hbase_version)
conf.add_property(
'spark.executor.extraClassPath', '%s/lib/*' %
self._hbase(cluster_context).home_dir(cluster_context))
return [conf, versions, env_sh]
def test_get_config_value(self):
foo = conf_f.PropertiesFile('foo')
foo._config_dict = {'foo': 'bar'}
self.assertEqual('bar', foo._get_config_value('foo'))
self.assertIsNone(foo._get_config_value('bar'))
def test_parse(self):
foo = conf_f.PropertiesFile('foo')
foo.parse(self.content)
expected = {'key1': 'value1', 'key2': 'value2'}
self.assertDictEqual(expected, foo._config_dict)
def test_remote_path(self):
foo = conf_f.PropertiesFile('foo')
foo.remote_path = '/bar'
self.assertEqual('/bar/foo', foo.remote_path)
