def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = uuidutils.generate_uuid()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml':
c_helper.generate_xml_configs(
ng.configuration, ng.storage_paths, nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None,
hive.hostname if hive else None,
extra['hive_mysql_passwd'] if hive else None),
'setup_script':
c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(oozie is not None
and oozie.node_group.id == ng.id))
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def convert(self, config, plugin_name, version, cluster_template_create):
normalized_config = clusterspec.ClusterSpec(config).normalize()
#TODO(jspeidel): can we get the name (first arg) from somewhere?
node_groups = []
for ng in normalized_config.node_groups:
node_group = {
"name": ng.name,
"flavor_id": ng.flavor,
"node_processes": ng.node_processes,
"count": ng.count
}
node_groups.append(node_group)
cluster_configs = dict()
for entry in normalized_config.cluster_configs:
ci = entry.config
# get the associated service dictionary
target = entry.config.applicable_target
service_dict = cluster_configs.get(target, {})
service_dict[ci.name] = entry.value
cluster_configs[target] = service_dict
ctx = context.ctx()
return cluster_template_create(ctx,
{"name": uuidutils.generate_uuid(),
"plugin_name": plugin_name,
"hadoop_version": version,
"node_groups": node_groups,
"cluster_configs": cluster_configs})
def convert(self, config, plugin_name, version, cluster_template_create):
normalized_config = clusterspec.ClusterSpec(config).normalize()
#TODO(jspeidel): can we get the name (first arg) from somewhere?
node_groups = []
for ng in normalized_config.node_groups:
node_group = {
"name": ng.name,
"flavor_id": ng.flavor,
"node_processes": ng.node_processes,
"count": ng.count
}
node_groups.append(node_group)
cluster_configs = dict()
for entry in normalized_config.cluster_configs:
ci = entry.config
# get the associated service dictionary
target = entry.config.applicable_target
service_dict = cluster_configs.get(target, {})
service_dict[ci.name] = entry.value
cluster_configs[target] = service_dict
ctx = context.ctx()
return cluster_template_create(
ctx, {
"name": uuidutils.generate_uuid(),
"plugin_name": plugin_name,
"hadoop_version": version,
"node_groups": node_groups,
"cluster_configs": cluster_configs
})
def create_workflow_dir(where, job):
constructed_dir = '/user/hadoop/'
constructed_dir = _add_postfix(constructed_dir)
constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid())
with remote.get_remote(where) as r:
h.create_dir(r, constructed_dir)
return constructed_dir
def convert_to_cluster_template(plugin_name, version, config_file):
plugin = plugin_base.PLUGINS.get_plugin(plugin_name)
tenant_id = context.current().tenant_id
name = uuidutils.generate_uuid()
ct = m.ClusterTemplate(name, tenant_id, plugin_name, version)
plugin.convert(ct, config_file)
return s.persist_cluster_template(ct)
def create_workflow_dir(where, job, hdfs_user):
constructed_dir = '/user/hadoop/'
constructed_dir = _add_postfix(constructed_dir)
constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid())
with remote.get_remote(where) as r:
h.create_dir(r, constructed_dir, hdfs_user)
return constructed_dir
def test_validate_uuid(self):
schema = {
"type": "string",
"format": "uuid",
}
uuid = uuidutils.generate_uuid()
self._validate_success(schema, uuid)
self._validate_failure(schema, uuid.replace("-", ""))
def test_validate_uuid(self):
schema = {
"type": "string",
"format": "uuid",
}
uuid = uuidutils.generate_uuid()
self._validate_success(schema, uuid)
self._validate_failure(schema, uuid.replace("-", ""))
def test_validate_flavor(self):
schema = {
'type': "flavor",
}
self._validate_success(schema, 0)
self._validate_success(schema, 1)
self._validate_success(schema, "0")
self._validate_success(schema, "1")
self._validate_success(schema, uuidutils.generate_uuid())
self._validate_failure(schema, True)
self._validate_failure(schema, 0.1)
self._validate_failure(schema, "0.1")
self._validate_failure(schema, "asd")
def test_validate_flavor(self):
schema = {
'type': "flavor",
}
self._validate_success(schema, 0)
self._validate_success(schema, 1)
self._validate_success(schema, "0")
self._validate_success(schema, "1")
self._validate_success(schema, uuidutils.generate_uuid())
self._validate_failure(schema, True)
self._validate_failure(schema, 0.1)
self._validate_failure(schema, "0.1")
self._validate_failure(schema, "asd")
def convert(self, config, plugin_name, version, cluster_template_create):
normalized_config = clusterspec.ClusterSpec(config).normalize()
#TODO(jspeidel): can we get the name (first arg) from somewhere?
node_groups = []
for ng in normalized_config.node_groups:
node_group = {
"name": ng.name,
"flavor_id": ng.flavor,
"node_processes": ng.node_processes,
"count": ng.count
}
node_groups.append(node_group)
cluster_configs = dict()
config_resource = self.get_configs(version)
for entry in normalized_config.cluster_configs:
user_input = next((ui for ui in config_resource
if entry.config.name == ui.name), None)
if user_input is not None:
ci = entry.config
# get the associated service dictionary
target = entry.config.applicable_target
service_dict = cluster_configs.get(target, {})
service_dict[ci.name] = entry.value
cluster_configs[target] = service_dict
else:
LOG.debug('Template based input "{0}" is being filtered out as'
' it is not considered a user input'
.format(entry.config.name))
ctx = context.ctx()
return cluster_template_create(ctx,
{"name": uuidutils.generate_uuid(),
"plugin_name": plugin_name,
"hadoop_version": version,
"node_groups": node_groups,
"cluster_configs": cluster_configs})
def _extract_configs_to_extra(self, cluster):
nn = utils.get_namenode(cluster)
jt = utils.get_jobtracker(cluster)
oozie = utils.get_oozie(cluster)
hive = utils.get_hiveserver(cluster)
extra = dict()
if hive:
extra['hive_mysql_passwd'] = uuidutils.generate_uuid()
for ng in cluster.node_groups:
extra[ng.id] = {
'xml': c_helper.generate_xml_configs(
ng.configuration,
ng.storage_paths,
nn.hostname,
jt.hostname if jt else None,
oozie.hostname if oozie else None,
hive.hostname if hive else None,
extra['hive_mysql_passwd'] if hive else None),
'setup_script': c_helper.generate_setup_script(
ng.storage_paths,
c_helper.extract_environment_confs(ng.configuration),
append_oozie=(
oozie is not None and oozie.node_group.id == ng.id)
)
}
if c_helper.is_data_locality_enabled(cluster):
topology_data = th.generate_topology_map(
cluster, CONF.enable_hypervisor_awareness)
extra['topology_data'] = "\n".join(
[k + " " + v for k, v in topology_data.items()]) + "\n"
return extra
def _generate_unicode_uuid():
return unicode(uuidutils.generate_uuid())
def generate_request_id():
return 'req-%s' % uuidutils.generate_uuid()
def _generate_unicode_uuid():
return unicode(uuidutils.generate_uuid())
def generate_request_id():
return "req-%s" % uuidutils.generate_uuid()