def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = uuidutils.generate_uuid() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=(oozie is not None and oozie.node_group.id == ng.id)) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def convert(self, config, plugin_name, version, cluster_template_create): normalized_config = clusterspec.ClusterSpec(config).normalize() #TODO(jspeidel): can we get the name (first arg) from somewhere? node_groups = [] for ng in normalized_config.node_groups: node_group = { "name": ng.name, "flavor_id": ng.flavor, "node_processes": ng.node_processes, "count": ng.count } node_groups.append(node_group) cluster_configs = dict() for entry in normalized_config.cluster_configs: ci = entry.config # get the associated service dictionary target = entry.config.applicable_target service_dict = cluster_configs.get(target, {}) service_dict[ci.name] = entry.value cluster_configs[target] = service_dict ctx = context.ctx() return cluster_template_create(ctx, {"name": uuidutils.generate_uuid(), "plugin_name": plugin_name, "hadoop_version": version, "node_groups": node_groups, "cluster_configs": cluster_configs})
def convert(self, config, plugin_name, version, cluster_template_create): normalized_config = clusterspec.ClusterSpec(config).normalize() #TODO(jspeidel): can we get the name (first arg) from somewhere? node_groups = [] for ng in normalized_config.node_groups: node_group = { "name": ng.name, "flavor_id": ng.flavor, "node_processes": ng.node_processes, "count": ng.count } node_groups.append(node_group) cluster_configs = dict() for entry in normalized_config.cluster_configs: ci = entry.config # get the associated service dictionary target = entry.config.applicable_target service_dict = cluster_configs.get(target, {}) service_dict[ci.name] = entry.value cluster_configs[target] = service_dict ctx = context.ctx() return cluster_template_create( ctx, { "name": uuidutils.generate_uuid(), "plugin_name": plugin_name, "hadoop_version": version, "node_groups": node_groups, "cluster_configs": cluster_configs })
def create_workflow_dir(where, job): constructed_dir = '/user/hadoop/' constructed_dir = _add_postfix(constructed_dir) constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid()) with remote.get_remote(where) as r: h.create_dir(r, constructed_dir) return constructed_dir
def convert_to_cluster_template(plugin_name, version, config_file): plugin = plugin_base.PLUGINS.get_plugin(plugin_name) tenant_id = context.current().tenant_id name = uuidutils.generate_uuid() ct = m.ClusterTemplate(name, tenant_id, plugin_name, version) plugin.convert(ct, config_file) return s.persist_cluster_template(ct)
def create_workflow_dir(where, job, hdfs_user): constructed_dir = '/user/hadoop/' constructed_dir = _add_postfix(constructed_dir) constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid()) with remote.get_remote(where) as r: h.create_dir(r, constructed_dir, hdfs_user) return constructed_dir
def test_validate_uuid(self): schema = { "type": "string", "format": "uuid", } uuid = uuidutils.generate_uuid() self._validate_success(schema, uuid) self._validate_failure(schema, uuid.replace("-", ""))
def test_validate_flavor(self): schema = { 'type': "flavor", } self._validate_success(schema, 0) self._validate_success(schema, 1) self._validate_success(schema, "0") self._validate_success(schema, "1") self._validate_success(schema, uuidutils.generate_uuid()) self._validate_failure(schema, True) self._validate_failure(schema, 0.1) self._validate_failure(schema, "0.1") self._validate_failure(schema, "asd")
def convert(self, config, plugin_name, version, cluster_template_create): normalized_config = clusterspec.ClusterSpec(config).normalize() #TODO(jspeidel): can we get the name (first arg) from somewhere? node_groups = [] for ng in normalized_config.node_groups: node_group = { "name": ng.name, "flavor_id": ng.flavor, "node_processes": ng.node_processes, "count": ng.count } node_groups.append(node_group) cluster_configs = dict() config_resource = self.get_configs(version) for entry in normalized_config.cluster_configs: user_input = next((ui for ui in config_resource if entry.config.name == ui.name), None) if user_input is not None: ci = entry.config # get the associated service dictionary target = entry.config.applicable_target service_dict = cluster_configs.get(target, {}) service_dict[ci.name] = entry.value cluster_configs[target] = service_dict else: LOG.debug('Template based input "{0}" is being filtered out as' ' it is not considered a user input' .format(entry.config.name)) ctx = context.ctx() return cluster_template_create(ctx, {"name": uuidutils.generate_uuid(), "plugin_name": plugin_name, "hadoop_version": version, "node_groups": node_groups, "cluster_configs": cluster_configs})
def _extract_configs_to_extra(self, cluster): nn = utils.get_namenode(cluster) jt = utils.get_jobtracker(cluster) oozie = utils.get_oozie(cluster) hive = utils.get_hiveserver(cluster) extra = dict() if hive: extra['hive_mysql_passwd'] = uuidutils.generate_uuid() for ng in cluster.node_groups: extra[ng.id] = { 'xml': c_helper.generate_xml_configs( ng.configuration, ng.storage_paths, nn.hostname, jt.hostname if jt else None, oozie.hostname if oozie else None, hive.hostname if hive else None, extra['hive_mysql_passwd'] if hive else None), 'setup_script': c_helper.generate_setup_script( ng.storage_paths, c_helper.extract_environment_confs(ng.configuration), append_oozie=( oozie is not None and oozie.node_group.id == ng.id) ) } if c_helper.is_data_locality_enabled(cluster): topology_data = th.generate_topology_map( cluster, CONF.enable_hypervisor_awareness) extra['topology_data'] = "\n".join( [k + " " + v for k, v in topology_data.items()]) + "\n" return extra
def _generate_unicode_uuid(): return unicode(uuidutils.generate_uuid())
def generate_request_id(): return 'req-%s' % uuidutils.generate_uuid()
def generate_request_id(): return "req-%s" % uuidutils.generate_uuid()