def generate_report(data, report_template, report_filename, subunit_filename): LOG.debug('Generating report, template: %s, output: %s', report_template, report_filename or '<dummy>') data['records'] += calculate_stats(data['records'], data['tests']) sla_records = verify_sla(data['records'], data['tests']) if subunit_filename: save_to_subunit(sla_records, subunit_filename) # add more filters to jinja jinja_env = jinja2.Environment(variable_start_string='[[[', variable_end_string=']]]', comment_start_string='[[#', comment_end_string='#]]') jinja_env.filters['json'] = json.dumps jinja_env.filters['yaml'] = functools.partial(yaml.safe_dump, indent=2, default_flow_style=False) template = utils.read_file(report_template) compiled_template = jinja_env.from_string(template) rendered_template = compiled_template.render(dict(report=data)) if report_filename: LOG.debug('Writing report to: %s', report_filename) try: utils.write_file(rendered_template, report_filename) LOG.info('Report saved to: %s', report_filename) except IOError as e: LOG.error('Failed to write report file: %s', e)
def test_deploy_from_hot_with_env_file(self, nova_nodes_mock, openstack_mock, create_stack_mock, stack_output_mock): test_file = 'shaker/scenarios/test/sample_with_env.yaml' absolute_path = utils.resolve_relative_path(test_file) scenario = utils.read_yaml_file(absolute_path) stack_name = 'shaker_abcdefg' server_endpoint = "127.0.0.01" base_dir = os.path.dirname(absolute_path) deployment = deploy.Deployment() deployment.stack_name = stack_name deployment.external_net = 'test-external_net' deployment.image_name = 'test-image' deployment.flavor_name = 'test-flavor' deployment.dns_nameservers = '8.8.8.8' deployment.openstack_client = openstack_mock # read the env file to determine what cidr is set to # minus the last digit env_file = utils.read_file(scenario['deployment']['env_file'], base_dir) cidr = re.findall(r'[0-9]+(?:\.[0-9]+){2}', env_file)[0] nova_nodes_mock.return_value = [{'host': 'host-1', 'zone': 'nova'}] create_stack_mock.return_value = uuid.uuid4() heat_outputs = { stack_name + '_master_0_instance_name': 'instance-0000052f', stack_name + '_master_0_ip': '192.0.0.3', stack_name + '_slave_0_ip': '192.0.0.4', stack_name + '_slave_0_instance_name': 'instance-0000052c'} stack_output_mock.return_value = heat_outputs expected = { 'shaker_abcdefg_master_0': {'availability_zone': 'nova:host-1', 'id': 'shaker_abcdefg_master_0', 'ip': cidr + '.3', 'mode': 'master', 'node': 'host-1', 'slave_id': 'shaker_abcdefg_slave_0', 'zone': 'nova'}, 'shaker_abcdefg_slave_0': {'availability_zone': 'nova:host-1', 'id': 'shaker_abcdefg_slave_0', 'ip': cidr + '.4', 'master_id': 'shaker_abcdefg_master_0', 'mode': 'slave', 'node': 'host-1', 'zone': 'nova'}} agents = deployment._deploy_from_hot(scenario['deployment'], server_endpoint, base_dir=base_dir) self.assertEqual(expected, agents)
def main(): utils.init_config_and_logging(config.REPORT_OPTS + config.INPUT_OPTS) LOG.debug('Reading JSON data from: %s', cfg.CONF.input) report_data = json.loads(utils.read_file(cfg.CONF.input)) generate_report(report_data, cfg.CONF.report_template, cfg.CONF.report, cfg.CONF.subunit, cfg.CONF.book)
def _render_env_template(self, env_file, base_dir): env_template = utils.read_file(env_file, base_dir=base_dir) env_values = {'CONF': cfg.CONF, 'unique': self.stack_name} compiled_env = jinja2.Template(env_template) rendered_env = compiled_env.render(env_values) environment = utils.read_yaml(rendered_env) return environment
def build_image(): openstack_client = init() flavor_name = cfg.CONF.flavor_name image_name = cfg.CONF.image_name if nova.is_flavor_exists(openstack_client.nova, flavor_name): LOG.info('Using existing flavor: %s', flavor_name) else: openstack_client.nova.flavors.create(name=flavor_name, ram=512, vcpus=1, disk=3) LOG.info('Created flavor %s', flavor_name) if glance.get_image(openstack_client.glance, image_name): LOG.info('Using existing image: %s', image_name) else: template = None template_filename = cfg.CONF.image_builder_template try: template = utils.read_file(template_filename) except IOError: LOG.error('Error reading template file: %s. ' 'Please verify correctness of --image-builder-template ' 'parameter', template_filename) exit(1) external_net = (cfg.CONF.external_net or neutron.choose_external_net(openstack_client.neutron)) stack_params = { 'stack_name': 'shaker_%s' % uuid.uuid4(), 'parameters': {'external_net': external_net, 'flavor': flavor_name}, 'template': template, } stack = openstack_client.heat.stacks.create(**stack_params)['stack'] LOG.debug('New stack: %s', stack) heat.wait_stack_completion(openstack_client.heat, stack['id']) outputs = heat.get_stack_outputs(openstack_client.heat, stack['id']) LOG.debug('Stack outputs: %s', outputs) LOG.debug('Waiting for server to shutdown') server_id = outputs['server_info'].get('id') nova.wait_server_shutdown(openstack_client.nova, server_id) LOG.debug('Making snapshot') openstack_client.nova.servers.create_image( server_id, image_name) LOG.debug('Waiting for server to snapshot') nova.wait_server_snapshot(openstack_client.nova, server_id) LOG.debug('Clearing up') openstack_client.heat.stacks.delete(stack['id']) LOG.info('Created image: %s', image_name)
def main(): utils.init_config_and_logging(config.REPORT_OPTS + config.INPUT_OPTS) outputs = [] for input_filename in cfg.CONF.input: LOG.debug('Reading JSON data from: %s', input_filename) outputs.append(json.loads(utils.read_file(input_filename))) aggregated = utils.merge_dicts(outputs) generate_report(aggregated, cfg.CONF.report_template, cfg.CONF.report, cfg.CONF.subunit, cfg.CONF.book)
def _deploy_from_hot(self, specification, server_endpoint, base_dir=None): accommodation = normalize_accommodation( specification.get('accommodation') or specification.get('vm_accommodation')) agents = generate_agents(self._get_compute_nodes(accommodation), accommodation, self.stack_name) # render template by jinja vars_values = { 'agents': agents, 'unique': self.stack_name, } heat_template = utils.read_file(specification['template'], base_dir=base_dir) compiled_template = jinja2.Template(heat_template) rendered_template = compiled_template.render(vars_values) LOG.debug('Rendered template: %s', rendered_template) # create stack by Heat try: merged_parameters = { 'server_endpoint': server_endpoint, 'external_net': self.external_net, 'image': self.image_name, 'flavor': self.flavor_name, 'dns_nameservers': self.dns_nameservers, } except AttributeError as e: LOG.error( 'Failed to gather required parameters to create ' 'heat stack: %s', e) exit(1) merged_parameters.update(specification.get('template_parameters', {})) self.has_stack = True stack_id = heat.create_stack(self.openstack_client.heat, self.stack_name, rendered_template, merged_parameters) # get info about deployed objects outputs = heat.get_stack_outputs(self.openstack_client.heat, stack_id) override = self._get_override(specification.get('override')) agents = filter_agents(agents, outputs, override) if (not self.privileged_mode) and accommodation.get('density', 1) == 1: get_host_fn = functools.partial(nova.get_server_host_id, self.openstack_client.nova) agents = distribute_agents(agents, get_host_fn) return agents
def build_image_with_heat(openstack_client, image_name, flavor_name, dns_nameservers): template = None template_filename = cfg.CONF.image_builder_template try: am = lambda f: config.IMAGE_BUILDER_TEMPLATES + '%s.yaml' % f template = utils.read_file(template_filename, alias_mapper=am) except IOError: LOG.error( 'Error reading template file: %s. ' 'Please verify correctness of --image-builder-template ' 'parameter', template_filename) exit(1) external_net = (cfg.CONF.external_net or neutron.choose_external_net(openstack_client.neutron)) stack_name = 'shaker_%s' % uuid.uuid4() stack_parameters = { 'external_net': external_net, 'flavor': flavor_name, 'dns_nameservers': dns_nameservers } stack_id = None try: stack_id = heat.create_stack(openstack_client.heat, stack_name, template, stack_parameters) outputs = heat.get_stack_outputs(openstack_client.heat, stack_id) LOG.debug('Stack outputs: %s', outputs) LOG.debug('Waiting for server to shutdown') server_id = outputs['server_info'].get('id') nova.wait_server_shutdown(openstack_client.nova, server_id) LOG.debug('Making snapshot') openstack_client.nova.servers.create_image(server_id, image_name) LOG.debug('Waiting for server to snapshot') nova.wait_server_snapshot(openstack_client.nova, server_id) LOG.info('Created image: %s', image_name) except BaseException as e: if isinstance(e, KeyboardInterrupt): LOG.info('Caught SIGINT. Terminating') else: error_msg = 'Error while building the image: %s' % e LOG.error(error_msg) LOG.exception(e) finally: if stack_id and cfg.CONF.cleanup_on_error: LOG.debug('Cleaning up the stack: %s', stack_id) openstack_client.heat.stacks.delete(stack_id)
def _deploy_from_hot(self, specification, server_endpoint, base_dir=None): accommodation = normalize_accommodation( specification.get('accommodation') or specification.get('vm_accommodation')) agents = generate_agents(self._get_compute_nodes(accommodation), accommodation, self.stack_name) # render template by jinja vars_values = { 'agents': agents, 'unique': self.stack_name, } heat_template = utils.read_file(specification['template'], base_dir=base_dir) compiled_template = jinja2.Template(heat_template) rendered_template = compiled_template.render(vars_values) LOG.debug('Rendered template: %s', rendered_template) # create stack by Heat merged_parameters = { 'server_endpoint': server_endpoint, 'external_net': self.external_net, 'image': self.image_name, 'flavor': self.flavor_name, 'dns_nameservers': self.dns_nameservers, } merged_parameters.update(specification.get('template_parameters', {})) self.has_stack = True stack_id = heat.create_stack( self.openstack_client.heat, self.stack_name, rendered_template, merged_parameters) # get info about deployed objects outputs = heat.get_stack_outputs(self.openstack_client.heat, stack_id) override = self._get_override(specification.get('override')) agents = filter_agents(agents, outputs, override) if (not self.privileged_mode) and accommodation.get('density', 1) == 1: get_host_fn = functools.partial(nova.get_server_host_id, self.openstack_client.nova) agents = distribute_agents(agents, get_host_fn) return agents
def _deploy_from_hot(self, specification, base_dir=None): agents = generate_agents( nova.get_available_compute_nodes(self.openstack_client.nova), specification['vm_accommodation'], self.stack_name) # render template by jinja vars_values = { 'agents': agents, 'unique': self.stack_name, } heat_template = utils.read_file(specification['template'], base_dir=base_dir) compiled_template = jinja2.Template(heat_template) rendered_template = compiled_template.render(vars_values) LOG.debug('Rendered template: %s', rendered_template) # create stack by Heat merged_parameters = { 'server_endpoint': self.server_endpoint, 'external_net': self.external_net, 'image': self.image_name, 'flavor': self.flavor_name, } merged_parameters.update(specification.get('template_parameters', {})) stack_params = { 'stack_name': self.stack_name, 'parameters': merged_parameters, 'template': rendered_template, } LOG.debug('Creating stack with parameters: %s', stack_params) stack = self.openstack_client.heat.stacks.create( **stack_params)['stack'] LOG.info('New stack: %s', stack) heat.wait_stack_completion(self.openstack_client.heat, stack['id']) self.stack_deployed = True # get info about deployed objects outputs = heat.get_stack_outputs(self.openstack_client.heat, stack['id']) return filter_agents(agents, outputs)
def _deploy_support_stacks(self, support_templates, base_dir): for stack in support_templates: try: support_name = stack['name'] support_template = utils.read_file(stack['template'], base_dir=base_dir) support_env_file = stack.get('env_file', None) if support_env_file is not None: support_env_file = self._render_env_template( support_env_file, base_dir) # user should set default values in supoort template # or provide a heat environment file to update # parameters for support templates support_template_params = {} support_id = heat.create_stack(self.openstack_client.heat, support_name, support_template, support_template_params, support_env_file) # track support stacks for cleanup current_stack = self.TrackStack(name=support_name, id=support_id) self.support_stacks.append(current_stack) LOG.debug('Tracking support stacks: %s', self.support_stacks) except heat.exc.Conflict as err: # continue even if support stack already exists. This # allows re-use of existing support stacks if multiple # runs reference the same support stack. LOG.info('Ignoring stack exists errors: %s', err) # clear the exception so polling heat later doesn't # continue to show the exception in the logs if sys.version_info < (3, 0): sys.exc_clear() except heat.exc.StackFailure as err: self.stackid = err.args[0] raise
def _deploy_from_hot(self, specification, base_dir=None): agents = generate_agents( nova.get_available_compute_nodes(self.openstack_client.nova), specification.get('accommodation') or specification.get('vm_accommodation'), self.stack_name) # render template by jinja vars_values = { 'agents': agents, 'unique': self.stack_name, } heat_template = utils.read_file(specification['template'], base_dir=base_dir) compiled_template = jinja2.Template(heat_template) rendered_template = compiled_template.render(vars_values) LOG.debug('Rendered template: %s', rendered_template) # create stack by Heat merged_parameters = { 'server_endpoint': self.server_endpoint, 'external_net': self.external_net, 'image': self.image_name, 'flavor': self.flavor_name, } merged_parameters.update(specification.get('template_parameters', {})) stack_id = heat.create_stack( self.openstack_client.heat, self.stack_name, rendered_template, merged_parameters) self.stack_created = True # get info about deployed objects outputs = heat.get_stack_outputs(self.openstack_client.heat, stack_id) override = self._get_override(specification.get('override')) return filter_agents(agents, outputs, override)
def _deploy_from_hot(self, specification): vm_accommodation = specification['vm_accommodation'] heat_template_name = specification['template'] template_parameters = specification['template_parameters'] heat_template = utils.read_file(heat_template_name) groups = self._make_groups(vm_accommodation) # render template by jinja vars_values = { 'groups': groups, } compiled_template = jinja2.Template(heat_template) rendered_template = compiled_template.render(vars_values) LOG.debug('Rendered template: %s', rendered_template) # create stack by Heat template_parameters['private_net_name'] = 'net_%s' % uuid.uuid4() template_parameters['server_endpoint'] = self.server_endpoint stack_params = { 'stack_name': self.stack_name, 'parameters': template_parameters, 'template': rendered_template, } stack = self.heat_client.stacks.create(**stack_params)['stack'] LOG.info('New stack: %s', stack) heat.wait_stack_completion(self.heat_client, stack['id']) # get info about deployed objects outputs_list = self.heat_client.stacks.get( stack['id']).to_dict()['outputs'] outputs = dict((item['output_key'], item) for item in outputs_list) # convert groups into agents return self._make_agents(groups, outputs)
def _deploy_from_hot(self, specification, base_dir=None): agents = generate_agents( nova.get_available_compute_nodes(self.openstack_client.nova), specification.get('accommodation') or specification.get('vm_accommodation'), self.stack_name) # render template by jinja vars_values = { 'agents': agents, 'unique': self.stack_name, } heat_template = utils.read_file(specification['template'], base_dir=base_dir) compiled_template = jinja2.Template(heat_template) rendered_template = compiled_template.render(vars_values) LOG.debug('Rendered template: %s', rendered_template) # create stack by Heat merged_parameters = { 'server_endpoint': self.server_endpoint, 'external_net': self.external_net, 'image': self.image_name, 'flavor': self.flavor_name, } merged_parameters.update(specification.get('template_parameters', {})) stack_id = heat.create_stack(self.openstack_client.heat, self.stack_name, rendered_template, merged_parameters) self.stack_created = True # get info about deployed objects outputs = heat.get_stack_outputs(self.openstack_client.heat, stack_id) override = self._get_override(specification.get('override')) return filter_agents(agents, outputs, override)
def generate_report(data, report_template, report_filename, subunit_filename, book_folder): calculate_stats(data['records'], data['tests']) sla_records = verify_sla(data['records'], data['tests']) data['sla'] = output_sla(sla_records) if subunit_filename: save_to_subunit(sla_records, subunit_filename) # add more filters to jinja jinja_env = jinja2.Environment(variable_start_string='[[[', variable_end_string=']]]', comment_start_string='[[#', comment_end_string='#]]') jinja_env.filters['json'] = json.dumps jinja_env.filters['yaml'] = functools.partial(yaml.safe_dump, indent=2, default_flow_style=False) alias_mapper = lambda f: config.REPORT_TEMPLATES + '%s.html' % f template = utils.read_file(report_template, alias_mapper=alias_mapper) compiled_template = jinja_env.from_string(template) rendered_template = compiled_template.render(dict(report=data)) if report_filename: LOG.debug('Writing report to: %s', report_filename) try: utils.write_file(rendered_template, report_filename) LOG.info('Report saved to: %s', report_filename) except IOError as e: LOG.error('Failed to write report file: %s', e) if book_folder: writer.write_book(book_folder, data)
def build_image(): openstack_client = init() flavor_name = cfg.CONF.flavor_name image_name = cfg.CONF.image_name dns_nameservers = cfg.CONF.dns_nameservers if nova.does_flavor_exist(openstack_client.nova, flavor_name): LOG.info('Using existing flavor: %s', flavor_name) else: try: nova.create_flavor(openstack_client.nova, name=flavor_name, ram=cfg.CONF.flavor_ram, vcpus=cfg.CONF.flavor_vcpus, disk=cfg.CONF.flavor_disk) LOG.info('Created flavor %s', flavor_name) except nova.ForbiddenException: LOG.error('User does not have permissions to create the flavor. ' 'Specify user with admin privileges or specify existing ' 'flavor via --flavor-name parameter.') exit(1) if glance.get_image(openstack_client.glance, image_name): LOG.info('Using existing image: %s', image_name) else: template = None template_filename = cfg.CONF.image_builder_template try: am = lambda f: config.IMAGE_BUILDER_TEMPLATES + '%s.yaml' % f template = utils.read_file(template_filename, alias_mapper=am) except IOError: LOG.error('Error reading template file: %s. ' 'Please verify correctness of --image-builder-template ' 'parameter', template_filename) exit(1) external_net = (cfg.CONF.external_net or neutron.choose_external_net(openstack_client.neutron)) stack_name = 'shaker_%s' % uuid.uuid4() stack_parameters = {'external_net': external_net, 'flavor': flavor_name, 'dns_nameservers': dns_nameservers} stack_id = None try: stack_id = heat.create_stack(openstack_client.heat, stack_name, template, stack_parameters) outputs = heat.get_stack_outputs(openstack_client.heat, stack_id) LOG.debug('Stack outputs: %s', outputs) LOG.debug('Waiting for server to shutdown') server_id = outputs['server_info'].get('id') nova.wait_server_shutdown(openstack_client.nova, server_id) LOG.debug('Making snapshot') openstack_client.nova.servers.create_image( server_id, image_name) LOG.debug('Waiting for server to snapshot') nova.wait_server_snapshot(openstack_client.nova, server_id) LOG.info('Created image: %s', image_name) except BaseException as e: if isinstance(e, KeyboardInterrupt): LOG.info('Caught SIGINT. Terminating') else: error_msg = 'Error while building the image: %s' % e LOG.error(error_msg) LOG.exception(e) finally: if stack_id and cfg.CONF.cleanup_on_error: LOG.debug('Cleaning up the stack: %s', stack_id) openstack_client.heat.stacks.delete(stack_id)
def build_image(): openstack_client = init() flavor_name = cfg.CONF.flavor_name image_name = cfg.CONF.image_name if nova.is_flavor_exists(openstack_client.nova, flavor_name): LOG.info('Using existing flavor: %s', flavor_name) else: openstack_client.nova.flavors.create(name=flavor_name, ram=1024, vcpus=1, disk=3) LOG.info('Created flavor %s', flavor_name) if glance.get_image(openstack_client.glance, image_name): LOG.info('Using existing image: %s', image_name) else: template = None template_filename = cfg.CONF.image_builder_template try: template = utils.read_file(template_filename) except IOError: LOG.error( 'Error reading template file: %s. ' 'Please verify correctness of --image-builder-template ' 'parameter', template_filename) exit(1) external_net = (cfg.CONF.external_net or neutron.choose_external_net( openstack_client.neutron)) stack_params = { 'stack_name': 'shaker_%s' % uuid.uuid4(), 'parameters': { 'external_net': external_net, 'flavor': flavor_name }, 'template': template, } stack = openstack_client.heat.stacks.create(**stack_params)['stack'] LOG.debug('New stack: %s', stack) heat.wait_stack_completion(openstack_client.heat, stack['id']) outputs = heat.get_stack_outputs(openstack_client.heat, stack['id']) LOG.debug('Stack outputs: %s', outputs) LOG.debug('Waiting for server to shutdown') server_id = outputs['server_info'].get('id') nova.wait_server_shutdown(openstack_client.nova, server_id) LOG.debug('Making snapshot') openstack_client.nova.servers.create_image(server_id, image_name) LOG.debug('Waiting for server to snapshot') nova.wait_server_snapshot(openstack_client.nova, server_id) LOG.debug('Clearing up') openstack_client.heat.stacks.delete(stack['id']) LOG.info('Created image: %s', image_name)