def update_configs(self): # Initialize the key pair name if self.config_scale['public_key_file']: # verify the public key file exists if not os.path.exists(self.config_scale['public_key_file']): LOG.error('Error: Invalid public key file: ' + self.config_scale['public_key_file']) sys.exit(1) else: # pick the user's public key if there is one pub_key = os.path.expanduser('~/.ssh/id_rsa.pub') if os.path.isfile(pub_key): self.config_scale['public_key_file'] = pub_key LOG.info('Using %s as public key for all VMs' % (pub_key)) # A bit of config dict surgery, extract out the client and server side # and transplant the remaining (common part) into the client and server dict self.server_cfg = self.config_scale.pop('server') self.client_cfg = self.config_scale.pop('client') self.server_cfg.update(self.config_scale) self.client_cfg.update(self.config_scale) # Hardcode a few client side options self.client_cfg.update(hardcoded_client_cfg) # Adjust the VMs per network on the client side to match the total # VMs on the server side (1:1) # There is an additional VM in client kloud as a proxy node self.client_cfg['vms_per_network'] =\ self.get_total_vm_count(self.server_cfg) + 1 # Use the default image name for Glance # defaults to something like "kloudbuster_v3" if not self.server_cfg['image_name']: self.server_cfg['image_name'] = kb_vm_agent.get_image_name() if not self.client_cfg['image_name']: self.client_cfg['image_name'] = kb_vm_agent.get_image_name()
def check_and_upload_images(self, retry_count=150): retry = 0 creds_list = [create_auth_session(self.server_cred), create_auth_session(self.client_cred)] creds_dict = dict(zip(['Server kloud', 'Client kloud'], creds_list)) img_name_dict = dict(zip(['Server kloud', 'Client kloud'], [self.server_cfg.image_name, self.client_cfg.image_name])) for kloud, sess in creds_dict.items(): glance_client = glanceclient.Client('1', session=sess) try: # Search for the image img = glance_client.images.list(filters={'name': img_name_dict[kloud]}).next() continue except StopIteration: pass # Trying to upload images kb_image_name = kb_vm_agent.get_image_name() + '.qcow2' image_url = 'http://storage.apps.openstack.org/images/%s' % kb_image_name LOG.info("Image is not found in %s, uploading from OpenStack App Store..." % kloud) try: img = glance_client.images.create(name=img_name_dict[kloud], disk_format="qcow2", container_format="bare", is_public=True, copy_from=image_url) while img.status in ['queued', 'saving'] and retry < retry_count: img = glance_client.images.find(name=img.name) retry = retry + 1 time.sleep(2) if img.status != 'active': raise Exception except glance_exception.HTTPForbidden: LOG.error("Cannot upload image without admin access. Please make sure the " "image is uploaded and is either public or owned by you.") return False except Exception: LOG.error("Failed while uploading the image, please make sure the cloud " "under test has the access to URL: %s." % image_url) return False return True
def check_and_upload_images(self): keystone_list = [create_keystone_client(self.server_cred)[0], create_keystone_client(self.client_cred)[0]] keystone_dict = dict(zip(['Server kloud', 'Client kloud'], keystone_list)) img_name_dict = dict(zip(['Server kloud', 'Client kloud'], [self.server_cfg.image_name, self.client_cfg.image_name])) for kloud, keystone in keystone_dict.items(): glance_endpoint = keystone.service_catalog.url_for( service_type='image', endpoint_type='publicURL') glance_client = glanceclient.Client(glance_endpoint, token=keystone.auth_token) try: # Search for the image glance_client.images.list(filters={'name': img_name_dict[kloud]}).next() return True except StopIteration: pass # Trying to upload images kb_image_name = 'dib/' + kb_vm_agent.get_image_name() + '.qcow2' if not os.path.exists(kb_image_name): LOG.error("VM Image not in Glance and could not find " + kb_image_name + " to upload, please refer to dib/README.rst for how to build" " image for KloudBuster.") return False LOG.info("Image is not found in %s, uploading %s..." % (kloud, kb_image_name)) with open(kb_image_name) as fimage: try: image = glance_client.images.create(name=img_name_dict[kloud], disk_format="qcow2", container_format="bare", visibility='public') glance_client.images.upload(image['id'], fimage) except glance_exception.HTTPForbidden: LOG.error("Cannot upload image without admin access. Please make sure the " "image is uploaded and is either public or owned by you.") return False return True
def main(): cli_opts = [ cfg.StrOpt("config", short="c", default=None, help="Override default values with a config file", metavar="<config file>"), cfg.BoolOpt("storage", default=False, help="Running KloudBuster to test storage performance"), cfg.BoolOpt("multicast", default=False, help="Running KloudBuster to test multicast performance"), cfg.StrOpt("topology", short="t", default=None, help="Topology file for compute hosts", metavar="<topology file>"), cfg.StrOpt("tenants-list", short="l", default=None, help="Existing tenant and user lists for reusing", metavar="<tenants file>"), cfg.StrOpt("rc", default=None, help="Tested cloud openrc credentials file (same as --tested-rc)", metavar="<rc file>"), cfg.StrOpt("tested-rc", default=None, help="Tested cloud openrc credentials file", metavar="<rc file>"), cfg.StrOpt("testing-rc", default=None, help="Testing cloud openrc credentials file", metavar="<rc file>"), cfg.StrOpt("passwd", default=None, secret=True, help="Tested cloud password (same as --tested-pwd)", metavar="<password>"), cfg.StrOpt("tested-passwd", default=None, secret=True, help="Tested cloud password", metavar="<password>"), cfg.StrOpt("testing-passwd", default=None, secret=True, help="Testing cloud password", metavar="<password>"), cfg.StrOpt("html", default=None, help='store results in HTML file', metavar="<dest html file>"), cfg.StrOpt("label", default=None, help='label for the title in HTML file', metavar="<title>"), cfg.BoolOpt("headless", default=False, help="do not show chart in the browser (default=False, only used if --html)"), cfg.StrOpt("json", default=None, help='store results in JSON format file', metavar="<dest json file>"), cfg.StrOpt("csv", default=None, help='store results in CSV format, multicast only.', metavar="<csv file>"), cfg.BoolOpt("no-env", default=False, help="Do not read env variables"), cfg.BoolOpt("show-config", default=False, help="Show the default configuration"), cfg.StrOpt("charts-from-json", default=None, help='create charts from json results and exit (requires --html)', metavar="<source json file>"), ] CONF.register_cli_opts(cli_opts) CONF.set_default("verbose", True) full_version = __version__ + ', VM image: ' + kb_vm_agent.get_image_name() CONF(sys.argv[1:], project="kloudbuster", version=full_version) logging.setup("kloudbuster") if CONF.rc and not CONF.tested_rc: CONF.tested_rc = CONF.rc if CONF.passwd and not CONF.tested_passwd: CONF.tested_passwd = CONF.passwd if CONF.charts_from_json: if not CONF.html: LOG.error('Destination html filename must be specified using --html.') sys.exit(1) with open(CONF.charts_from_json, 'r') as jfp: json_results = json.load(jfp) generate_charts(json_results, CONF.html, None) sys.exit(0) if CONF.show_config: print resource_string(__name__, "cfg.scale.yaml") sys.exit(0) if CONF.multicast and CONF.storage: LOG.error('--multicast and --storage can not both be chosen.') sys.exit(1) try: kb_config = KBConfig() kb_config.init_with_cli() except TypeError: LOG.error('Error parsing the configuration file') sys.exit(1) # The KloudBuster class is just a wrapper class # levarages tenant and user class for resource creations and deletion kloudbuster = KloudBuster( kb_config.cred_tested, kb_config.cred_testing, kb_config.server_cfg, kb_config.client_cfg, kb_config.topo_cfg, kb_config.tenants_list, storage_mode=CONF.storage, multicast_mode=CONF.multicast) if kloudbuster.check_and_upload_images(): kloudbuster.run() if CONF.json: '''Save results in JSON format file.''' LOG.info('Saving results in json file: ' + CONF.json + "...") with open(CONF.json, 'w') as jfp: json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True) if CONF.multicast and CONF.csv and 'kb_result' in kloudbuster.final_result: '''Save results in JSON format file.''' if len(kloudbuster.final_result['kb_result']) > 0: LOG.info('Saving results in csv file: ' + CONF.csv + "...") with open(CONF.csv, 'w') as jfp: jfp.write(KBRunner_Multicast.json_to_csv(kloudbuster.final_result['kb_result'][0])) if CONF.html: generate_charts(kloudbuster.final_result, CONF.html, kb_config.config_scale)
def main(): cli_opts = [ cfg.StrOpt("config", short="c", default=None, help="Override default values with a config file"), cfg.StrOpt("topology", short="t", default=None, help="Topology files for compute hosts"), cfg.StrOpt("tenants-list", short="l", default=None, help="Existing tenant and user lists for reusing"), cfg.StrOpt("tested-rc", default=None, help="Tested cloud openrc credentials file"), cfg.StrOpt("testing-rc", default=None, help="Testing cloud openrc credentials file"), cfg.StrOpt("tested-passwd", default=None, secret=True, help="Tested cloud password"), cfg.StrOpt("testing-passwd", default=None, secret=True, help="Testing cloud password"), cfg.StrOpt("json", default=None, help='store results in JSON format file'), cfg.BoolOpt("no-env", default=False, help="Do not read env variables"), cfg.BoolOpt("show-config", default=False, help="Show the default configuration") ] CONF.register_cli_opts(cli_opts) CONF.set_default("verbose", True) full_version = __version__ + ', VM image: ' + kb_vm_agent.get_image_name() CONF(sys.argv[1:], project="kloudbuster", version=full_version) if CONF.show_config: print resource_string(__name__, "cfg.scale.yaml") sys.exit(0) logging.setup("kloudbuster") try: kb_config = KBConfig() kb_config.init_with_cli() except TypeError: LOG.error('Error parsing the configuration file') sys.exit(1) # The KloudBuster class is just a wrapper class # levarages tenant and user class for resource creations and deletion kloudbuster = KloudBuster( kb_config.cred_tested, kb_config.cred_testing, kb_config.server_cfg, kb_config.client_cfg, kb_config.topo_cfg, kb_config.tenants_list) if kloudbuster.check_and_upload_images(): kloudbuster.run() if CONF.json: '''Save results in JSON format file.''' LOG.info('Saving results in json file: ' + CONF.json + "...") with open(CONF.json, 'w') as jfp: json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True)
def update_configs(self): # Initialize the key pair name if self.config_scale['public_key_file']: # verify the public key file exists if not os.path.exists(self.config_scale['public_key_file']): LOG.error('Error: Invalid public key file: ' + self.config_scale['public_key_file']) sys.exit(1) else: # pick the user's public key if there is one pub_key = os.path.expanduser('~/.ssh/id_rsa.pub') if os.path.isfile(pub_key): self.config_scale['public_key_file'] = pub_key LOG.info('Using %s as public key for all VMs' % (pub_key)) else: LOG.warning('No public key is found or specified to instantiate VMs. ' 'You will not be able to access the VMs spawned by KloudBuster.') if self.storage_mode: disk_size = self.config_scale.client.storage_stage_configs.disk_size io_file_size = self.config_scale.client.storage_stage_configs.io_file_size if not disk_size: LOG.error('You have to specify a disk size in order to run storage tests.') raise KBConfigParseException() if io_file_size > disk_size: LOG.error('io_file_size must be less or eqaul than disk_size.') raise KBConfigParseException() if self.alt_cfg: self.config_scale = self.config_scale + AttrDict(self.alt_cfg) # Use the default image name for Glance # defaults to something like "kloudbuster_v3" if not self.config_scale['image_name']: self.config_scale['image_name'] = kb_vm_agent.get_image_name() # A bit of config dict surgery, extract out the client and server side # and transplant the remaining (common part) into the client and server dict self.server_cfg = AttrDict(self.config_scale.pop('server')) self.client_cfg = AttrDict(self.config_scale.pop('client')) self.server_cfg.update(self.config_scale) self.client_cfg.update(self.config_scale) # Hardcode a few client side options self.client_cfg.update(hardcoded_client_cfg) # Adjust the VMs per network on the client side to match the total # VMs on the server side (1:1) # There is an additional VM in client kloud as a proxy node if self.storage_mode: self.client_cfg['vms_per_network'] = \ self.client_cfg.storage_stage_configs.vm_count + 1 else: self.client_cfg['vms_per_network'] = \ self.get_total_vm_count(self.server_cfg) + 1 # If multicast mode, the number of receivers is specified in the multicast config instead. if self.multicast_mode: self.server_cfg['vms_per_network'] =\ self.client_cfg['multicast_tool_configs']['receivers'][-1] self.config_scale['server'] = self.server_cfg self.config_scale['client'] = self.client_cfg # missing rate or rate_iops = 0 = no-limit # note we need to use key based access to modify the content # (self.config_scale['client'].storage_tool_configs will make a shallow copy) for tc in self.config_scale['client']['storage_tool_configs']: if 'rate' not in tc: tc['rate'] = '0' if 'rate_iops' not in tc: tc['rate_iops'] = 0