def update_configs(self):
# Initialize the key pair name
if self.config_scale['public_key_file']:
# verify the public key file exists
if not os.path.exists(self.config_scale['public_key_file']):
LOG.error('Error: Invalid public key file: ' + self.config_scale['public_key_file'])
sys.exit(1)
else:
# pick the user's public key if there is one
pub_key = os.path.expanduser('~/.ssh/id_rsa.pub')
if os.path.isfile(pub_key):
self.config_scale['public_key_file'] = pub_key
LOG.info('Using %s as public key for all VMs' % (pub_key))
# A bit of config dict surgery, extract out the client and server side
# and transplant the remaining (common part) into the client and server dict
self.server_cfg = self.config_scale.pop('server')
self.client_cfg = self.config_scale.pop('client')
self.server_cfg.update(self.config_scale)
self.client_cfg.update(self.config_scale)
# Hardcode a few client side options
self.client_cfg.update(hardcoded_client_cfg)
# Adjust the VMs per network on the client side to match the total
# VMs on the server side (1:1)
# There is an additional VM in client kloud as a proxy node
self.client_cfg['vms_per_network'] =\
self.get_total_vm_count(self.server_cfg) + 1
# Use the default image name for Glance
# defaults to something like "kloudbuster_v3"
if not self.server_cfg['image_name']:
self.server_cfg['image_name'] = kb_vm_agent.get_image_name()
if not self.client_cfg['image_name']:
self.client_cfg['image_name'] = kb_vm_agent.get_image_name()
def check_and_upload_images(self, retry_count=150):
retry = 0
creds_list = [create_auth_session(self.server_cred),
create_auth_session(self.client_cred)]
creds_dict = dict(zip(['Server kloud', 'Client kloud'], creds_list))
img_name_dict = dict(zip(['Server kloud', 'Client kloud'],
[self.server_cfg.image_name, self.client_cfg.image_name]))
for kloud, sess in creds_dict.items():
glance_client = glanceclient.Client('1', session=sess)
try:
# Search for the image
img = glance_client.images.list(filters={'name': img_name_dict[kloud]}).next()
continue
except StopIteration:
pass
# Trying to upload images
kb_image_name = kb_vm_agent.get_image_name() + '.qcow2'
image_url = 'http://storage.apps.openstack.org/images/%s' % kb_image_name
LOG.info("Image is not found in %s, uploading from OpenStack App Store..." % kloud)
try:
img = glance_client.images.create(name=img_name_dict[kloud],
disk_format="qcow2",
container_format="bare",
is_public=True,
copy_from=image_url)
while img.status in ['queued', 'saving'] and retry < retry_count:
img = glance_client.images.find(name=img.name)
retry = retry + 1
time.sleep(2)
if img.status != 'active':
raise Exception
except glance_exception.HTTPForbidden:
LOG.error("Cannot upload image without admin access. Please make sure the "
"image is uploaded and is either public or owned by you.")
return False
except Exception:
LOG.error("Failed while uploading the image, please make sure the cloud "
"under test has the access to URL: %s." % image_url)
return False
return True
def check_and_upload_images(self):
keystone_list = [create_keystone_client(self.server_cred)[0],
create_keystone_client(self.client_cred)[0]]
keystone_dict = dict(zip(['Server kloud', 'Client kloud'], keystone_list))
img_name_dict = dict(zip(['Server kloud', 'Client kloud'],
[self.server_cfg.image_name, self.client_cfg.image_name]))
for kloud, keystone in keystone_dict.items():
glance_endpoint = keystone.service_catalog.url_for(
service_type='image', endpoint_type='publicURL')
glance_client = glanceclient.Client(glance_endpoint, token=keystone.auth_token)
try:
# Search for the image
glance_client.images.list(filters={'name': img_name_dict[kloud]}).next()
return True
except StopIteration:
pass
# Trying to upload images
kb_image_name = 'dib/' + kb_vm_agent.get_image_name() + '.qcow2'
if not os.path.exists(kb_image_name):
LOG.error("VM Image not in Glance and could not find " + kb_image_name +
" to upload, please refer to dib/README.rst for how to build"
" image for KloudBuster.")
return False
LOG.info("Image is not found in %s, uploading %s..." % (kloud, kb_image_name))
with open(kb_image_name) as fimage:
try:
image = glance_client.images.create(name=img_name_dict[kloud],
disk_format="qcow2",
container_format="bare",
visibility='public')
glance_client.images.upload(image['id'], fimage)
except glance_exception.HTTPForbidden:
LOG.error("Cannot upload image without admin access. Please make sure the "
"image is uploaded and is either public or owned by you.")
return False
return True
def main():
cli_opts = [
cfg.StrOpt("config",
short="c",
default=None,
help="Override default values with a config file",
metavar="<config file>"),
cfg.BoolOpt("storage",
default=False,
help="Running KloudBuster to test storage performance"),
cfg.BoolOpt("multicast",
default=False,
help="Running KloudBuster to test multicast performance"),
cfg.StrOpt("topology",
short="t",
default=None,
help="Topology file for compute hosts",
metavar="<topology file>"),
cfg.StrOpt("tenants-list",
short="l",
default=None,
help="Existing tenant and user lists for reusing",
metavar="<tenants file>"),
cfg.StrOpt("rc",
default=None,
help="Tested cloud openrc credentials file (same as --tested-rc)",
metavar="<rc file>"),
cfg.StrOpt("tested-rc",
default=None,
help="Tested cloud openrc credentials file",
metavar="<rc file>"),
cfg.StrOpt("testing-rc",
default=None,
help="Testing cloud openrc credentials file",
metavar="<rc file>"),
cfg.StrOpt("passwd",
default=None,
secret=True,
help="Tested cloud password (same as --tested-pwd)",
metavar="<password>"),
cfg.StrOpt("tested-passwd",
default=None,
secret=True,
help="Tested cloud password",
metavar="<password>"),
cfg.StrOpt("testing-passwd",
default=None,
secret=True,
help="Testing cloud password",
metavar="<password>"),
cfg.StrOpt("html",
default=None,
help='store results in HTML file',
metavar="<dest html file>"),
cfg.StrOpt("label",
default=None,
help='label for the title in HTML file',
metavar="<title>"),
cfg.BoolOpt("headless",
default=False,
help="do not show chart in the browser (default=False, only used if --html)"),
cfg.StrOpt("json",
default=None,
help='store results in JSON format file',
metavar="<dest json file>"),
cfg.StrOpt("csv",
default=None,
help='store results in CSV format, multicast only.',
metavar="<csv file>"),
cfg.BoolOpt("no-env",
default=False,
help="Do not read env variables"),
cfg.BoolOpt("show-config",
default=False,
help="Show the default configuration"),
cfg.StrOpt("charts-from-json",
default=None,
help='create charts from json results and exit (requires --html)',
metavar="<source json file>"),
]
CONF.register_cli_opts(cli_opts)
CONF.set_default("verbose", True)
full_version = __version__ + ', VM image: ' + kb_vm_agent.get_image_name()
CONF(sys.argv[1:], project="kloudbuster", version=full_version)
logging.setup("kloudbuster")
if CONF.rc and not CONF.tested_rc:
CONF.tested_rc = CONF.rc
if CONF.passwd and not CONF.tested_passwd:
CONF.tested_passwd = CONF.passwd
if CONF.charts_from_json:
if not CONF.html:
LOG.error('Destination html filename must be specified using --html.')
sys.exit(1)
with open(CONF.charts_from_json, 'r') as jfp:
json_results = json.load(jfp)
generate_charts(json_results, CONF.html, None)
sys.exit(0)
if CONF.show_config:
print resource_string(__name__, "cfg.scale.yaml")
sys.exit(0)
if CONF.multicast and CONF.storage:
LOG.error('--multicast and --storage can not both be chosen.')
sys.exit(1)
try:
kb_config = KBConfig()
kb_config.init_with_cli()
except TypeError:
LOG.error('Error parsing the configuration file')
sys.exit(1)
# The KloudBuster class is just a wrapper class
# levarages tenant and user class for resource creations and deletion
kloudbuster = KloudBuster(
kb_config.cred_tested, kb_config.cred_testing,
kb_config.server_cfg, kb_config.client_cfg,
kb_config.topo_cfg, kb_config.tenants_list,
storage_mode=CONF.storage, multicast_mode=CONF.multicast)
if kloudbuster.check_and_upload_images():
kloudbuster.run()
if CONF.json:
'''Save results in JSON format file.'''
LOG.info('Saving results in json file: ' + CONF.json + "...")
with open(CONF.json, 'w') as jfp:
json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True)
if CONF.multicast and CONF.csv and 'kb_result' in kloudbuster.final_result:
'''Save results in JSON format file.'''
if len(kloudbuster.final_result['kb_result']) > 0:
LOG.info('Saving results in csv file: ' + CONF.csv + "...")
with open(CONF.csv, 'w') as jfp:
jfp.write(KBRunner_Multicast.json_to_csv(kloudbuster.final_result['kb_result'][0]))
if CONF.html:
generate_charts(kloudbuster.final_result, CONF.html, kb_config.config_scale)
def main():
cli_opts = [
cfg.StrOpt("config",
short="c",
default=None,
help="Override default values with a config file"),
cfg.StrOpt("topology",
short="t",
default=None,
help="Topology files for compute hosts"),
cfg.StrOpt("tenants-list",
short="l",
default=None,
help="Existing tenant and user lists for reusing"),
cfg.StrOpt("tested-rc",
default=None,
help="Tested cloud openrc credentials file"),
cfg.StrOpt("testing-rc",
default=None,
help="Testing cloud openrc credentials file"),
cfg.StrOpt("tested-passwd",
default=None,
secret=True,
help="Tested cloud password"),
cfg.StrOpt("testing-passwd",
default=None,
secret=True,
help="Testing cloud password"),
cfg.StrOpt("json",
default=None,
help='store results in JSON format file'),
cfg.BoolOpt("no-env",
default=False,
help="Do not read env variables"),
cfg.BoolOpt("show-config",
default=False,
help="Show the default configuration")
]
CONF.register_cli_opts(cli_opts)
CONF.set_default("verbose", True)
full_version = __version__ + ', VM image: ' + kb_vm_agent.get_image_name()
CONF(sys.argv[1:], project="kloudbuster", version=full_version)
if CONF.show_config:
print resource_string(__name__, "cfg.scale.yaml")
sys.exit(0)
logging.setup("kloudbuster")
try:
kb_config = KBConfig()
kb_config.init_with_cli()
except TypeError:
LOG.error('Error parsing the configuration file')
sys.exit(1)
# The KloudBuster class is just a wrapper class
# levarages tenant and user class for resource creations and deletion
kloudbuster = KloudBuster(
kb_config.cred_tested, kb_config.cred_testing,
kb_config.server_cfg, kb_config.client_cfg,
kb_config.topo_cfg, kb_config.tenants_list)
if kloudbuster.check_and_upload_images():
kloudbuster.run()
if CONF.json:
'''Save results in JSON format file.'''
LOG.info('Saving results in json file: ' + CONF.json + "...")
with open(CONF.json, 'w') as jfp:
json.dump(kloudbuster.final_result, jfp, indent=4, sort_keys=True)
def update_configs(self):
# Initialize the key pair name
if self.config_scale['public_key_file']:
# verify the public key file exists
if not os.path.exists(self.config_scale['public_key_file']):
LOG.error('Error: Invalid public key file: ' + self.config_scale['public_key_file'])
sys.exit(1)
else:
# pick the user's public key if there is one
pub_key = os.path.expanduser('~/.ssh/id_rsa.pub')
if os.path.isfile(pub_key):
self.config_scale['public_key_file'] = pub_key
LOG.info('Using %s as public key for all VMs' % (pub_key))
else:
LOG.warning('No public key is found or specified to instantiate VMs. '
'You will not be able to access the VMs spawned by KloudBuster.')
if self.storage_mode:
disk_size = self.config_scale.client.storage_stage_configs.disk_size
io_file_size = self.config_scale.client.storage_stage_configs.io_file_size
if not disk_size:
LOG.error('You have to specify a disk size in order to run storage tests.')
raise KBConfigParseException()
if io_file_size > disk_size:
LOG.error('io_file_size must be less or eqaul than disk_size.')
raise KBConfigParseException()
if self.alt_cfg:
self.config_scale = self.config_scale + AttrDict(self.alt_cfg)
# Use the default image name for Glance
# defaults to something like "kloudbuster_v3"
if not self.config_scale['image_name']:
self.config_scale['image_name'] = kb_vm_agent.get_image_name()
# A bit of config dict surgery, extract out the client and server side
# and transplant the remaining (common part) into the client and server dict
self.server_cfg = AttrDict(self.config_scale.pop('server'))
self.client_cfg = AttrDict(self.config_scale.pop('client'))
self.server_cfg.update(self.config_scale)
self.client_cfg.update(self.config_scale)
# Hardcode a few client side options
self.client_cfg.update(hardcoded_client_cfg)
# Adjust the VMs per network on the client side to match the total
# VMs on the server side (1:1)
# There is an additional VM in client kloud as a proxy node
if self.storage_mode:
self.client_cfg['vms_per_network'] = \
self.client_cfg.storage_stage_configs.vm_count + 1
else:
self.client_cfg['vms_per_network'] = \
self.get_total_vm_count(self.server_cfg) + 1
# If multicast mode, the number of receivers is specified in the multicast config instead.
if self.multicast_mode:
self.server_cfg['vms_per_network'] =\
self.client_cfg['multicast_tool_configs']['receivers'][-1]
self.config_scale['server'] = self.server_cfg
self.config_scale['client'] = self.client_cfg
# missing rate or rate_iops = 0 = no-limit
# note we need to use key based access to modify the content
# (self.config_scale['client'].storage_tool_configs will make a shallow copy)
for tc in self.config_scale['client']['storage_tool_configs']:
if 'rate' not in tc:
tc['rate'] = '0'
if 'rate_iops' not in tc:
tc['rate_iops'] = 0
