def main():
Flags.PARSER.add_argument(
'--input_file',
type=str,
required=True,
help='CSV containing values to validate',
)
Flags.PARSER.add_argument(
'--datasource',
type=str,
default=DATASOURCE.name,
help='Datasource to validate against',
)
Flags.PARSER.add_argument(
'--output_file',
type=str,
required=True,
help='Output CSV to write validation results',
)
Flags.InitArgs()
input_file = Flags.ARGS.input_file
dimensions = _extract_dimensions(input_file)
LOG.info('Starting validation over dimensions: %s', dimensions)
validator = PivotedCSVValidator(Flags.ARGS.datasource, dimensions)
validator.parse_and_run(input_file, Flags.ARGS.output_file)
return validator.passed_validation
def process_batches(self):
'''
Pack results from Druid into our format.
'''
LOG.info('Processing batches...')
geo_results = self.results['byGeo']
batch_warnings = []
for batch in self.batches:
for geo_key, geo_result in list(batch.result.items()):
if geo_result.has_data():
if geo_key not in geo_results:
geo_results[geo_key] = GeoResult(geo_result.metadata)
geo_results[geo_key].data.update(geo_result.data)
batch_warnings.extend(geo_results[geo_key].warnings)
if len(batch_warnings):
LOG.warn('!! You overwrote existing data. Run in dev for detailed debug.')
if not IS_PRODUCTION:
# Only show these errors in dev. In production, these outputs
# can overwhelm slow disks.
print('\n'.join(batch_warnings))
return True
def measure_flow(self, label, target_ip):
label = self.add_location(label)
FlowPrinter.print_desc(label)
# results for this flow as a dict
perf_output = self.client.run_client(label, target_ip,
self.server,
bandwidth=self.config.vm_bandwidth,
az_to=self.server.az)
if self.config.keep_first_flow_and_exit:
CONLOG.info(self.rescol.ppr.pformat(perf_output))
FILELOG.info(json.dumps(perf_output, sort_keys=True))
LOG.info('Stopping execution after first flow, cleanup all VMs/networks manually')
sys.exit(0)
if self.config.stop_on_error:
# check if there is any error in the results
results_list = perf_output['results']
for res_dict in results_list:
if 'error' in res_dict:
LOG.error('Stopping execution on error, cleanup all VMs/networks manually')
CONLOG.info(self.rescol.ppr.pformat(perf_output))
FILELOG.info(json.dumps(perf_output, sort_keys=True))
sys.exit(2)
self.rescol.add_flow_result(perf_output)
CONLOG.info(self.rescol.ppr.pformat(perf_output))
FILELOG.info(json.dumps(perf_output, sort_keys=True))
def __add_router_interface(self):
# and pick the first in the list - the list should be non empty and
# contain only 1 subnet since it is supposed to be a private network
# But first check that the router does not already have this subnet
# so retrieve the list of all ports, then check if there is one port
# - matches the subnet
# - and is attached to the router
# Assumed that both management networks are created together so checking for one of them
ports = self.neutron_client.list_ports()['ports']
for port in ports:
# Skip the check on stale ports
if port['fixed_ips']:
port_ip = port['fixed_ips'][0]
if (port['device_id'] == self.ext_router['id']) and \
(port_ip['subnet_id'] == self.vm_int_net[0]['subnets'][0]):
LOG.info(
'Ext router already associated to the internal network.'
)
return
for int_net in self.vm_int_net:
body = {'subnet_id': int_net['subnets'][0]}
self.neutron_client.add_interface_router(self.ext_router['id'],
body)
LOG.debug('Ext router associated to ' + int_net['name'])
# If ipv6 is enabled than add second subnet
if self.ipv6_enabled:
body = {'subnet_id': int_net['subnets'][1]}
self.neutron_client.add_interface_router(
self.ext_router['id'], body)
def save_json_result(result, json_file, std_json_path, service_chain,
service_chain_count, flow_count, frame_sizes):
"""Save results in json format file."""
filepaths = []
if json_file:
filepaths.append(json_file)
if std_json_path:
name_parts = [
service_chain,
str(service_chain_count),
str(flow_count)
] + list(frame_sizes)
filename = '-'.join(name_parts) + '.json'
filepaths.append(os.path.join(std_json_path, filename))
if filepaths:
for file_path in filepaths:
LOG.info('Saving results in json file: %s...', file_path)
with open(file_path, 'w') as jfp:
json.dump(result,
jfp,
indent=4,
sort_keys=True,
separators=(',', ': '),
default=lambda obj: obj.to_json())
def __add_router_interface(self):
# and pick the first in the list - the list should be non empty and
# contain only 1 subnet since it is supposed to be a private network
# But first check that the router does not already have this subnet
# so retrieve the list of all ports, then check if there is one port
# - matches the subnet
# - and is attached to the router
# Assumed that both management networks are created together so checking for one of them
ports = self.neutron_client.list_ports()['ports']
for port in ports:
# Skip the check on stale ports
if port['fixed_ips']:
port_ip = port['fixed_ips'][0]
if (port['device_id'] == self.ext_router['id']) and \
(port_ip['subnet_id'] == self.vm_int_net[0]['subnets'][0]):
LOG.info('Ext router already associated to the internal network.')
return
for int_net in self.vm_int_net:
body = {
'subnet_id': int_net['subnets'][0]
}
self.neutron_client.add_interface_router(self.ext_router['id'], body)
LOG.debug('Ext router associated to ' + int_net['name'])
# If ipv6 is enabled than add second subnet
if self.ipv6_enabled:
body = {
'subnet_id': int_net['subnets'][1]
}
self.neutron_client.add_interface_router(self.ext_router['id'], body)
def run_server(self, generator_config, filename='/etc/trex_cfg.yaml'):
"""Run TRex server for specified traffic profile.
:param traffic_profile: traffic profile object based on config file
:param filename: path where to save TRex config file
"""
cfg = self.__save_config(generator_config, filename)
cores = generator_config.cores
vtep_vlan = generator_config.gen_config.get('vtep_vlan')
sw_mode = "--software" if generator_config.software_mode else ""
vlan_opt = "--vlan" if (generator_config.vlan_tagging or vtep_vlan) else ""
if generator_config.mbuf_factor:
mbuf_opt = "--mbuf-factor " + str(generator_config.mbuf_factor)
else:
mbuf_opt = ""
hdrh_opt = "--hdrh" if generator_config.hdrh else ""
# --unbind-unused-ports: for NIC that have more than 2 ports such as Intel X710
# this will instruct trex to unbind all ports that are unused instead of
# erroring out with an exception (i40e only)
cmd = ['nohup', '/bin/bash', '-c',
'./t-rex-64 -i -c {} --iom 0 --no-scapy-server '
'--unbind-unused-ports --close-at-end {} {} '
'{} {} --cfg {} &> /tmp/trex.log & disown'.format(cores, sw_mode,
vlan_opt,
hdrh_opt,
mbuf_opt, cfg)]
LOG.info(' '.join(cmd))
subprocess.Popen(cmd, cwd=self.trex_dir)
LOG.info('TRex server is running...')
def main(argv):
app = Flask(__name__)
restful_api = Api(app)
from log import LOG
from settings import settings
try:
save_pid()
load_unversioned_api(restful_api)
load_v1_api(restful_api)
except Exception as e:
if settings.DEBUG:
traceback.print_exc()
LOG.error(str(e))
LOG.error('exit 1')
sys.exit(1)
LOG.info('Started')
app.run(host=settings.BINDING_ADDR,
port=settings.PORT,
debug=settings.DEBUG,
threaded=settings.USE_THREAD,
use_reloader=settings.USE_RELOADER)
def check_pod(self, pod, app_controller):
if not pod['is_ready'] or not pod['is_running']:
return
if app_controller.memory_controller is None:
return
memory_percent = round(
float(pod['mem_usage'] - pod['mem_cache']) /
float(pod['max_mem_limit']) * 100, 1)
if memory_percent >= app_controller.memory_controller.warn_percent:
msg = '{}:{}:cache={},usage={},percent={}%'.format(
app_controller.name, pod['name'], pod['mem_cache'],
pod['mem_usage'], memory_percent)
alert_record = AlertRecord(key=pod['name'],
alert_type=AlertType.MEM,
alert_level=AlertLevel.WARN,
timestamp=int(time.time()),
msg=msg,
hostname=pod['host_IP'],
mail_list=app_controller.mail_list,
phone_list=app_controller.phone_list)
if Alert.alert(alert_record):
LOG.info('sent mail: {}'.format(msg))
LOG.warn(msg)
else:
Alert.remove_alert_history(pod['name'])
def get(self):
LOG.info('get ImageList')
prefix = None
image_type = request.args.get('type', None)
app_name = request.args.get('app_name', None)
if (image_type not in ['app_base', 'app', 'team_ol']
and app_name is not None):
return make_status_response(400, 'Arguments are not valid')
if image_type is not None:
prefix = image_type + '/'
if app_name is not None:
prefix += app_name + '/'
images = self.registry.get_images(prefix)
reply = {}
reply['kind'] = 'ImageList'
reply['images'] = []
for name, tags in images.items():
reply['images'].append(Image(name, tags).to_dict())
response = flask.make_response(json.dumps(reply))
response.headers['Access-Control-Allow-Origin'] = '*'
return response
def delete(self):
LOG.info('delete ImageList')
prefix = None
image_type = request.args.get('type', None)
if not self.is_image_type_valid(image_type):
return make_status_response(400, 'Arguments are not valid')
app_name = request.args.get('app_name', None)
name = request.args.get('name', None)
version = request.args.get('version', None)
if name is None or version is None:
return make_status_response(400, 'Image name and version must be specified.')
if image_type != 'lib' and app_name is None:
return make_status_response(400, 'app_name must be specified')
if image_type == 'lib':
image_full_name = 'lib/' + name
else:
image_full_name = '{}/{}/{}'.format(image_type, app_name, name)
try:
self.registry.delete_image(image_full_name, version)
except HttpException as e:
if e.status_code == 404:
return make_status_response(404, 'image not found')
except Exception as e:
return make_status_response(500, '')
return make_status_response(200, 'OK')
def run(self):
L.info("Command Send : %s" % self.command)
args = self.command.split(" ")
subproc_args = { 'stdin' : subprocess.PIPE,
'stdout' : subprocess.PIPE,
'stderr' : subprocess.STDOUT,
}
try:
proc = subprocess.Popen(args, **subproc_args)
except OSError:
L.info("Failed to execute command: %s" % args[0])
sys.exit(1)
(stdouterr, stdin) = (proc.stdout, proc.stdin)
Li = []
result = None
if stdouterr is None:
pass
else:
while True:
line = stdouterr.readline()
if not line:
break
Li.append(line.rstrip())
result = "".join(Li)
code = proc.wait()
time.sleep(2)
L.debug("Command Resturn Code: %d" % code)
self.queue.put(result)
def reset_irq_affinity(self, uuid, irqs=None, msi_irqs=None):
"""Reset irq affinity for instance
The instance has already been deleted or
related PCI not used by it anymore.
"""
if irqs or msi_irqs:
# reset irq affinity for specified irqs
_irqs = irqs
_msi_irqs = msi_irqs
elif uuid in self.inst_dict:
# reset all irq affinity for deleted instance
_irqs = self.inst_dict[uuid][0]
_msi_irqs = self.inst_dict[uuid][1]
else:
LOG.debug("No pci affinity need to be reset for instance=%s!" %
uuid)
return
try:
with open('/proc/irq/default_smp_affinity') as f:
cpulist = f.readline().strip()
LOG.debug("default smp affinity bitmap:%s" % cpulist)
for x in [_irqs, _msi_irqs]:
if len(x) > 0:
pci_utils.set_irq_affinity(True, x, cpulist)
except Exception as e:
LOG.error("Failed to reset smp affinity! error=%s" % e)
LOG.info("Reset smp affinity done for instance=%s!" % uuid)
def receiveMsg_TCP(self, datapacket=False):
if self.socket_tcp is None:
raise _error.ConnectionError("no tcp socket for receiving")
raw = ""
note = ""
data, readcounter = self.receive_bytes(self.socket_tcp,
const.HEADER.size)
if readcounter > 1:
note += "HEADER SEGMENTED INTO %s SEGMENTS!" % readcounter
raw += data
size, command = networking.parsePacketHeader(data)
if not command in (const.PACKET_SERVER_FRAME,
const.PACKET_SERVER_SYNC):
if command in const.packet_names:
LOG.debug("received size: %d, command: %s (%d)" %
(size, const.packet_names[command], command))
else:
LOG.debug("received size: %d, command: %d" % (size, command))
size -= const.HEADER.size # remove size of the header ...
data, readcounter = self.receive_bytes(self.socket_tcp, size)
if readcounter > 1:
note += "DATA SEGMENTED INTO %s SEGMENTS!" % readcounter
raw += data
if not self.running:
return None
if len(note) > 0:
LOG.info(note)
content = data
if datapacket:
return DataPacket(size, command, content)
else:
return size, command, content
def create_flavor(self, flavor_type, vcpus, ram, disk, ephemeral=0, extra_specs=None):
flavor = self.novaclient.flavors.create(name=flavor_type, vcpus=vcpus, ram=ram, disk=disk,
ephemeral=ephemeral)
if extra_specs:
flavor.set_keys(extra_specs)
LOG.info('Flavor %s created.' % flavor_type)
return flavor
async def on_user_exit(self, event: TrackingEvent) -> None:
if event.user.name in self.users:
del self.users[event.user.name]
if len(self.users) == 0:
LOG.info('Last user let, shutting down system')
e_bus.emit('tracking.action.shutdown', Event())
def run(self):
L.info("Command Send : %s" % self.command)
args = self.command.split(" ")
subproc_args = { 'stdin' : subprocess.PIPE,
'stdout' : subprocess.PIPE,
'stderr' : subprocess.STDOUT,
}
try:
proc = subprocess.Popen(args, **subproc_args)
except OSError:
L.info("Failed to execute command: %s" % args[0])
sys.exit(1)
(stdouterr, stdin) = (proc.stdout, proc.stdin)
if stdouterr is None:
pass
else:
with open(self.filepath, "w") as f:
while True:
line = stdouterr.readline()
if not line:
break
f.write(line)
code = proc.wait()
time.sleep(2)
L.debug("Command Resturn Code: %d" % code)
def __init__(self, nova_client, name):
self.name = name
LOG.info('Discovering flavor %s...', name)
try:
self.flavor = nova_client.flavors.find(name=name)
except NotFound:
self.flavor = None
def __run_search_iteration(self, rate):
"""Run one iteration at the given rate level.
rate: the rate to send on each port in percent (0 to 100)
"""
self._modify_load(rate)
# poll interval stats and collect them
for stats in self.run_traffic():
self.interval_collector.add(stats)
time_elapsed_ratio = self.runner.time_elapsed() / self.run_config['duration_sec']
if time_elapsed_ratio >= 1:
self.cancel_traffic()
if not self.skip_sleep():
time.sleep(self.config.pause_sec)
self.interval_collector.reset()
# get stats from the run
stats = self.runner.client.get_stats()
current_traffic_config = self._get_traffic_config()
warning = self.compare_tx_rates(current_traffic_config['direction-total']['rate_pps'],
stats['total_tx_rate'])
if warning is not None:
stats['warning'] = warning
# save reliable stats from whole iteration
self.iteration_collector.add(stats, current_traffic_config['direction-total']['rate_pps'])
LOG.info('Average drop rate: %f', stats['overall']['drop_rate_percent'])
return stats, current_traffic_config['direction-total']
def get_controller_info(ssh_access, net, res_col, retry_count):
if not ssh_access:
return
LOG.info('Fetching OpenStack deployment details...')
sshcon = sshutils.SSH(ssh_access, connect_retry_count=retry_count)
if sshcon is None:
LOG.error('Cannot connect to the controller node')
return
res = {}
res['distro'] = sshcon.get_host_os_version()
res['openstack_version'] = sshcon.check_openstack_version()
res['cpu_info'] = sshcon.get_cpu_info()
if net:
l2type = res_col.get_result('l2agent_type')
encap = res_col.get_result('encapsulation')
if l2type:
if encap:
res['nic_name'] = sshcon.get_nic_name(l2type, encap,
net.internal_iface_dict)
res['l2agent_version'] = sshcon.get_l2agent_version(l2type)
# print results
CONLOG.info(res_col.ppr.pformat(res))
FILELOG.info(json.dumps(res, sort_keys=True))
res_col.add_properties(res)
def reset_table_sequence_id(entity_class, transaction):
mapper = class_mapper(entity_class)
table_name = mapper.local_table
if len(mapper.primary_key) > 1:
LOG.info(
'Cannot update primary key sequence for table \'%s\' as '
'it has a composite primary key. ',
table_name,
)
else:
id_column = class_mapper(entity_class).primary_key[0].name
reset_sequence_id_command = _RESET_SEQUENECE_COMMAND_FORMAT.format(
table_name=table_name, id_column=id_column)
# Disabling this warning as we don't want to expose the `execute`
# method in our transaction class.
# pylint:disable=W0212
transaction._session.execute(reset_sequence_id_command)
LOG.info(
'Successfully staged updated primary key sequence for '
'table \'%s\'',
table_name,
)
def receiveMsg_TCP(self, datapacket = False):
if self.socket_tcp is None:
raise _error.ConnectionError("no tcp socket for receiving")
raw = ""
note = ""
data, readcounter = self.receive_bytes(self.socket_tcp, const.HEADER.size)
if readcounter > 1:
note += "HEADER SEGMENTED INTO %s SEGMENTS!" % readcounter
raw += data
size, command = networking.parsePacketHeader(data)
if not command in (const.PACKET_SERVER_FRAME, const.PACKET_SERVER_SYNC):
if command in const.packet_names:
LOG.debug("received size: %d, command: %s (%d)"% (size, const.packet_names[command], command))
else:
LOG.debug("received size: %d, command: %d"% (size, command))
size -= const.HEADER.size # remove size of the header ...
data, readcounter = self.receive_bytes(self.socket_tcp, size)
if readcounter > 1:
note += "DATA SEGMENTED INTO %s SEGMENTS!" % readcounter
raw += data
if not self.running:
return None
if len(note) > 0:
LOG.info(note)
content = data
if datapacket:
return DataPacket(size, command, content)
else:
return size, command, content
#content = struct.unpack(str(size) + 's', data)
#content = content[0]
def fetch_dimension_metadata(self, dimensions, interval):
# NOTE(stephen): Right now we only care about the cardinality of each
# dimension.
LOG.info('Fetching dimension metadata for %s dimensions',
len(dimensions))
output = {}
for dimension in dimensions:
query = create_dimension_metadata_query(self._datasource.name,
dimension, [interval])
result = self._query_client.run_raw_query(query)
sketch_size = 0
if result:
# The sketch size is represented as powers of 2. The sketch size
# must be larger than the true column cardinality to ensure that
# we produce an exact distinct count (otherwise we will produce
# an approximate).
# Use the number of bits it takes to represent the approximate
# distinct count as a shorthand for computing the sketch size.
bits = int(result[0]['event'][dimension]).bit_length()
sketch_size = 2**bits
output[dimension] = sketch_size
LOG.debug('Sketch size %s for dimension %s', sketch_size,
dimension)
LOG.info('Finished fetching dimension metadata')
return output
def write_cache(app_name, app_json):
'''
just keep one line
:param cfile: cache file
:param app_name: application name
:param app_json: application info with pod status & update_time
:return:
'''
cfile = '/tmp/' + app_name
delete_old_cache(cfile, app_name)
try:
f = file(cfile, "a")
obj = {"app_name": app_name}
datalist = []
for i in app_json:
rcdata = get_rc_name_in_heat_resource2(i)
if rcdata is not None:
datalist.append(rcdata)
obj['data'] = datalist
f.write(json.dumps(obj))
f.write('\n')
#LOG.info(datalist)
except Exception, e:
LOG.info(e.message)
def create_server(self,
vmname,
image,
flavor,
key_name,
nic,
sec_group,
avail_zone=None,
user_data=None,
config_drive=None,
files=None):
"""Create a new server."""
if sec_group:
security_groups = [sec_group['id']]
else:
security_groups = None
# Also attach the created security group for the test
LOG.info('Creating instance %s with AZ: "%s"', vmname, avail_zone)
instance = self.novaclient.servers.create(
name=vmname,
image=image,
flavor=flavor,
key_name=key_name,
nics=nic,
availability_zone=avail_zone,
userdata=user_data,
config_drive=config_drive,
files=files,
security_groups=security_groups)
return instance
def start_rabbitmq_client():
"""Start Rabbitmq client to listen instance notifications from Nova"""
cfg = get_rabbit_config()
rabbit_url = "rabbit://%s:%s@%s:%s/%s" % (cfg['user_id'], cfg['password'],
cfg['host'], cfg['port'],
cfg['virt_host'])
LOG.info(rabbit_url)
target = oslo_messaging.Target(exchange="nova",
topic="notifications",
server="info",
version="2.1",
fanout=True)
transport = oslo_messaging.get_notification_transport(CONF, url=rabbit_url)
endpoints = [
InstCreateNotificationEp(),
InstResizeNotificationEp(),
InstDelNotificationEp()
]
server = oslo_messaging.get_notification_listener(transport, [target],
endpoints, "threading")
thread = threading.Thread(target=rpc_work, args=(server, ))
thread.start()
LOG.info("Rabbitmq Client Started!")
return server
def get_application(application_name, is_summary=False):
if not is_summary:
session = requests.session()
url = 'http://{}:8080/api/v1/namespaces/{}/pods'.format(
app.config['K8S_IP'], application_name)
reply = session.get(url)
pod_list_json = reply.json()['items']
url = 'http://{}:8080/api/v1/namespaces/{}/replicationcontrollers'.format(
app.config['K8S_IP'], application_name)
reply = session.get(url)
rc_list_json = reply.json()['items']
try:
stack = heat_client.get_stack(application_name)
#print stack
stack_json = stack.to_dict()
res = read_cache(application_name)
if res is None:
LOG.info(" cache is None ")
res = heat_client.get_resource_list(application_name)
res = write_cache(application_name, res)
if not is_summary:
''' change to PaasApplication2 '''
paas_app = PaasApplication2(application_name, stack_json, res,
rc_list_json, pod_list_json)
else:
paas_app = PaasApplication2(application_name, stack_json, None,
None, None)
return paas_app
except Exception, e:
LOG.warning(type(e))
LOG.warning(e)
return None
def _add_child_graph(self, parent, graphs):
'''
Add a graph which is a child of the given parent.
@type parent: Graph
@param parent:
The graph to spawn the child from.
@type graphs: list(Graph)
@param graphs:
The list to put the child into.
'''
LOG.info("Cloning from parent: %s", parent)
while True:
# Create a mutated child
child = parent.clone(self._next_graph_name())
self._mutate(child, self._mutation_factor)
# Remove any inner nodes which have no inputs. We keep doing this
# since removing one inner node might drop the depth of another.
changed = True
max_depth = child.num_layers - 1
while changed:
changed = False
for node in tuple(child.mid):
if node.depth == 0 or node.depth >= max_depth:
child.remove_node(node)
changed = True
if child.is_connected():
graphs.append(child)
LOG.info("Adding child graph: %s", child)
return
def remove_public_key(self, name):
keypair_list = self.novaclient.keypairs.list()
for key in keypair_list:
if key.name == name:
self.novaclient.keypairs.delete(name)
LOG.info('Removed public key %s', name)
break
def should_cut(self, post_number: int, color=None) -> None:
LOG.debug("should_cut", post_number, color)
# Did they specify a color (e.g. "last red" or "first white")?
if color is None:
# No all colors, so just skip None
posts_in_use = [
index for index, mapping in enumerate(self.mapping)
if mapping is not None
]
else:
# Specific color
posts_in_use = [
index for index, mapping in enumerate(self.mapping)
if COLOR_POSITIONS[mapping] == color
]
if post_number < 0:
# Count from end
self.right_post = posts_in_use[post_number]
else:
# Post number (Warning: post #1 means index 0!)
self.right_post = posts_in_use[post_number - 1]
LOG.info("right_post=", self.right_post)
for post in self.post_pins:
post.irq(self.on_wrong_post, trigger=Pin.IRQ_RISING)
self.post_pins[self.right_post].irq(self.on_right_post,
trigger=Pin.IRQ_RISING)
def wrap(*args):
time1 = time.time()
ret = fn(*args)
time2 = time.time()
LOG.info('*** %s function took %0.3f ms', fn.__name__,
(time2 - time1) * 1000.0)
return ret
def convert_current_users(transaction):
LOG.info('Migrating current users.')
for user in transaction.find_all_by_fields(User, {}):
user.status_id = (UserStatusEnum.ACTIVE.value
if user.active else UserStatusEnum.INACTIVE.value)
transaction.add_or_update(user)
LOG.info('Successfully migrated current users.')
def get_ndr_and_pdr(self):
"""Start the NDR/PDR iteration and return the results."""
dst = 'Bidirectional' if self.run_config['bidirectional'] else 'Unidirectional'
targets = {}
if self.config.ndr_run:
LOG.info('*** Searching NDR for %s (%s)...', self.run_config['l2frame_size'], dst)
targets['ndr'] = self.config.measurement.NDR
if self.config.pdr_run:
LOG.info('*** Searching PDR for %s (%s)...', self.run_config['l2frame_size'], dst)
targets['pdr'] = self.config.measurement.PDR
self.run_config['start_time'] = time.time()
self.interval_collector = IntervalCollector(self.run_config['start_time'])
self.interval_collector.attach_notifier(self.notifier)
self.iteration_collector = IterationCollector(self.run_config['start_time'])
results = {}
self.__range_search(0.0, 200.0, targets, results)
results['iteration_stats'] = {
'ndr_pdr': self.iteration_collector.get()
}
if self.config.ndr_run:
LOG.info('NDR load: %s', results['ndr']['rate_percent'])
results['ndr']['time_taken_sec'] = \
results['ndr']['timestamp_sec'] - self.run_config['start_time']
if self.config.pdr_run:
LOG.info('PDR load: %s', results['pdr']['rate_percent'])
results['pdr']['time_taken_sec'] = \
results['pdr']['timestamp_sec'] - results['ndr']['timestamp_sec']
else:
LOG.info('PDR load: %s', results['pdr']['rate_percent'])
results['pdr']['time_taken_sec'] = \
results['pdr']['timestamp_sec'] - self.run_config['start_time']
return results
def delete_server_by_name(self, vmname):
servers_list = self.get_server_list()
for server in servers_list:
if server.name == vmname:
LOG.info('Deleting server %s', server)
self.novaclient.servers.delete(server)
return True
return False
def delete_flavor(self, flavor):
try:
flavor.delete()
LOG.info('Flavor %s deleted.' % flavor.name)
return True
except novaclient.exceptions:
LOG.error('Failed deleting flavor %s.' % flavor.name)
return False
def sendToLog(self):
"""
Dispatcher to LOG
@rtype: boolean
@returns: True
"""
LOG.info("EVENT: %s" % (self.msg))
return True
def delete_floating_ip(self, floatingip):
LOG.info("Deleting floating ip " + floatingip)
for _ in range(1, 5):
try:
self.neutron_client.delete_floatingip(floatingip)
break
except IpAddressInUseClient:
time.sleep(1)
def delete_net(self, network):
if network:
name = network['name']
# it may take some time for ports to be cleared so we need to retry
for _ in range(1, 5):
try:
self.neutron_client.delete_network(network['id'])
LOG.info('Network %s deleted.', name)
break
except NetworkInUseClient:
time.sleep(1)
def save_to_db(self, cfg):
'''Save results to MongoDB database.'''
LOG.info("Saving results to MongoDB database...")
post_id = pns_mongo.\
pns_add_test_result_to_mongod(cfg.vmtp_mongod_ip,
cfg.vmtp_mongod_port,
cfg.vmtp_db,
cfg.vmtp_collection,
self.results)
if post_id is None:
LOG.error("Failed to add result to DB")
def get_tags(self):
LOG.info("Getting tags from openttd finger server")
LOG.debug("HTTP GET %s" % const.OPENTTD_FINGER_TAGS_URL)
self.request("GET", const.OPENTTD_FINGER_TAGS_URL)
r1 = self.getresponse()
LOG.debug("%d %s" % (r1.status, r1.reason))
if r1.status != 200:
raise Exception("Couldn't request tags list")
data1 = r1.read()
data2 = [i.strip().split() for i in data1.split('\n') if i]
data2 = [(int(i[0]), dateutil.parser.parse(i[1]).date(), i[2].strip()) for i in data2]
self.tags = data2
def create_net(self, network_name, subnet_name, cidr, dns_nameservers,
subnet_name_ipv6=None, cidr_ipv6=None, ipv6_mode=None,
enable_dhcp=True):
for network in self.networks:
if network['name'] == network_name:
LOG.info('Found existing internal network: %s', network_name)
return network
body = {
'network': {
'name': network_name,
'admin_state_up': True
}
}
network = self.neutron_client.create_network(body)['network']
body = {
'subnet': {
'name': subnet_name,
'cidr': cidr,
'network_id': network['id'],
'enable_dhcp': True,
'ip_version': 4,
'dns_nameservers': dns_nameservers
}
}
if not enable_dhcp:
body['subnet']['enable_dhcp'] = False
subnet = self.neutron_client.create_subnet(body)['subnet']
# add subnet id to the network dict since it has just been added
network['subnets'] = [subnet['id']]
# If ipv6 is enabled than create and add ipv6 network
if ipv6_mode:
body = {
'subnet': {
'name': subnet_name_ipv6,
'cidr': cidr_ipv6,
'network_id': network['id'],
'enable_dhcp': True,
'ip_version': 6,
'ipv6_ra_mode': ipv6_mode,
'ipv6_address_mode': ipv6_mode
}
}
if not enable_dhcp:
body['subnet']['enable_dhcp'] = False
subnet = self.neutron_client.create_subnet(body)['subnet']
# add the subnet id to the network dict
network['subnets'].append(subnet['id'])
LOG.info('Created internal network: %s.', network_name)
return network
def run(self):
L.info("Light Command Send : %s" % self.command)
args = self.command.split(" ")
subproc_args = { 'stdin' : subprocess.PIPE,
'stdout' : subprocess.PIPE,
'stderr' : subprocess.STDOUT,
}
try:
proc = subprocess.call(args, **subproc_args)
except OSError:
L.info("Failed to execute command: %s" % args[0])
sys.exit(1)
L.debug("Command Resturn Code: %d" % proc)
def dispose(self):
# Delete the internal networks only of we did not reuse an existing
# network
if not self.config.reuse_network_name:
self.__remove_router_interface()
for int_net in self.vm_int_net:
self.delete_net(int_net)
# delete the router only if its name matches the pns router name
if self.ext_router_created:
try:
if self.ext_router['name'] == self.ext_router_name:
self.neutron_client.remove_gateway_router(
self.ext_router['id'])
self.neutron_client.delete_router(self.ext_router['id'])
LOG.info('External router %s deleted.', self.ext_router['name'])
except TypeError:
LOG.info("No external router set")
def retry_if_http_error(exception):
"""
Defines which type of exceptions allow for a retry of the request
:param exception: the raised exception
:return: True if retrying the request is possible
"""
error = False
if isinstance(exception, requests.HTTPError):
if exception.response.status_code == 503:
LOG.info('Requesting retry: response code: ' + str(exception.response.status_code))
LOG.error('Exception: ' + exception.__repr__())
error = True
elif isinstance(exception, requests.ConnectionError):
LOG.info('Requesting retry: ConnectionError')
LOG.error('Exception: ' + exception.__repr__())
error = True
return error
def __exec_file(self, cmd, filename="tmp.txt", timeout=TIMEOUT):
filepath = os.path.join(define.APP_TMP, filename)
Li = []
proc = BantorraFileProcess(cmd, filepath)
proc.start()
proc.join(timeout)
if proc.is_alive():
proc.kill()
time.sleep(3)
L.debug("proc.terminate. %s" % proc.is_alive())
try:
with open(filepath, "r") as f:
while True:
line = f.readline()
if not line:
break
Li.append(line)
except:
L.info("No such file or directory")
result = "".join(Li); F.remove(filepath)
return result
def check_rpm_package_installed(self, rpm_pkg):
'''
Given a host and a package name, check if it is installed on the
system.
'''
check_pkg_cmd = "rpm -qa | grep " + rpm_pkg
(status, cmd_output, _) = self.execute(check_pkg_cmd)
if status:
return None
pkg_pattern = ".*" + rpm_pkg + ".*"
rpm_pattern = re.compile(pkg_pattern, re.IGNORECASE)
for line in cmd_output.splitlines():
mobj = rpm_pattern.match(line)
if mobj:
return mobj.group(0)
LOG.info("%s pkg installed ", rpm_pkg)
return None
def teardown(self):
'''
Clean up the floating ip and VMs
'''
LOG.info('Cleaning up...')
if self.server:
self.server.dispose()
if self.client:
self.client.dispose()
if not self.config.reuse_existing_vm and self.net:
self.net.dispose()
# Remove the public key
if self.comp:
self.comp.remove_public_key(self.config.public_key_name)
# Finally remove the security group
try:
if self.comp:
self.comp.security_group_delete(self.sec_group)
except ClientException:
# May throw novaclient.exceptions.BadRequest if in use
LOG.warning('Security group in use: not deleted')
if self.image_uploaded and self.config.delete_image_after_run:
self.comp.delete_image(self.glance_client, self.config.image_name)
def __init__(self, backend='neutron'):
odl_ip = CONF.shim_odl.odl_host
odl_port = CONF.shim_odl.odl_port
user = CONF.shim_odl.odl_user
passwd = CONF.shim_odl.odl_passwd
LOG.info("odl_host: %s" % odl_ip)
LOG.info("odl_port: %s" % odl_port)
LOG.info('odl_user: %s' % user)
LOG.info('odl_passwd: %s' % passwd)
if backend == 'neutron':
self.url = ("http://%(ip)s:%(port)s/controller/nb/v2/neutron" %
{'ip': odl_ip,
'port': odl_port})
if backend == 'restconf':
self.url = ("http://%(ip)s:%(port)s/restconf/config" %
{'ip': odl_ip,
'port': odl_port})
self.auth = (user, passwd)
self.timeout = 10
def sendToLog(self):
"""
The event dispatcher to LOG
@precondition: self.type is not None
@rtype: boolean
@returns: True if the preconditions are true
"""
if self.type is None:
return False
if self.type == "public":
LOG.info("Ingame public chat response: %s" % (self.msg))
elif self.type == "team":
LOG.info("Ingame team chat response: <To: %s> %s" % (self.playercompany, self.msg))
elif self.type == "private":
LOG.info("Ingame private chat response: <To: %s> %s" % (self.playername, self.msg))
return True
def sendToLog(self):
"""
Dispatcher to log
@precondition: self.type is not None
@rtype: boolean
@returns: True if the precondition is True
"""
if not self.type is None:
if self.type == "public":
LOG.info("Ingame chat: <%s> %s" % (self.playername,self.msg))
elif self.type == "team":
LOG.info("Ingame company chat: [%d]<%s> %s" % (self.playercompany, self.playername,self.msg))
elif self.type == "private":
LOG.info("Ingame private chat: <%s> %s" % (self.playername,self.msg))
return True
return False
def sendToLog(self):
if self.private:
prefix = "IRC private chat response to %s: " % self.playername
else:
prefix = "IRC chat response: "
LOG.info(prefix + self.__str__())
def sendToLog(self):
if self.private:
prefix = "IRC private chat: "
else:
prefix = "IRC chat: "
LOG.info(prefix + self.__str__())
def print_desc(desc):
global flow_num
flow_num = flow_num + 1
CONLOG.info("=" * 60)
LOG.info('Flow %d: %s', flow_num, desc)
def run_vmtp(opts):
'''Run VMTP
:param opts: Parameters that to be passed to VMTP in type argparse.Namespace(). See:
http://vmtp.readthedocs.org/en/latest/usage.html#running-vmtp-as-a-library
for examples of the usage on this API.
:return: A dictionary which contains the results in details.
'''
if (sys.argv == ['']):
# Running from a Python call
def_opts = parse_opts_from_cli()
for key, value in vars(def_opts).iteritems():
if key not in opts:
opts.__setattr__(key, value)
config = merge_opts_to_configs(opts)
rescol = ResultsCollector()
# Run the native host tests if specified by user
if opts.hosts:
# A list of 0 to 2 HostSshAccess elements
# remove any duplicate
opts.hosts = list(set(opts.hosts))
native_hosts = []
if_name = None
for host in opts.hosts:
# decode and extract the trailing if name first
# there is an if name if there are at least 2 ':' in the argument
# e.g. "[email protected]:secret:eth0"
if host.count(':') >= 2:
last_column_index = host.rfind(':')
# can be empty
last_arg = host[last_column_index + 1:]
if not if_name and last_arg:
if_name = last_arg
host = host[:last_column_index]
native_hosts.append(get_ssh_access('host', host, config))
native_tp_results = test_native_tp(native_hosts, if_name, config)
else:
native_tp_results = []
for item in native_tp_results:
rescol.add_flow_result(item)
CONLOG.info(rescol.ppr.pformat(item))
FILELOG.info(json.dumps(item, sort_keys=True))
# Parse the credentials of the OpenStack cloud, and run the benchmarking
cred = credentials.Credentials(opts.rc, opts.passwd, opts.no_env)
if cred.rc_auth_url:
if config.debug:
LOG.info('Using ' + cred.rc_auth_url)
vmtp_instance = VmtpTest(config, cred, rescol)
vmtp_instance.run()
vmtp_net = vmtp_instance.net
# Retrieve controller information if requested
# controller node ssh access to collect metadata for the run.
ctrl_host_access = get_ssh_access('controller-node', opts.controller_node, config)
get_controller_info(ctrl_host_access,
vmtp_net,
rescol,
config.ssh_retry_count)
# Print the report
print_report(rescol.results)
# Post-processing of the results, adding some metadata
if cred.rc_auth_url:
rescol.add_property('auth_url', cred.rc_auth_url)
rescol.mask_credentials()
rescol.generate_runid()
if opts.test_description:
rescol.add_property('test_description', opts.test_description)
# Save results to a JSON file
if config.json_file:
rescol.save(config)
# Save results to MongoDB
if config.vmtp_mongod_ip:
rescol.save_to_db(config)
return rescol.results
def __init__(self, neutron_client, config):
self.neutron_client = neutron_client
self.networks = neutron_client.list_networks()['networks']
self.ext_net = None
self.ext_router = None
self.ext_router_created = False
self.config = config
# mgmt/data network:
# - first for same network
# - second for network to network communication
self.vm_int_net = []
self.ext_router_name = None
# Store state if the network is ipv4/ipv6 dual stack
self.ipv6_enabled = False
# If reusing existing management network just find this network
if self.config.reuse_network_name:
try:
# An existing management network must be reused
int_net = self.lookup_network(self.config.reuse_network_name)
self.vm_int_net.append(int_net)
except IndexError:
raise vmtp.VmtpException("Unable to find the network to be reused.")
return
else:
##############################################
# If a user provided ext_net_name is not available,
# then find the first network that is external
##############################################
for network in self.networks:
if network['router:external']:
try:
if network['name'] == config.ext_net_name:
self.ext_net = network
break
if not self.ext_net:
self.ext_net = network
except AttributeError:
###############################################
# A attribute error indicates, no user defined
# external network defined, so use the first one
###############################################
self.ext_net = network
break
if self.ext_net:
LOG.info("Using external network: %s.", self.ext_net['name'])
# Find or create the router to the external network
ext_net_id = self.ext_net['id']
routers = neutron_client.list_routers()['routers']
for router in routers:
external_gw_info = router['external_gateway_info']
if external_gw_info:
if external_gw_info['network_id'] == ext_net_id:
self.ext_router = router
LOG.info('Found external router: %s', self.ext_router['name'])
break
# create a new external router if none found and a name was given
self.ext_router_name = config.router_name
if (not self.ext_router) and self.ext_router_name:
self.ext_router = self.create_router(self.ext_router_name,
self.ext_net['id'])
LOG.info('Created ext router %s.', self.ext_router_name)
self.ext_router_created = True
else:
LOG.warning("No external network found.")
if config.ipv6_mode:
self.ipv6_enabled = True
# Create the networks and subnets depending on v4 or v6
enable_dhcp = not config.no_dhcp
if config.ipv6_mode:
for (net, subnet, cidr, subnet_v6, cidr_v6) in zip(config.internal_network_name,
config.internal_subnet_name,
config.internal_cidr,
config.internal_subnet_name_v6,
config.internal_cidr_v6):
int_net = self.create_net(net, subnet, cidr,
config.dns_nameservers,
subnet_v6, cidr_v6, config.ipv6_mode,
enable_dhcp=enable_dhcp)
self.vm_int_net.append(int_net)
if config.same_network_only:
break
else:
for (net, subnet, cidr) in zip(config.internal_network_name,
config.internal_subnet_name,
config.internal_cidr):
int_net = self.create_net(net, subnet, cidr,
config.dns_nameservers,
enable_dhcp=enable_dhcp)
self.vm_int_net.append(int_net)
if config.same_network_only:
break
# Add both internal networks to router interface to enable
# network to network connectivity
if self.ext_net:
self.__add_router_interface()
self.l2agent_type = self._get_l2agent_type()
self.internal_iface_dict = self._get_internal_iface_dict()
def display(self, fmt, *args):
LOG.info(('[%s] ' + fmt) % ((self.name,) + args))
def setup(self):
# This is a template host access that will be used for all instances
# (the only specific field specific to each instance is the host IP)
# For test VM access, we never use password and always need a key pair
self.instance_access = sshutils.SSHAccess()
self.instance_access.username = self.config.ssh_vm_username
# if the configuration does not have a
# key pair specified, we check if the user has a personal key pair
# if no key pair is configured or usable, a temporary key pair will be created
if self.config.public_key_file and self.config.private_key_file:
self.instance_access.public_key_file = self.config.public_key_file
self.instance_access.private_key_file = self.config.private_key_file
else:
pub_key = os.path.expanduser('~/.ssh/id_rsa.pub')
priv_key = os.path.expanduser('~/.ssh/id_rsa')
if os.path.isfile(pub_key) and os.path.isfile(priv_key):
self.instance_access.public_key_file = pub_key
self.instance_access.private_key_file = priv_key
else:
LOG.error('Default keypair ~/.ssh/id_rsa[.pub] does not exist. Please '
'either create one in your home directory, or specify your '
'keypair information in the config file before running VMTP.')
sys.exit(1)
if self.config.debug and self.instance_access.public_key_file:
LOG.info('VM public key: ' + self.instance_access.public_key_file)
LOG.info('VM private key: ' + self.instance_access.private_key_file)
# If we need to reuse existing vms just return without setup
if not self.config.reuse_existing_vm:
creds = self.cred.get_credentials()
if self.cred.rc_identity_api_version == 3:
auth = keystone_v3.Password(**creds)
else:
auth = keystone_v2.Password(**creds)
sess = session.Session(auth=auth, verify=self.cred.rc_cacert)
# Create the nova and neutron instances
nova_client = novaclient.Client('2', session=sess)
neutron = neutronclient.Client('2.0', session=sess)
self.comp = compute.Compute(nova_client, self.config)
# Add the appropriate public key to openstack
self.comp.init_key_pair(self.config.public_key_name, self.instance_access)
self.image_instance = self.comp.find_image(self.config.image_name)
if self.image_instance is None:
if self.config.vm_image_url != "":
LOG.info('%s: image for VM not found, trying to upload it ...',
self.config.image_name)
keystoneclient.Client(self.cred.rc_identity_api_version,
session=sess, auth_url=creds['auth_url'])
self.glance_client = glanceclient.Client('1', session=sess)
self.comp.upload_image_via_url(
self.glance_client,
self.config.image_name,
self.config.vm_image_url)
self.image_instance = self.comp.find_image(self.config.image_name)
self.image_uploaded = True
else:
# Exit the pogram
LOG.error('%s: image to launch VM not found. ABORTING.',
self.config.image_name)
sys.exit(1)
self.assert_true(self.image_instance)
LOG.info('Found image %s to launch VM, will continue', self.config.image_name)
self.flavor_type = self.comp.find_flavor(self.config.flavor_type)
self.net = network.Network(neutron, self.config)
self.rescol.add_property('l2agent_type', self.net.l2agent_type)
LOG.info("OpenStack agent: " + self.net.l2agent_type)
try:
network_type = self.net.vm_int_net[0]['provider:network_type']
LOG.info("OpenStack network type: " + network_type)
self.rescol.add_property('encapsulation', network_type)
except KeyError as exp:
network_type = 'Unknown'
LOG.info("Provider network type not found: ", str(exp))
# Create a new security group for the test
self.sec_group = self.comp.security_group_create()
if not self.sec_group:
raise VmtpException("Security group creation failed")
if self.config.reuse_existing_vm:
self.server.internal_ip = self.config.vm_server_internal_ip
self.client.internal_ip = self.config.vm_client_internal_ip
if self.config.vm_server_external_ip:
self.server.ssh_access.host = self.config.vm_server_external_ip
else:
self.server.ssh_access.host = self.config.vm_server_internal_ip
if self.config.vm_client_external_ip:
self.client.ssh_access.host = self.config.vm_client_external_ip
else:
self.client.ssh_access.host = self.config.vm_client_internal_ip
return
# this is the standard way of running the test
# NICs to be used for the VM
if self.config.reuse_network_name:
# VM needs to connect to existing management and new data network
# Reset the management network name
int_net_name = list(self.config.internal_network_name)
int_net_name[0] = self.config.reuse_network_name
self.config.internal_network_name = int_net_name
else:
# Make sure we have an external network and an external router
# self.assert_true(self.net.ext_net)
# self.assert_true(self.net.ext_router)
self.assert_true(self.net.vm_int_net)
# Get hosts for the availability zone to use
# avail_list = self.comp.list_hypervisor(config.availability_zone)
avail_list = self.comp.get_az_host_list()
if not avail_list:
sys.exit(5)
# compute the list of client vm placements to run
# the first host is always where the server runs
server_az = avail_list[0]
if len(avail_list) > 1:
# 2 hosts are known
if self.config.inter_node_only:
# in this case we do not want the client to run on the same host
# as the server
avail_list.pop(0)
self.client_az_list = avail_list
self.server = PerfInstance(self.config.vm_name_server,
self.config,
self.comp,
self.net,
server=True)
self.server.display('Creating server VM...')
self.create_instance(self.server, server_az,
self.net.vm_int_net[0])
def run(self):
# Main loop
print 'worker',self.no,'starting...'
SIGN = "MZLIUPEIZHARUIMM:"
freelist = self.m.handles[:]
num_processed = 0
num_tasks = 0
while self.queue.on() or num_tasks > num_processed:
#while self.queue.size()>0 or num_tasks > num_processed:
# If there is an url to process and a free curl object, add to multi stack
while self.queue.size()>0 and freelist:
c=freelist.pop()
if c.usedtimes > 30:
c.close()
c=self.init_conn()
task = self.queue.pop(c.pxy)
if task==None:
freelist.append(c)
# time.sleep(1)
break
task.clear()
task.firstline=SIGN+task.url
c.setopt(pycurl.URL, task.url)
c.setopt(pycurl.WRITEFUNCTION,task.do)
self.m.add_handle(c)
# store some info
c.task = task
c.usedtimes += 1
c.start_time=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
num_tasks += 1
# Run the internal curl state machine for the multi stack
while 1:
ret, num_handles = self.m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
# Check for curl objects which have terminated, and add them to the freelist
with self.lock:
nowtime=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
while 1:
num_q, ok_list, err_list = self.m.info_read()
for c in ok_list:
hcode = c.getinfo(pycurl.HTTP_CODE)
if hcode >= 200 and hcode < 400:
LOG.info("Success: "+c.start_time+"\t"+nowtime+"\t"+str(c.task.id)+"\t"+"httpcode="+str(c.getinfo(pycurl.HTTP_CODE))+"\t"+c.task.url)
htmllen=len(c.task.html)+1
c.task.firstline +="\t"+c.getinfo(pycurl.EFFECTIVE_URL)+"\t"+str(htmllen)
self.queue.saveme(c.task.firstline+"\n"+c.task.html+"\n")
self.saveto_redis(self.r,c.task.url,c.task.html,nowtime)
else:
LOG.info("BadResponse: "+c.start_time+"\t"+nowtime+"\t"+str(c.task.id)+"\t"+"httpcode="+str(c.getinfo(pycurl.HTTP_CODE))+"\t"+c.task.url)
self.m.remove_handle(c)
freelist.append(c)
for c, errno, errmsg in err_list:
c.task.retry += 1
if c.task.can_try():
self.queue.push(c.task)
if c.task.can_try():
LOG.info("Retry: "+c.start_time+"\t"+nowtime+"\t"+str(c.task.id)+"\t"+str(c.task.retry)+"\t"+c.task.url+"\t"+str(errno)+"\t"+errmsg)
else:
LOG.info("Failed: "+c.start_time+"\t"+nowtime+"\t"+str(c.task.id)+"\t"+str(c.task.retry)+"\t"+c.task.url+"\t"+str(errno)+"\t"+errmsg)
self.m.remove_handle(c)
c.close()
c=self.init_conn()
freelist.append(c)
num_processed = num_processed + len(ok_list) + len(err_list)
if num_q == 0:
break
# Currently no more I/O is pending, could do something in the meantime
# (display a progress bar, etc.).
# We just call select() to sleep until some more data is available.
self.m.select(1.0)
# Cleanup
for c in self.m.handles:
c.close()
self.m.close()
def merge_opts_to_configs(opts):
default_cfg_file = resource_string(__name__, "cfg.default.yaml")
# read the default configuration file and possibly an override config file
# the precedence order is as follows:
# $HOME/.vmtp.yaml if exists
# -c <file> from command line if provided
# cfg.default.yaml
config = config_loads(default_cfg_file)
local_cfg = os.path.expanduser('~/.vmtp.yaml')
if os.path.isfile(local_cfg):
config = config_load(local_cfg, config)
if opts.config:
config = config_load(opts.config, config)
if opts.show_config:
print default_cfg_file
sys.exit(0)
if opts.version:
print(__version__)
sys.exit(0)
config.debug = opts.debug
config.stop_on_error = opts.stop_on_error
config.keep_first_flow_and_exit = opts.keep_first_flow_and_exit
config.inter_node_only = opts.inter_node_only
config.same_network_only = opts.same_network_only
if config.public_key_file and not os.path.isfile(config.public_key_file):
LOG.warning('Invalid public_key_file:' + config.public_key_file)
config.public_key_file = None
if config.private_key_file and not os.path.isfile(config.private_key_file):
LOG.warning('Invalid private_key_file:' + config.private_key_file)
config.private_key_file = None
# direct: use SR-IOV ports for all the test VMs
if opts.vnic_type not in [None, 'direct', 'macvtap', 'normal']:
LOG.error('Invalid vnic-type: ' + opts.vnic_type)
sys.exit(1)
config.vnic_type = opts.vnic_type
config.hypervisors = opts.hypervisors
if opts.availability_zone:
config.availability_zone = opts.availability_zone
# time to run each perf test in seconds
if opts.time:
config.time = int(opts.time)
else:
config.time = 10
if opts.json:
config.json_file = opts.json
else:
config.json_file = None
# Initialize the external host access
config.ext_host = get_ssh_access('external-host', opts.ext_host, config)
###################################################
# VM Image URL
###################################################
if opts.vm_image_url:
config.vm_image_url = opts.vm_image_url
###################################################
# MongoDB Server connection info.
###################################################
if opts.mongod_server:
config.vmtp_mongod_ip = opts.mongod_server
else:
config.vmtp_mongod_ip = None
if 'vmtp_mongod_port' not in config:
# Set MongoDB default port if not set.
config.vmtp_mongod_port = 27017
# the bandwidth limit for VMs
if opts.vm_bandwidth:
opts.vm_bandwidth = opts.vm_bandwidth.upper().strip()
ex_unit = 'KMG'.find(opts.vm_bandwidth[-1])
try:
if ex_unit == -1:
raise ValueError
val = int(opts.vm_bandwidth[0:-1])
except ValueError:
LOG.error('Invalid --bandwidth parameter. A valid input must '
'specify only one unit (K|M|G).')
sys.exit(1)
config.vm_bandwidth = int(val * (10 ** (ex_unit * 3)))
# the pkt size for TCP, UDP and ICMP
if opts.tcp_pkt_sizes:
config.tcp_pkt_sizes = decode_size_list('--tcpbuf', opts.tcp_pkt_sizes)
if opts.udp_pkt_sizes:
config.udp_pkt_sizes = decode_size_list('--udpbuf', opts.udp_pkt_sizes)
if opts.icmp_pkt_sizes:
config.icmp_pkt_sizes = decode_size_list('--icmp_pkt_sizes', opts.icmp_pkt_sizes)
if opts.reuse_network_name:
config.reuse_network_name = opts.reuse_network_name
if opts.os_dataplane_network:
config.os_dataplane_network = opts.os_dataplane_network
config.config_drive = opts.config_drive
config.no_floatingip = opts.no_floatingip
config.no_dhcp = opts.no_dhcp
config.delete_image_after_run = opts.delete_image_after_run
#####################################################
# Set Ganglia server ip and port if the monitoring (-m)
# option is enabled.
#####################################################
config.gmond_svr_ip = None
config.gmond_svr_port = None
if opts.monitor:
# Add the default gmond port if not present
if ':' not in opts.monitor:
opts.monitor += ':8649'
mobj = re.match(r'(\d+\.\d+\.\d+\.\d+):(\d+)', opts.monitor)
if mobj:
config.gmond_svr_ip = mobj.group(1)
config.gmond_svr_port = mobj.group(2)
LOG.info("Ganglia monitoring enabled (%s:%s)",
config.gmond_svr_ip, config.gmond_svr_port)
config.time = 30
else:
LOG.warning('Invalid --monitor syntax: ' + opts.monitor)
###################################################
# Once we parse the config files, normalize
# the paths so that all paths are absolute paths.
###################################################
normalize_paths(config)
# Check the tp-tool name
config.protocols = opts.protocols.upper()
if 'M' in config.protocols or opts.multicast_addr:
# nuttcp required for multicast
opts.tp_tool = 'nuttcp'
config.tp_tool = nuttcp_tool.NuttcpTool
# If M provided, but not multicast_addr, use default (231.1.1.1)
config.multicast_addr = opts.multicast_addr if opts.multicast_addr else "231.1.1.1"
# If --multicast_addr provided, ensure 'M' is in protocols.
if 'M' not in config.protocols:
config.protocols += 'M'
elif 'T' in config.protocols or 'U' in config.protocols:
if opts.tp_tool.lower() == 'nuttcp':
config.tp_tool = nuttcp_tool.NuttcpTool
elif opts.tp_tool.lower() == 'iperf':
config.tp_tool = iperf_tool.IperfTool
else:
LOG.warning('Invalid transport tool: ' + opts.tp_tool)
sys.exit(1)
else:
config.tp_tool = None
return config
