from django.db import transaction
from django.shortcuts import render
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_exempt
from django import template
# Our App imports
from common.views import NeverCacheRedirectView
from tracker import models
config_path = os.path.realpath(os.path.dirname(__file__)+"/../..")
if config_path not in sys.path:
sys.path.append(config_path)
import config as common_config
CONFIG = common_config.config_load()
# IP Address which are considered "In Room"
LOCALIPS = CONFIG['config']['localips']
class funnydict(dict):
def __getattr__(self, key):
return self[key]
@never_cache
def stream(request, group):
"""Gives an end user a streaming server for a group.
Picks the least loaded by bitrate.
from django.db import transaction
from django.db import models as django_models
from django.shortcuts import render
from django.views.decorators.cache import never_cache
from django.views.decorators.csrf import csrf_exempt
from django import template
# Our App imports
from common.views.simple import NeverCacheRedirectView
from tracker import models
config_path = os.path.realpath(os.path.dirname(__file__) + "/../..")
if config_path not in sys.path:
sys.path.append(config_path)
import config as common_config
CONFIG = common_config.config_load()
# IP Address which are considered "In Room"
LOCALIPS = CONFIG['config']['localips']
class funnydict(dict):
def __getattr__(self, key):
return self[key]
@never_cache
def stream(request, group):
"""Gives an end user a streaming server for a group.
Picks the least loaded by bitrate.
def main():
global fluent_logger
run_summary_required = False
try:
log.setup()
# load default config file
config, default_cfg = load_default_config()
# create factory for platform specific classes
try:
factory_module = importlib.import_module(config['factory_module'])
factory = getattr(factory_module, config['factory_class'])()
except AttributeError:
raise Exception(
"Requested factory module '{m}' or class '{c}' was not found.".
format(m=config['factory_module'], c=config['factory_class']))
# create config plugin for this platform
config_plugin = factory.get_config_plugin_class()(config)
config = config_plugin.get_config()
opts, unknown_opts = _parse_opts_from_cli()
log.set_level(debug=opts.debug)
if opts.version:
print pbr.version.VersionInfo('nfvbench').version_string_with_vcs()
sys.exit(0)
if opts.summary:
with open(opts.summary) as json_data:
result = json.load(json_data)
if opts.user_label:
result['config']['user_label'] = opts.user_label
print NFVBenchSummarizer(result, fluent_logger)
sys.exit(0)
# show default config in text/yaml format
if opts.show_default_config:
print default_cfg
sys.exit(0)
config.name = ''
if opts.config:
# do not check extra_specs in flavor as it can contain any key/value pairs
whitelist_keys = ['extra_specs']
# override default config options with start config at path parsed from CLI
# check if it is an inline yaml/json config or a file name
if os.path.isfile(opts.config):
LOG.info('Loading configuration file: %s', opts.config)
config = config_load(opts.config, config, whitelist_keys)
config.name = os.path.basename(opts.config)
else:
LOG.info('Loading configuration string: %s', opts.config)
config = config_loads(opts.config, config, whitelist_keys)
# setup the fluent logger as soon as possible right after the config plugin is called,
# if there is any logging or result tag is set then initialize the fluent logger
for fluentd in config.fluentd:
if fluentd.logging_tag or fluentd.result_tag:
fluent_logger = FluentLogHandler(config.fluentd)
LOG.addHandler(fluent_logger)
break
# traffic profile override options
override_custom_traffic(config, opts.frame_sizes, opts.unidir)
# copy over cli options that are used in config
config.generator_profile = opts.generator_profile
if opts.sriov:
config.sriov = True
if opts.log_file:
config.log_file = opts.log_file
if opts.service_chain:
config.service_chain = opts.service_chain
if opts.service_chain_count:
config.service_chain_count = opts.service_chain_count
if opts.no_vswitch_access:
config.no_vswitch_access = opts.no_vswitch_access
if opts.hypervisor:
# can be any of 'comp1', 'nova:', 'nova:comp1'
config.compute_nodes = opts.hypervisor
if opts.vxlan:
config.vxlan = True
if opts.restart:
config.restart = True
# port to port loopback (direct or through switch)
if opts.l2_loopback:
config.l2_loopback = True
if config.service_chain != ChainType.EXT:
LOG.info('Changing service chain type to EXT')
config.service_chain = ChainType.EXT
if not config.no_arp:
LOG.info('Disabling ARP')
config.no_arp = True
config.vlans = [int(opts.l2_loopback), int(opts.l2_loopback)]
LOG.info('Running L2 loopback: using EXT chain/no ARP')
if opts.use_sriov_middle_net:
if (not config.sriov) or (config.service_chain != ChainType.PVVP):
raise Exception(
"--use-sriov-middle-net is only valid for PVVP with SRIOV")
config.use_sriov_middle_net = True
if config.sriov and config.service_chain != ChainType.EXT:
# if sriov is requested (does not apply to ext chains)
# make sure the physnet names are specified
check_physnet("left", config.internal_networks.left)
check_physnet("right", config.internal_networks.right)
if config.service_chain == ChainType.PVVP and config.use_sriov_middle_net:
check_physnet("middle", config.internal_networks.middle)
# show running config in json format
if opts.show_config:
print json.dumps(config, sort_keys=True, indent=4)
sys.exit(0)
# update the config in the config plugin as it might have changed
# in a copy of the dict (config plugin still holds the original dict)
config_plugin.set_config(config)
if opts.status or opts.cleanup or opts.force_cleanup:
status_cleanup(config, opts.cleanup, opts.force_cleanup)
# add file log if requested
if config.log_file:
log.add_file_logger(config.log_file)
openstack_spec = config_plugin.get_openstack_spec() if config.openrc_file \
else None
nfvbench_instance = NFVBench(config, openstack_spec, config_plugin,
factory)
if opts.server:
server = WebServer(nfvbench_instance, fluent_logger)
try:
port = int(opts.port)
except ValueError:
server.run(host=opts.host)
else:
server.run(host=opts.host, port=port)
# server.run() should never return
else:
with utils.RunLock():
run_summary_required = True
if unknown_opts:
err_msg = 'Unknown options: ' + ' '.join(unknown_opts)
LOG.error(err_msg)
raise Exception(err_msg)
# remove unfilled values
opts = {
k: v
for k, v in vars(opts).iteritems() if v is not None
}
# get CLI args
params = ' '.join(str(e) for e in sys.argv[1:])
result = nfvbench_instance.run(opts, params)
if 'error_message' in result:
raise Exception(result['error_message'])
if 'result' in result and result['status']:
nfvbench_instance.save(result['result'])
nfvbench_instance.prepare_summary(result['result'])
except Exception as exc:
run_summary_required = True
LOG.error({
'status': NFVBench.STATUS_ERROR,
'error_message': traceback.format_exc()
})
print str(exc)
finally:
if fluent_logger:
# only send a summary record if there was an actual nfvbench run or
# if an error/exception was logged.
fluent_logger.send_run_summary(run_summary_required)
def run(self):
service_list = []
try:
log.info("""
----------------------------
--- Picammory Starting ---
----------------------------
""")
os.chdir(os.path.expanduser('~/picammory/picammory/'))
log.info("os.getcwd() = '%s'", os.getcwd())
#---
# for gpio_pin in (5, 23, 24, 25):
# subprocess.call(['/usr/local/bin/gpio', 'export', str(gpio_pin), 'out'])
# wiringpi2.wiringPiSetupSys()
#---
config.config_load('picammory.ini')
#---
# LEDProcessor.start()
# service_list.append(LEDProcessor)
#
# LEDProcessor.blue(True)
#---
MailService.start()
service_list.append(MailService)
MailService.send_message('Picammory', 'Picammory Server Starting')
# LEDProcessor.green(True)
#---
FtpUploader.start()
service_list.append(FtpUploader)
# LEDProcessor.red(True)
#---
log.info("Start Cam Processor")
self.cam_processor = CamProcessor(os.path.expanduser('~/picammory_storage/'))
service_list.append(self.cam_processor)
# LEDProcessor.blue(False)
#---
# LEDProcessor.green(False)
# LEDProcessor.red(False)
#--- Wait until one thread stop...
stopped_service = None
while(stopped_service is None):
for service in service_list:
if not service.is_alive():
log.info("Service %s is dead", service)
stopped_service = service
break
time.sleep(1)
except EnvironmentError as e:
log.exception("Picammory failed 1")
MailService.send_message("Picammory Error", "EnvironmentError: SprinklermoryServer failed: %d (%s)" % (e.errno, e.strerror))
except Exception as e:
log.exception("Picammory failed 2")
MailService.send_message("Picammory Error", "Exception: Scheduler failed: %s" % (e))
finally:
for service in service_list:
service.terminate()
log.info(
'''
------------------------
--- Picammory Exit ---
------------------------
''')
sys.exit(1)
import signal
import subprocess
import sys
import time
import traceback
mydir = os.path.dirname(__file__)
config_path = os.path.realpath(mydir+"../..")
if config_path not in sys.path:
sys.path.append(config_path)
import config
from gi.repository import Gst
Gst.init(None)
loaded_config = config.config_load()
channels = loaded_config.groups()
def generate_thumb(dst, url):
print "%s snapshot starting" % dst
pipeline = "uridecodebin uri=\"%s\" \
! videoconvert \
! videoscale \
! video/x-raw,width=300,height=168 \
! pngenc snapshot=true \
! filesink location=%s" % (url, dst)
assert not os.path.exists(dst)
p = subprocess.Popen("gst-launch-1.0 "+pipeline, shell=True, preexec_fn=os.setsid)
start = time.time()
import os
import sys
import errno
import requests
import shutil
import subprocess
config_path = os.path.realpath(os.path.dirname(__file__)+"../..")
if config_path not in sys.path:
sys.path.append(config_path)
import config
from gi.repository import Gst
Gst.init(None)
loaded_config = config.config_load()
channels = loaded_config.groups()
def generate_thumb(channel, url):
pipeline = "uridecodebin uri=%s \
! videoconvert \
! videoscale \
! video/x-raw,width=300,height=217 \
! pngenc snapshot=true \
! filesink location=/srv/%s/snapshot.png" % (url, channel)
pipeline = Gst.parse_launch(pipeline)
bus = pipeline.get_bus()
pipeline.set_state(Gst.State.PLAYING)
bus.poll(Gst.MessageType.EOS, Gst.CLOCK_TIME_NONE)
pipeline.set_state(Gst.State.NULL)
import math
import copy
import shutil
import logging
import scipy.sparse
from tqdm import tqdm
from collections import namedtuple
from easydict import EasyDict as edict
from config import CONFIG, config_load
# Load CONFIG
parser = argparse.ArgumentParser(description='Training code')
parser.add_argument('--config', default='config.yaml', type=str, help='yaml config file')
args = parser.parse_args()
config_load(args.config)
# config_load('config.yaml')
print ('==> CONFIG is: \n', CONFIG, '\n')
# Set logger
logger = logging.getLogger(__name__)
format = logging.Formatter("%(asctime)s - %(message)s") # output format
sh = logging.StreamHandler(stream=sys.stdout) # output to standard output
sh.setFormatter(format)
logger.addHandler(sh)
if CONFIG.DEBUG:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# Create LOG_DIR and SNAPSHOT_DIR
paper_renderer = paperrenderer.PaperRenderer (map_layout)
chart_renderer = chartrenderer.ChartRenderer (geometry)
geometry.compute_extents_of_downloaded_tiles ()
(lat1, lon1) = geometry.transform_page_mm_to_lat_lon (map_layout.map_to_left_margin_mm, map_layout.map_to_top_margin_mm + map_layout.map_height_mm)
(lat2, lon2) = geometry.transform_page_mm_to_lat_lon (map_layout.map_to_left_margin_mm + map_layout.map_width_mm, map_layout.map_to_top_margin_mm)
print ("Map bounds: {0} {1} {2} {3}".format (lat1, lon1, lat2, lon2))
paper_renderer.render (args.format, args.output, chart_renderer)
if __name__ == "__main__":
try:
config_data = config.config_load ()
except IOError as e:
print ("La Mapería is not configured yet.")
answ = input ("Would you like to configure La Mapería right now? [Y/n] ").lower ()
if answ.startswith ('y') or not answ:
config_data = config_wizard ()
else:
print ("I'm not smart enough to work without a configuration. Exiting...")
exit (1)
except ValueError as e:
print ("The configuration in {} is not valid: {}".format (config.config_get_configuration_filename (),
e.args[0]))
exit (1)
main (config_data)
def merge_opts_to_configs(opts):
default_cfg_file = resource_string(__name__, "cfg.default.yaml")
# read the default configuration file and possibly an override config file
# the precedence order is as follows:
# $HOME/.vmtp.yaml if exists
# -c <file> from command line if provided
# cfg.default.yaml
config = config_loads(default_cfg_file)
local_cfg = os.path.expanduser('~/.vmtp.yaml')
if os.path.isfile(local_cfg):
config = config_load(local_cfg, config)
if opts.config:
config = config_load(opts.config, config)
if opts.show_config:
print default_cfg_file
sys.exit(0)
if opts.version:
print(__version__)
sys.exit(0)
config.debug = opts.debug
config.stop_on_error = opts.stop_on_error
config.inter_node_only = opts.inter_node_only
config.same_network_only = opts.same_network_only
if config.public_key_file and not os.path.isfile(config.public_key_file):
print('Warning: invalid public_key_file:' + config.public_key_file)
config.public_key_file = None
if config.private_key_file and not os.path.isfile(config.private_key_file):
print('Warning: invalid private_key_file:' + config.private_key_file)
config.private_key_file = None
# direct: use SR-IOV ports for all the test VMs
if opts.vnic_type not in [None, 'direct', 'macvtap', 'normal']:
print('Invalid vnic-type: ' + opts.vnic_type)
sys.exit(1)
config.vnic_type = opts.vnic_type
config.hypervisors = opts.hypervisors
if opts.availability_zone:
config.availability_zone = opts.availability_zone
# time to run each perf test in seconds
if opts.time:
config.time = int(opts.time)
else:
config.time = 10
if opts.json:
config.json_file = opts.json
else:
config.json_file = None
# Initialize the external host access
config.ext_host = get_ssh_access('external-host', opts.ext_host, config)
###################################################
# VM Image URL
###################################################
if opts.vm_image_url:
config.vm_image_url = opts.vm_image_url
###################################################
# MongoDB Server connection info.
###################################################
if opts.mongod_server:
config.vmtp_mongod_ip = opts.mongod_server
else:
config.vmtp_mongod_ip = None
if 'vmtp_mongod_port' not in config:
# Set MongoDB default port if not set.
config.vmtp_mongod_port = 27017
# the bandwidth limit for VMs
if opts.vm_bandwidth:
opts.vm_bandwidth = opts.vm_bandwidth.upper().strip()
ex_unit = 'KMG'.find(opts.vm_bandwidth[-1])
try:
if ex_unit == -1:
raise ValueError
val = int(opts.vm_bandwidth[0:-1])
except ValueError:
print 'Invalid --bandwidth parameter. A valid input must '\
'specify only one unit (K|M|G).'
sys.exit(1)
config.vm_bandwidth = int(val * (10 ** (ex_unit * 3)))
# the pkt size for TCP and UDP
if opts.tcp_pkt_sizes:
try:
config.tcp_pkt_sizes = opts.tcp_pkt_sizes.split(',')
for i in xrange(len(config.tcp_pkt_sizes)):
config.tcp_pkt_sizes[i] = int(config.tcp_pkt_sizes[i])
except ValueError:
print 'Invalid --tcpbuf parameter. A valid input must be '\
'integers seperated by comma.'
sys.exit(1)
if opts.udp_pkt_sizes:
try:
config.udp_pkt_sizes = opts.udp_pkt_sizes.split(',')
for i in xrange(len(config.udp_pkt_sizes)):
config.udp_pkt_sizes[i] = int(config.udp_pkt_sizes[i])
except ValueError:
print 'Invalid --udpbuf parameter. A valid input must be '\
'integers seperated by comma.'
sys.exit(1)
if opts.reuse_network_name:
config.reuse_network_name = opts.reuse_network_name
if opts.os_dataplane_network:
config.os_dataplane_network = opts.os_dataplane_network
config.delete_image_after_run = opts.delete_image_after_run
#####################################################
# Set Ganglia server ip and port if the monitoring (-m)
# option is enabled.
#####################################################
config.gmond_svr_ip = None
config.gmond_svr_port = None
if opts.monitor:
# Add the default gmond port if not present
if ':' not in opts.monitor:
opts.monitor += ':8649'
mobj = re.match(r'(\d+\.\d+\.\d+\.\d+):(\d+)', opts.monitor)
if mobj:
config.gmond_svr_ip = mobj.group(1)
config.gmond_svr_port = mobj.group(2)
print "Ganglia monitoring enabled (%s:%s)" % \
(config.gmond_svr_ip, config.gmond_svr_port)
config.time = 30
else:
print 'Invalid --monitor syntax: ' + opts.monitor
###################################################
# Once we parse the config files, normalize
# the paths so that all paths are absolute paths.
###################################################
normalize_paths(config)
# Check the tp-tool name
config.protocols = opts.protocols.upper()
if 'M' in config.protocols or opts.multicast_addr:
# nuttcp required for multicast
opts.tp_tool = 'nuttcp'
config.tp_tool = nuttcp_tool.NuttcpTool
# If M provided, but not multicast_addr, use default (231.1.1.1)
config.multicast_addr = opts.multicast_addr if opts.multicast_addr else "231.1.1.1"
# If --multicast_addr provided, ensure 'M' is in protocols.
if 'M' not in config.protocols:
config.protocols += 'M'
elif 'T' in config.protocols or 'U' in config.protocols:
if opts.tp_tool.lower() == 'nuttcp':
config.tp_tool = nuttcp_tool.NuttcpTool
elif opts.tp_tool.lower() == 'iperf':
config.tp_tool = iperf_tool.IperfTool
else:
print 'Invalid transport tool: ' + opts.tp_tool
sys.exit(1)
else:
config.tp_tool = None
return config
def run(self):
service_list = []
try:
log.info("""
----------------------------
--- Picammory Starting ---
----------------------------
""")
os.chdir(os.path.expanduser('~/picammory/picammory/'))
log.info("os.getcwd() = '%s'", os.getcwd())
#---
# for gpio_pin in (5, 23, 24, 25):
# subprocess.call(['/usr/local/bin/gpio', 'export', str(gpio_pin), 'out'])
# wiringpi2.wiringPiSetupSys()
#---
config.config_load('picammory.ini')
#---
# LEDProcessor.start()
# service_list.append(LEDProcessor)
#
# LEDProcessor.blue(True)
#---
MailService.start()
service_list.append(MailService)
MailService.send_message('Picammory', 'Picammory Server Starting')
# LEDProcessor.green(True)
#---
FtpUploader.start()
service_list.append(FtpUploader)
# LEDProcessor.red(True)
#---
log.info("Start Cam Processor")
self.cam_processor = CamProcessor(
os.path.expanduser('~/picammory_storage/'))
service_list.append(self.cam_processor)
# LEDProcessor.blue(False)
#---
# LEDProcessor.green(False)
# LEDProcessor.red(False)
#--- Wait until one thread stop...
stopped_service = None
while (stopped_service is None):
for service in service_list:
if not service.is_alive():
log.info("Service %s is dead", service)
stopped_service = service
break
time.sleep(1)
except EnvironmentError as e:
log.exception("Picammory failed 1")
MailService.send_message(
"Picammory Error",
"EnvironmentError: SprinklermoryServer failed: %d (%s)" %
(e.errno, e.strerror))
except Exception as e:
log.exception("Picammory failed 2")
MailService.send_message("Picammory Error",
"Exception: Scheduler failed: %s" % (e))
finally:
for service in service_list:
service.terminate()
log.info('''
------------------------
--- Picammory Exit ---
------------------------
''')
sys.exit(1)
