def run_scenario(settings, params, curr_exp, n_exp):
"""Run a single scenario experiment
Parameters
----------
settings : Settings
The simulator settings
params : Tree
experiment parameters tree
curr_exp : int
sequence number of the experiment
n_exp : int
Number of scheduled experiments
Returns
-------
results : 3-tuple
A (params, results, duration) 3-tuple. The first element is a dictionary
which stores all the attributes of the experiment. The second element
is a dictionary which stores the results. The third element is an
integer expressing the wall-clock duration of the experiment (in
seconds)
"""
try:
start_time = time.time()
proc_name = mp.current_process().name
logger = logging.getLogger('runner-%s' % proc_name)
# Get list of metrics required
metrics = settings.DATA_COLLECTORS
# Copy parameters so that they can be manipulated
tree = copy.deepcopy(params)
# Set topology
topology_spec = tree['topology']
topology_name = topology_spec.pop('name')
if topology_name not in TOPOLOGY_FACTORY:
logger.error(
'No topology factory implementation for %s was found.' %
topology_name)
return None
topology = TOPOLOGY_FACTORY[topology_name](**topology_spec)
workload_spec = tree['workload']
workload_name = workload_spec.pop('name')
if workload_name not in WORKLOAD:
logger.error('No workload implementation named %s was found.' %
workload_name)
return None
workload = WORKLOAD[workload_name](topology, **workload_spec)
# Assign caches to nodes
if 'cache_placement' in tree:
cachepl_spec = tree['cache_placement']
cachepl_name = cachepl_spec.pop('name')
if cachepl_name not in CACHE_PLACEMENT:
logger.error('No cache placement named %s was found.' %
cachepl_name)
return None
network_cache = cachepl_spec.pop('network_cache')
# Cache budget is the cumulative number of cache entries across
# the whole network
cachepl_spec['cache_budget'] = workload.n_contents * network_cache
CACHE_PLACEMENT[cachepl_name](topology, **cachepl_spec)
# Assign contents to sources
# If there are many contents, after doing this, performing operations
# requiring a topology deep copy, i.e. to_directed/undirected, will
# take long.
contpl_spec = tree['content_placement']
contpl_name = contpl_spec.pop('name')
if contpl_name not in CONTENT_PLACEMENT:
logger.error(
'No content placement implementation named %s was found.' %
contpl_name)
return None
CONTENT_PLACEMENT[contpl_name](topology, workload.contents,
**contpl_spec)
# caching and routing strategy definition
strategy = tree['strategy']
if strategy['name'] not in STRATEGY:
logger.error('No implementation of strategy %s was found.' %
strategy['name'])
return None
# cache eviction policy definition
cache_policy = tree['cache_policy']
if cache_policy['name'] not in CACHE_POLICY:
logger.error('No implementation of cache policy %s was found.' %
cache_policy['name'])
return None
# Configuration parameters of network model
netconf = tree['netconf']
# Text description of the scenario run to print on screen
scenario = tree['desc'] if 'desc' in tree else "Description N/A"
logger.info('Experiment %d/%d | Preparing scenario: %s', curr_exp,
n_exp, scenario)
if any(m not in DATA_COLLECTOR for m in metrics):
logger.error(
'There are no implementations for at least one data collector specified'
)
return None
collectors = {m: {} for m in metrics}
logger.info('Experiment %d/%d | Start simulation', curr_exp, n_exp)
results = exec_experiment(topology, workload, netconf, strategy,
cache_policy, collectors)
duration = time.time() - start_time
logger.info('Experiment %d/%d | End simulation | Duration %s.',
curr_exp, n_exp, timestr(duration, True))
return (params, results, duration)
except KeyboardInterrupt:
logger.error('Received keyboard interrupt. Terminating')
sys.exit(-signal.SIGINT)
except Exception as e:
err_type = str(type(e)).split("'")[1].split(".")[1]
err_message = e.message
logger.error('Experiment %d/%d | Failed | %s: %s\n%s', curr_exp, n_exp,
err_type, err_message, traceback.format_exc())
def run_scenario(settings, params, curr_exp, n_exp):
"""Run a single scenario experiment
Parameters
----------
settings : Settings
The simulator settings
params : Tree
experiment parameters tree
curr_exp : int
sequence number of the experiment
n_exp : int
Number of scheduled experiments
Returns
-------
results : 3-tuple
A (params, results, duration) 3-tuple. The first element is a dictionary
which stores all the attributes of the experiment. The second element
is a dictionary which stores the results. The third element is an
integer expressing the wall-clock duration of the experiment (in
seconds)
"""
try:
start_time = time.time()
proc_name = mp.current_process().name
logger = logging.getLogger('runner-%s' % proc_name)
# Get list of metrics required
metrics = settings.DATA_COLLECTORS
# Copy parameters so that they can be manipulated
tree = copy.deepcopy(params)
# Set topology
topology_spec = tree['topology']
topology_name = topology_spec.pop('name')
if topology_name not in TOPOLOGY_FACTORY:
logger.error('No topology factory implementation for %s was found.'
% topology_name)
return None
topology = TOPOLOGY_FACTORY[topology_name](**topology_spec)
workload_spec = tree['workload']
workload_name = workload_spec.pop('name')
if workload_name not in WORKLOAD:
logger.error('No workload implementation named %s was found.'
% workload_name)
return None
workload = WORKLOAD[workload_name](topology, **workload_spec)
# Assign caches to nodes
if 'cache_placement' in tree:
cachepl_spec = tree['cache_placement']
cachepl_name = cachepl_spec.pop('name')
if cachepl_name not in CACHE_PLACEMENT:
logger.error('No cache placement named %s was found.'
% cachepl_name)
return None
network_cache = cachepl_spec.pop('network_cache')
# Cache budget is the cumulative number of cache entries across
# the whole network
cachepl_spec['cache_budget'] = workload.n_contents * network_cache
CACHE_PLACEMENT[cachepl_name](topology, **cachepl_spec)
# Assign contents to sources
# If there are many contents, after doing this, performing operations
# requiring a topology deep copy, i.e. to_directed/undirected, will
# take long.
contpl_spec = tree['content_placement']
contpl_name = contpl_spec.pop('name')
if contpl_name not in CONTENT_PLACEMENT:
logger.error('No content placement implementation named %s was found.'
% contpl_name)
return None
CONTENT_PLACEMENT[contpl_name](topology, workload.contents, **contpl_spec)
# caching and routing strategy definition
strategy = tree['strategy']
if strategy['name'] not in STRATEGY:
logger.error('No implementation of strategy %s was found.' % strategy['name'])
return None
# cache eviction policy definition
cache_policy = tree['cache_policy']
if cache_policy['name'] not in CACHE_POLICY:
logger.error('No implementation of cache policy %s was found.' % cache_policy['name'])
return None
# Configuration parameters of network model
netconf = tree['netconf']
# Text description of the scenario run to print on screen
scenario = tree['desc'] if 'desc' in tree else "Description N/A"
logger.info('Experiment %d/%d | Preparing scenario: %s', curr_exp, n_exp, scenario)
if any(m not in DATA_COLLECTOR for m in metrics):
logger.error('There are no implementations for at least one data collector specified')
return None
collectors = {m: {} for m in metrics}
logger.info('Experiment %d/%d | Start simulation', curr_exp, n_exp)
results = exec_experiment(topology, workload, netconf, strategy, cache_policy, collectors)
duration = time.time() - start_time
logger.info('Experiment %d/%d | End simulation | Duration %s.',
curr_exp, n_exp, timestr(duration, True))
return (params, results, duration)
except KeyboardInterrupt:
logger.error('Received keyboard interrupt. Terminating')
sys.exit(-signal.SIGINT)
except Exception as e:
err_type = str(type(e)).split("'")[1].split(".")[1]
err_message = e.message
logger.error('Experiment %d/%d | Failed | %s: %s\n%s',
curr_exp, n_exp, err_type, err_message,
traceback.format_exc())
def run_scenario(settings, params, curr_exp, n_exp):
"""Run a single scenario experiment
Parameters
----------
settings : Settings
The simulator settings
params : Tree
experiment parameters tree
curr_exp : int
sequence number of the experiment
n_exp : int
Number of scheduled experiments
Returns
-------
results : 3-tuple
A (params, results, duration) 3-tuple. The first element is a dictionary
which stores all the attributes of the experiment. The second element
is a dictionary which stores the results. The third element is an
integer expressing the wall-clock duration of the experiment (in
seconds)
"""
try:
start_time = time.time()
proc_name = mp.current_process().name
logger = logging.getLogger('runner-%s' % proc_name)
# Get list of metrics required
metrics = settings.DATA_COLLECTORS
# Copy parameters so that they can be manipulated
tree = copy.deepcopy(params)
# Set topology
topology_spec = tree['topology']
topology_name = topology_spec.pop('name')
if topology_name not in TOPOLOGY_FACTORY:
logger.error(
'No topology factory implementation for %s was found.' %
topology_name)
return None
topology = TOPOLOGY_FACTORY[topology_name](**topology_spec)
workload_spec = tree['workload']
workload_name = workload_spec.pop('name')
if workload_name not in WORKLOAD:
logger.error('No workload implementation named %s was found.' %
workload_name)
return None
workload = WORKLOAD[workload_name](topology, **workload_spec)
# Assign computation to nodes
if 'computation_placement' in tree:
computationpl_spec = tree['computation_placement']
computationpl_name = computationpl_spec.pop('name')
if computationpl_name not in COMPUTATION_PLACEMENT:
logger.error('No computation placement named %s was found.' %
computationpl_name)
return None
COMPUTATION_PLACEMENT[computationpl_name](topology,
**computationpl_spec)
# Assign caches to nodes
if 'cache_placement' in tree:
cachepl_spec = tree['cache_placement']
cachepl_name = cachepl_spec.pop('name')
if cachepl_name not in CACHE_PLACEMENT:
logger.error('No cache placement named %s was found.' %
cachepl_name)
return None
network_cache = cachepl_spec.pop('network_cache')
# Cache budget is the cumulative number of cache entries across
# the whole network
cachepl_spec['cache_budget'] = workload.n_contents * network_cache
# TODO: ADD STORAGE-SPECIFIC PLACEMENT AND BUDGET VARIABLES!
# Onur: need the full budget to assign to receivers for SIT cache placement
cachepl_spec['n_contents'] = workload.n_contents
# NOTE: cache placement is now done together with comp. spot placement!
#CACHE_PLACEMENT[cachepl_name](topology, **cachepl_spec)
# Assign contents to sources
# If there are many contents, after doing this, performing operations
# requiring a topology deep copy, i.e. to_directed/undirected, will
# take long.
# TODO: NEED TO REVISE THIS WHOLE THING! (even though it seems to be working pretty much fine...)
contpl_spec = tree['content_placement']
contpl_name = contpl_spec.pop('name')
if contpl_name not in CONTENT_PLACEMENT:
logger.error(
'No content placement implementation named %s was found.' %
contpl_name)
return None
#if "_REPO_" in contpl_name:
# contpl_spec = {topics: workload.labels,
# types: ,
# freshness_pers,
# shelf_lives ,
# msg_sizes}
# TODO: THIS IS WHERE CONTENTS ARE PLACED!
# Maybe consider using the ^^^ above spec in the config.py file instead, and not using
# it in workload, or at least use it in both.
# OR Could do the following
# (just to save space and not use the same content specs 2 times in config.py):
# It seems like the contents are generated \/\/\/ within workload (workload.contents)!
if "UNIFORM_REPO" in contpl_name:
CONTENT_PLACEMENT[contpl_name](topology, workload.contents,
**contpl_spec)
elif "REPO" in contpl_name and "TRACE" not in workload_name:
CONTENT_PLACEMENT[contpl_name](topology, workload.data,
workload.freshness_pers,
workload.shelf_lives,
workload.sizes, **contpl_spec)
elif "TRACE" in workload_name and "REPO" in contpl_name:
CONTENT_PLACEMENT[contpl_name](topology, workload.data,
workload.freshness_pers,
workload.shelf_lives,
workload.sizes,
workload.labels_weights,
**contpl_spec)
elif "DATA" in workload_name and "REPO" in contpl_name:
CONTENT_PLACEMENT[contpl_name](topology, workload.data,
workload.freshness_pers,
workload.stor_shelf, workload.sizes,
**contpl_spec)
else:
CONTENT_PLACEMENT[contpl_name](topology, workload.contents,
**contpl_spec)
# TODO: THIS IS WHERE contents should be placed, according to ^^^ PLACEMENT SPECIFICATION!!!
# caching and routing strategy definition
strategy = tree['strategy']
warmup_strategy = tree['warmup_strategy']
if strategy['name'] not in STRATEGY:
logger.error('No implementation of strategy %s was found.' %
strategy['name'])
return None
if warmup_strategy['name'] not in STRATEGY:
logger.error(
'No implementation of warm-up strategy %s was found.' %
warmup_strategy['name'])
return None
# cache eviction policy definition
cache_policy = tree['cache_policy']
if cache_policy['name'] not in CACHE_POLICY:
logger.error('No implementation of cache policy %s was found.' %
cache_policy['name'])
return None
# cache eviction policy definition
repo_policy = tree['repo_policy']
if repo_policy['name'] not in REPO_POLICY:
logger.error('No implementation of repo policy %s was found.' %
repo_policy['name'])
return None
# task scheduling policy at the computation spots
sched_policy = tree['sched_policy']
# Configuration parameters of network model
netconf = tree['netconf']
# Text description of the scenario run to print on screen
scenario = tree['desc'] if 'desc' in tree else "Description N/A"
logger.info('Experiment %d/%d | Preparing scenario: %s', curr_exp,
n_exp, scenario)
if any(m not in DATA_COLLECTOR for m in metrics):
logger.error(
'There are no implementations for at least one data collector specified'
)
return None
collect_spec = tree['collector_params']
collectors = {m: {} for m in metrics}
# for m in collectors:
# if 'REPO' in m:
# collectors[m] = dict(collect_spec)
logger.info('Experiment %d/%d | Start simulation', curr_exp, n_exp)
results = exec_experiment(topology, workload, netconf, strategy,
cache_policy, repo_policy, collectors,
warmup_strategy, sched_policy)
duration = time.time() - start_time
logger.info('Experiment %d/%d | End simulation | Duration %s.',
curr_exp, n_exp, timestr(duration, True))
return (params, results, duration)
except KeyboardInterrupt:
logger.error('Received keyboard interrupt. Terminating')
sys.exit(-signal.SIGINT)
except Exception as e:
err_type = str(type(e)).split("'")[1].split(".")[1]
err_message = e.message
logger.error('Experiment %d/%d | Failed | %s: %s\n%s', curr_exp, n_exp,
err_type, err_message, traceback.format_exc())
def run_scenario(settings, params, curr_exp, n_exp):
"""Run a single scenario experiment
Parameters
----------
settings : Settings
The simulator settings
params : dict
Dictionary of parameters
curr_exp : int
sequence number of the experiment
n_exp : int
Number of scheduled experiments
Returns
-------
results : 3-tuple
A (params, results, duration) 3-tuple. The first element is a dictionary
which stores all the attributes of the experiment. The second element
is a dictionary which stores the results. The third element is an
integer expressing the wall-clock duration of the experiment (in
seconds)
"""
try:
start_time = time.time()
proc_name = mp.current_process().name
logger = logging.getLogger('runner-%s' % proc_name)
alpha = params['alpha']
topology_name = params['topology_name']
network_cache = params['network_cache']
strategy_name = params['strategy_name']
n_contents = params['n_contents']
strategy_params = params['strategy_params']
cache_policy = params['cache_policy'] if 'cache_policy' in params \
and params['cache_policy'] is not None \
else settings.CACHE_POLICY
metrics = settings.DATA_COLLECTORS
scenario = "%s, %s, alpha: %s, netcache: %s" % (topology_name, strategy_name, str(alpha), str(network_cache))
logger.info('Experiment %d/%d | Preparing scenario: %s', curr_exp, n_exp, scenario)
# Check parameters
if topology_name not in topology_factory_register:
logger.error('No topology factory implementation for %s was found.' % topology_name)
return None
if cache_policy not in cache_policy_register:
logger.error('No implementation of cache policy %s was found.' % cache_policy)
return None
if strategy_name not in strategy_register:
logger.error('No implementation of strategy %s was found.' % strategy_name)
return None
if any(m not in data_collector_register for m in metrics):
logger.error('There are no implementations for at least one data collector specified')
return None
# Get user-defined seed, if any
seed = settings.SEED if 'SEED' in settings else None
# Get topology and event generator
topology = topology_factory_register[topology_name](network_cache, n_contents, seed=seed)
events = uniform_req_gen(topology, n_contents, alpha,
rate=settings.NETWORK_REQUEST_RATE,
n_warmup=settings.N_WARMUP_REQUESTS,
n_measured=settings.N_MEASURED_REQUESTS,
seed=seed)
topology.graph['cache_policy'] = cache_policy
collectors = [(m, {}) for m in metrics]
strategy = (strategy_name, strategy_params)
logger.info('Experiment %d/%d | Start simulation', curr_exp, n_exp)
results = exec_experiment(topology, events, strategy, collectors)
duration = time.time() - start_time
logger.info('Experiment %d/%d | End simulation | Duration %s.',
curr_exp, n_exp, timestr(duration, True))
return (params, results, duration)
except KeyboardInterrupt:
logger.error('Received keyboard interrupt. Terminating')
sys.exit(-signal.SIGINT)
except Exception as e:
err_type = str(type(e)).split("'")[1].split(".")[1]
err_message = e.message
logger.error('Experiment %d/%d | Failed | %s: %s\n%s',
curr_exp, n_exp, err_type, err_message,
traceback.format_exc())
def run_scenario(settings, params, curr_exp, n_exp):
"""Run a single scenario experiment
Parameters
----------
settings : Settings
The simulator settings
params : dict
Dictionary of parameters
curr_exp : int
sequence number of the experiment
n_exp : int
Number of scheduled experiments
Returns
-------
results : 2-tuple
A 2-tuple of dictionaries. The first dict stores all the attributes of
the experiment. The second dict stores the results.
"""
try:
start_time = time.time()
proc_name = mp.current_process().name
logger = logging.getLogger('runner-%s' % proc_name)
alpha = params['alpha']
topology_name = params['topology_name']
network_cache = params['network_cache']
strategy_name = params['strategy_name']
n_contents = params['n_contents']
strategy_params = params['strategy_params']
cache_policy = params['cache_policy'] if 'cache_policy' in params \
and params['cache_policy'] is not None \
else settings.CACHE_POLICY
metrics = settings.DATA_COLLECTORS
## MT add
log_dir = settings.LOG_DIR
scenario_id = 'T=%s-C=%s-A=%s-S=%s-Run=%s' % (topology_name, str(network_cache), str(alpha), strategy_name, str(n_exp))
scenario = "%s, %s, alpha: %s, netcache: %s" % (topology_name, strategy_name, str(alpha), str(network_cache))
logger.info('Experiment %d/%d | Preparing scenario: %s', curr_exp, n_exp, scenario)
# Check parameters
if topology_name not in topology_factory_register:
logger.error('No topology factory implementation for %s was found.' % topology_name)
return None
if cache_policy not in cache_policy_register:
logger.error('No implementation of cache policy %s was found.' % cache_policy)
return None
if strategy_name not in strategy_register:
logger.error('No implementation of strategy %s was found.' % strategy_name)
return None
if any(m not in data_collector_register for m in metrics):
logger.error('There are no implementations for at least one data collector specified')
return None
# Get topology and event generator
topology = topology_factory_register[topology_name](network_cache, n_contents)
events = uniform_req_gen(topology, n_contents, alpha,
rate=settings.NETWORK_REQUEST_RATE,
n_warmup=settings.N_WARMUP_REQUESTS,
n_measured=settings.N_MEASURED_REQUESTS)
topology.graph['cache_policy'] = cache_policy
collectors = [(m, {}) for m in metrics]
strategy = (strategy_name, strategy_params)
logger.info('Experiment %d/%d | Start simulation', curr_exp, n_exp)
results = exec_experiment(topology, events, strategy, collectors)
duration = time.time() - start_time
logger.info('Experiment %d/%d | End simulation | Duration %s.',
curr_exp, n_exp, timestr(duration, True))
return (params, results, curr_exp, duration)
except KeyboardInterrupt:
logger.error('Received keyboard interrupt. Terminating')
sys.exit(-signal.SIGINT)