def execute_simulation(add_rio=False,
ol_start=0,
rio_length=18,
rio_visitors=380e3,
n_simulations=5):
sol_global = []
sol_rio = []
sol_moscow = []
sol_berlin = []
sol_beijing = []
sol_sydney = []
sol_new_york = []
params = {}
params['global'] = []
params['rio'] = []
params['moscow'] = []
params['berlin'] = []
params['beijing'] = []
params['sydney'] = []
params['new'] = []
for j in range(n_simulations):
print("running simulation {0} / {1}".format(j + 1, n_simulations))
state = State(regions, routes, verbose=True)
state.set_outbreak('Rio De Janeiro', 1e3) #'Rio De Janeiro', 1000)
sim = Simulator(state,
transfer_prob=0.005,
beta=2,
gamma=0.5,
verbose=True)
sol_global.append([])
sol_rio.append([])
sol_moscow.append([])
sol_berlin.append([])
sol_beijing.append([])
sol_sydney.append([])
sol_new_york.append([])
state_list = []
for i, state in enumerate(sim.run(iterations=120)):
state_list.append(state)
if i == ol_start and add_rio: # start outbreak x days before olympics
sim.add_event(2560,
days=rio_length,
total_transfer=rio_visitors)
sol_global[j].append(state.total_sir().as_tuple(total=True))
sol_rio[j].append(state.region_sir[2560].as_tuple(total=True))
sol_moscow[j].append(state.region_sir[4029].as_tuple(total=True))
sol_berlin[j].append(state.region_sir[351].as_tuple(total=True))
sol_beijing[j].append(state.region_sir[3364].as_tuple(total=True))
sol_sydney[j].append(state.region_sir[3361].as_tuple(total=True))
sol_new_york[j].append(state.region_sir[3797].as_tuple(total=True))
params['global'].append(
sir.ParameterEstimator(iter([x.total_sir() for x in state_list]),
method='max').beta)
params['rio'].append(
sir.ParameterEstimator(iter(
[x.region_sir[2560] for x in state_list]),
method='max').beta)
params['moscow'].append(
sir.ParameterEstimator(iter(
[x.region_sir[4029] for x in state_list]),
method='max').beta)
params['berlin'].append(
sir.ParameterEstimator(iter(
[x.region_sir[351] for x in state_list]),
method='max').beta)
params['beijing'].append(
sir.ParameterEstimator(iter(
[x.region_sir[3364] for x in state_list]),
method='max').beta)
params['sydney'].append(
sir.ParameterEstimator(iter(
[x.region_sir[3361] for x in state_list]),
method='max').beta)
params['new'].append(
sir.ParameterEstimator(iter(
[x.region_sir[2560] for x in state_list]),
method='max').beta)
if add_rio:
fig_name = "rio-{0}-{1}-{2:d}.pdf".format(ol_start, rio_length,
int(rio_visitors))
else:
fig_name = "no_rio.pdf"
plot_sir([
sol_global, sol_rio, sol_new_york, sol_berlin, sol_moscow, sol_beijing,
sol_sydney
], [
'Global', 'Rio De Janeiro', 'New York', 'Berlin', 'Moscow', 'Beijing',
'Sydney'
], fig_name)
# estimate means and variance
global_values = sol_global
peak_times_global = [np.argmax([x[1] for x in y]) for y in global_values]
peak_amount_global = [
y[peak][1] for peak, y in zip(peak_times_global, global_values)
]
peak_times_rio = [np.argmax([x[1] for x in y]) for y in sol_rio]
peak_times_new_york = [np.argmax([x[1] for x in y]) for y in sol_new_york]
peak_times_berlin = [np.argmax([x[1] for x in y]) for y in sol_berlin]
peak_times_moscow = [np.argmax([x[1] for x in y]) for y in sol_moscow]
peak_times_beijing = [np.argmax([x[1] for x in y]) for y in sol_beijing]
peak_times_sydney = [np.argmax([x[1] for x in y]) for y in sol_sydney]
t_deviations = scipy.stats.t.ppf(0.975, len(peak_times_rio) - 1)
# estimate variance with control variates
with open('control-{0}.csv'.format(add_rio), 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow([
'global_amount', 'global_amount_control', 'global_peak_time',
'global_peak_time_control', 'rio_time', 'rio_time_control',
'new_york_time', 'new_york_time_control', 'berlin_time',
'berlin_time_control', 'moscow_time', 'moscow_time_control',
'beijing_time', 'beijing_time_control', 'sydney_time',
'sydney_time_control'
])
for i in range(n_simulations):
writer.writerow([
peak_amount_global[i], params['global'][i],
peak_times_global[i], params['global'][i], peak_times_rio[i],
params['rio'][i], peak_times_rio[i], params['new'][i],
peak_times_rio[i], params['berlin'][i], peak_times_rio[i],
params['moscow'][i], peak_times_rio[i], params['beijing'][i],
peak_times_rio[i], params['sydney'][i]
])
amount_global_control_conf = control_variate_conf(peak_amount_global,
params['global'])
time_global_control_conf = control_variate_conf(peak_times_global,
params['global'])
time_rio_control_conf = control_variate_conf(peak_times_rio, params['rio'])
time_new_york_control_conf = control_variate_conf(peak_times_new_york,
params['new'])
time_berlin_control_conf = control_variate_conf(peak_times_berlin,
params['berlin'])
time_moscow_control_conf = control_variate_conf(peak_times_moscow,
params['moscow'])
time_beijing_control_conf = control_variate_conf(peak_times_beijing,
params['beijing'])
time_sydney_control_conf = control_variate_conf(peak_times_sydney,
params['sydney'])
return [(np.mean(peak_amount_global), t_deviations *
np.std(peak_amount_global, ddof=1) / math.sqrt(n_simulations),
amount_global_control_conf),
(np.mean(peak_times_global), t_deviations *
np.std(peak_times_global, ddof=1) / math.sqrt(n_simulations),
time_global_control_conf),
(np.mean(peak_times_rio), t_deviations *
np.std(peak_times_rio, ddof=1) / math.sqrt(n_simulations),
time_rio_control_conf),
(np.mean(peak_times_new_york), t_deviations *
np.std(peak_times_new_york, ddof=1) / math.sqrt(n_simulations),
time_new_york_control_conf),
(np.mean(peak_times_berlin), t_deviations *
np.std(peak_times_berlin, ddof=1) / math.sqrt(n_simulations),
time_berlin_control_conf),
(np.mean(peak_times_moscow), t_deviations *
np.std(peak_times_moscow, ddof=1) / math.sqrt(n_simulations),
time_moscow_control_conf),
(np.mean(peak_times_beijing), t_deviations *
np.std(peak_times_beijing, ddof=1) / math.sqrt(n_simulations),
time_beijing_control_conf),
(np.mean(peak_times_sydney), t_deviations *
np.std(peak_times_sydney, ddof=1) / math.sqrt(n_simulations),
time_sydney_control_conf)]
def execute_simulation(add_rio=False, ol_start=0, rio_length=18,
rio_visitors=380e3, n_simulations=5):
sol_global = []
sol_rio = []
sol_moscow = []
sol_berlin = []
sol_beijing = []
sol_sydney = []
sol_new_york = []
params = {}
params['global'] = []
params['rio'] = []
params['moscow'] = []
params['berlin'] = []
params['beijing'] = []
params['sydney'] = []
params['new'] = []
for j in range(n_simulations):
print("running simulation {0} / {1}".format(j + 1, n_simulations))
state = State(regions, routes, verbose=True)
state.set_outbreak('Rio De Janeiro', 1e3)#'Rio De Janeiro', 1000)
sim = Simulator(state, transfer_prob=0.005, beta=2, gamma=0.5,
verbose=True)
sol_global.append([])
sol_rio.append([])
sol_moscow.append([])
sol_berlin.append([])
sol_beijing.append([])
sol_sydney.append([])
sol_new_york.append([])
state_list = []
for i, state in enumerate(sim.run(iterations=120)):
state_list.append(state)
if i == ol_start and add_rio: # start outbreak x days before olympics
sim.add_event(2560, days=rio_length, total_transfer=rio_visitors)
sol_global[j].append(state.total_sir().as_tuple(total=True))
sol_rio[j].append(state.region_sir[2560].as_tuple(total=True))
sol_moscow[j].append(state.region_sir[4029].as_tuple(total=True))
sol_berlin[j].append(state.region_sir[351].as_tuple(total=True))
sol_beijing[j].append(state.region_sir[3364].as_tuple(total=True))
sol_sydney[j].append(state.region_sir[3361].as_tuple(total=True))
sol_new_york[j].append(state.region_sir[3797].as_tuple(total=True))
params['global'].append(sir.ParameterEstimator(
iter([x.total_sir() for x in state_list]), method='max').beta)
params['rio'].append(sir.ParameterEstimator(
iter([x.region_sir[2560] for x in state_list]), method='max').beta)
params['moscow'].append(sir.ParameterEstimator(
iter([x.region_sir[4029] for x in state_list]), method='max').beta)
params['berlin'].append(sir.ParameterEstimator(
iter([x.region_sir[351] for x in state_list]), method='max').beta)
params['beijing'].append(sir.ParameterEstimator(
iter([x.region_sir[3364] for x in state_list]), method='max').beta)
params['sydney'].append(sir.ParameterEstimator(
iter([x.region_sir[3361] for x in state_list]), method='max').beta)
params['new'].append(sir.ParameterEstimator(
iter([x.region_sir[2560] for x in state_list]), method='max').beta)
if add_rio:
fig_name = "rio-{0}-{1}-{2:d}.pdf".format(ol_start, rio_length,
int(rio_visitors))
else:
fig_name = "no_rio.pdf"
plot_sir([sol_global, sol_rio, sol_new_york, sol_berlin,
sol_moscow, sol_beijing, sol_sydney],
['Global', 'Rio De Janeiro', 'New York', 'Berlin',
'Moscow', 'Beijing', 'Sydney'], fig_name)
# estimate means and variance
global_values = sol_global
peak_times_global = [np.argmax([x[1] for x in y])
for y in global_values]
peak_amount_global = [y[peak][1]
for peak, y in zip(peak_times_global, global_values)]
peak_times_rio = [np.argmax([x[1] for x in y])
for y in sol_rio]
peak_times_new_york = [np.argmax([x[1] for x in y])
for y in sol_new_york]
peak_times_berlin = [np.argmax([x[1] for x in y])
for y in sol_berlin]
peak_times_moscow = [np.argmax([x[1] for x in y])
for y in sol_moscow]
peak_times_beijing = [np.argmax([x[1] for x in y])
for y in sol_beijing]
peak_times_sydney = [np.argmax([x[1] for x in y])
for y in sol_sydney]
t_deviations = scipy.stats.t.ppf(0.975, len(peak_times_rio)-1)
# estimate variance with control variates
with open('control-{0}.csv'.format(add_rio), 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['global_amount', 'global_amount_control',
'global_peak_time', 'global_peak_time_control',
'rio_time', 'rio_time_control',
'new_york_time', 'new_york_time_control',
'berlin_time', 'berlin_time_control',
'moscow_time', 'moscow_time_control',
'beijing_time', 'beijing_time_control',
'sydney_time', 'sydney_time_control'])
for i in range(n_simulations):
writer.writerow([peak_amount_global[i], params['global'][i],
peak_times_global[i], params['global'][i],
peak_times_rio[i], params['rio'][i],
peak_times_rio[i], params['new'][i],
peak_times_rio[i], params['berlin'][i],
peak_times_rio[i], params['moscow'][i],
peak_times_rio[i], params['beijing'][i],
peak_times_rio[i], params['sydney'][i]
])
amount_global_control_conf = control_variate_conf(peak_amount_global, params['global'])
time_global_control_conf = control_variate_conf(peak_times_global, params['global'])
time_rio_control_conf = control_variate_conf(peak_times_rio, params['rio'])
time_new_york_control_conf = control_variate_conf(peak_times_new_york, params['new'])
time_berlin_control_conf = control_variate_conf(peak_times_berlin, params['berlin'])
time_moscow_control_conf = control_variate_conf(peak_times_moscow, params['moscow'])
time_beijing_control_conf = control_variate_conf(peak_times_beijing, params['beijing'])
time_sydney_control_conf = control_variate_conf(peak_times_sydney, params['sydney'])
return [(np.mean(peak_amount_global),
t_deviations * np.std(peak_amount_global, ddof=1) / math.sqrt(n_simulations),
amount_global_control_conf),
(np.mean(peak_times_global),
t_deviations * np.std(peak_times_global, ddof=1) / math.sqrt(n_simulations),
time_global_control_conf),
(np.mean(peak_times_rio),
t_deviations * np.std(peak_times_rio, ddof=1) / math.sqrt(n_simulations),
time_rio_control_conf),
(np.mean(peak_times_new_york),
t_deviations * np.std(peak_times_new_york, ddof=1) / math.sqrt(n_simulations),
time_new_york_control_conf),
(np.mean(peak_times_berlin),
t_deviations * np.std(peak_times_berlin, ddof=1) / math.sqrt(n_simulations),
time_berlin_control_conf),
(np.mean(peak_times_moscow),
t_deviations * np.std(peak_times_moscow, ddof=1) / math.sqrt(n_simulations),
time_moscow_control_conf),
(np.mean(peak_times_beijing),
t_deviations * np.std(peak_times_beijing, ddof=1) / math.sqrt(n_simulations),
time_beijing_control_conf),
(np.mean(peak_times_sydney),
t_deviations * np.std(peak_times_sydney, ddof=1) / math.sqrt(n_simulations),
time_sydney_control_conf)
]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("sls_trace_file", help="Path to SLS trace file (JSON format).")
parser.add_argument("network_topology_file", help="Path to topo file (JSON format).")
parser.add_argument("node_mem_mb", type=int, help="Amount of memory in MB for each node.")
parser.add_argument("node_cores", type=int, help="Amount of virtual cores for each node.")
parser.add_argument("node_hb_ms", type=int, help="Period of the NM heartbeats in milliseconds.")
parser.add_argument("am_hb_ms", type=int, help="Period of the AM heartbeats in milliseconds.")
parser.add_argument("am_container_mb", type=int, help="Amount of memory in MB for each AM container.")
parser.add_argument("am_container_cores", type=int, help="Number of cores taken by each AM container.")
parser.add_argument("scheduler_type", type=get_enumparser(YarnSchedulerType, YarnSchedulerType.SYMBEX),
choices=filter(lambda x: x is not YarnSchedulerType.SYMBEX, YarnSchedulerType),
help="Type of scheduler to run", default=YarnSchedulerType.REGULAR.name, nargs="?")
elastic_args = parser.add_argument_group("ELASTIC arguments")
elastic_args.add_argument("penalty", type=get_enumparser(YarnPenalty), choices=YarnPenalty,
help="The performance penalty model to use", nargs="?")
elastic_args.add_argument("ib", type=float, help="The initial bump of the penalty model", nargs="?")
elastic_args.add_argument("--ep", type=int,
help="Percentage of nodes which are always elastic (works only for GREEDY & SMARTG)")
peek_args = parser.add_argument_group("PEEK arguments")
peek_args.add_argument("-p", "--peek-pushback-strategy", type=get_enumparser(PushbackStrategy),
choices=PushbackStrategy, nargs="?", help="Strategy to use to avoid pushback in PEEK.",
default=None)
race_args = parser.add_argument_group("RACE arguments")
from events.yarn.elastic.race import YarnRaceMetric
race_args.add_argument("--race-metric", type=get_enumparser(YarnRaceMetric), choices=YarnRaceMetric,
help="Metric by which to compare REGULAR vs. SMARTG in RACE.",
default=YarnRaceMetric.TOTAL_JRT.name)
race_args.add_argument("--race-lockstep-regular", action="store_true",
help="Switch flag to REGULAR until end of simulation.", default=False)
race_args.add_argument("--race-duration-range", type=parsenumlist,
help="Duration range (min, max) in ms of the " +
"RACE simulations w.r.t. the simulation timeline.",
default=(0, 0))
race_args.add_argument("--race-never-degrade", action="store_true",
help="Always favour REGULAR if any job gets worse running times by using SMARTG.",
default=False)
race_args.add_argument("--race-cutoff-perc", type=int,
help="Only simulate a percentage of the remaining containers.",
default=0)
symbex_args = parser.add_argument_group("SYMBEX arguments")
symbex_args.add_argument("--symbex", action="store_true",
help="Run a symbex-type exploration on SMARTG vs REGULAR scheduling decisions.",
default=False)
symbex_args.add_argument("--symbex-mode", type=get_enumparser(SymbexMode), choices=SymbexMode,
help="Symbex exploration to run.", default=SymbexMode.DECISION.name)
symbex_args.add_argument("--symbex-dfs", action="store_true",
help="Do a DFS-type exploration rather than a BFS one.",
default=False)
symbex_args.add_argument("--symbex-workers", type=int, help="Number of concurrent symbex worker threads to run.",
default=1)
errinj_args = parser.add_argument_group("Error injection arguments")
errinj_args.add_argument("--duration-error", type=int,
help="Percentage by which to mis-estimate the running times of the containers.")
errinj_args.add_argument("--duration-error-type", type=get_enumparser(YarnErrorType), choices=YarnErrorType,
help="Whether duration error is positive, negative or either.",
default=YarnErrorType.MIXED.name)
errinj_args.add_argument("--duration-error-mode", type=get_enumparser(YarnErrorMode), choices=YarnErrorMode,
help="Whether duration error is constant or random.",
default=YarnErrorMode.CONSTANT.name)
el_errinj_args = parser.add_argument_group("ELASTIC Error injection arguments")
el_errinj_args.add_argument("--duration-error-only-elastic", action="store_true",
help="Inject duration error only for ELASTIC containers", default=False)
el_errinj_args.add_argument("--mem-error", type=int, help="Percentage by which to mis-estimate the ideal memory " +
"of the containers.")
el_errinj_args.add_argument("--mem-error-type", type=get_enumparser(YarnErrorType), choices=YarnErrorType,
help="Whether the memory misestimation error is positive, negative or either.",
default=YarnErrorType.MIXED.name)
el_errinj_args.add_argument("--mem-error-mode", type=get_enumparser(YarnErrorMode), choices=YarnErrorMode,
help="Whether the memory misestimation error is constant or random.",
default=YarnErrorMode.CONSTANT.name)
el_errinj_args.add_argument("--ib-error", type=int, help="Percentage by which to mis-estimate the IB of tasks.")
el_errinj_args.add_argument("--ib-error-type", type=get_enumparser(YarnErrorType), choices=YarnErrorType,
help="Whether IB error is positive, negative or either.",
default=YarnErrorType.MIXED.name)
el_errinj_args.add_argument("--ib-error-mode", type=get_enumparser(YarnErrorMode), choices=YarnErrorMode,
help="Whether IB error is constant or random.", default=YarnErrorMode.CONSTANT.name)
yarn_args = parser.add_argument_group("YARN arguments")
yarn_args.add_argument("-r", "--use-reservations", action="store_true",
help="Whether an app can place a reservation on a node to guarantee its priority there.",
default=False)
yarn_args.add_argument("-g", "--use-gaps", action="store_true",
help="Used in conjunction with reservations, allows a container to be scheduled on a " +
"reserved node if the duration is less than the reservation duration.", default=False)
yarn_args.add_argument("--gaps-allow-ams", action="store_true",
help="Used in conjunction with reservations and gaps, allows AM containers to be scheduled" +
" on a reserved node.", default=False)
yarn_args.add_argument("--assign-multiple", action="store_true",
help="Run the simulation as if the assignMultiple flag is set (multiple assignments per " +
"heartbeat are allowed).", default=False)
parser.add_argument("--oracle-debug", action="store_true",
help="Display DEBUG messages for all oracle simulations (generates A LOT of output).")
parser.add_argument("--mem-overestimate-range", type=parsenumlist, help="Range (min, max) of multipliers for " +
"memory overestimates by users.",
default=(1, 1))
parser.add_argument("--mem-overestimate-assume", type=float, help="Multiplier by which the system assumes " +
"that users overestimate memory.", default=1)
parser.add_argument("--meganode", action="store_true", help="Pool all node resources together and " +
"run simulation on this one node.", default=False)
parser.add_argument("--occupancy-stats-interval", type=int,
help="Gather occupancy stats each interval (in seconds).", default=0)
parser.add_argument("--occupancy-stats-file", type=str, help="File in which to output occupancy stats.",
default=None)
parser.add_argument("-v", "--verbose", action="count")
args = parser.parse_args()
verify_user_config(args)
if args.verbose == 1:
logging.getLogger().setLevel(logging.INFO)
elif args.verbose == 2:
logging.getLogger().setLevel(logging.DEBUG)
# Start a new simulator
simulator = Simulator()
# Create and run the new YarnGenerator
yarn_generator = YarnGenerator()
state = yarn_generator.generate_state(simulator, args)
# Add the event to the simulator
simulator.add_event(yarn_generator.get_simulation_start_event())
# Catch SIGINT and drop to a console for debugging purposes.
signal.signal(signal.SIGINT, signal_handler)
# Run the simulation
if not args.symbex:
simulator.run()
else:
SymbexRunner(state, args.symbex_workers).run()
this_dir = path.dirname(path.realpath(__file__))
len_simulation = 90
state = State(regions, routes, verbose=True)
state.set_outbreak('Rio De Janeiro', 1000)
sim = Simulator(
state,
transfer_prob=0.005,
beta=2,
gamma=0.5,
verbose=True,
)
sim.add_event(2560, days=18, total_transfer=380e3)
base_map = WorldMap(resolution="c")
base_map.animate(sim, frames=len_simulation, max_infected=0.1)
base_map.ani.save(path.join(this_dir, '../../report/plots/gifs/rio.mp4'),
writer="mencoder_file",
fps=3,
savefig_kwargs={'bbox_inches': 'tight'})
state = State(regions, routes, verbose=True)
state.set_outbreak('Rio De Janeiro', 1000)
sim = Simulator(state, transfer_prob=0.005, beta=2, gamma=0.5, verbose=True)
base_map = WorldMap(resolution="c")
base_map.animate(sim, frames=len_simulation, max_infected=0.1)
base_map.ani.save(path.join(this_dir, '../../report/plots/gifs/no_rio.mp4'),
writer="mencoder_file",
fps=3,
import os.path as path
import time
from display import WorldMap
from simulator import State, Simulator
from world import regions, routes
this_dir = path.dirname(path.realpath(__file__))
len_simulation = 67
state = State(regions, routes, verbose=True)
state.set_outbreak('Rio De Janeiro', 1000)
sim = Simulator(state, transfer_prob=0.005, beta=2, gamma=0.5, verbose=True,)
sim.add_event(2560, days=18, total_transfer=380e3)
for i, state in enumerate(sim.run(len_simulation)):
fig_name = path.join(this_dir, '../../report/plots/gifs/frames/rio-{0}.pdf'.format(i))
if i == 46:
base_map = WorldMap(resolution="c")
base_map.scatter_infections(state, max_infected=0.1, time=i)
base_map.save_fig(fig_name)
break
state = State(regions, routes, verbose=True)
state.set_outbreak('Rio De Janeiro', 1000)
sim = Simulator(state, transfer_prob=0.005, beta=2, gamma=0.5, verbose=True)
for i, state in enumerate(sim.run(len_simulation)):
