def create_data(config, data_path):
dataset = LJSpeech(data_path)
train_dataset = SliceDataset(dataset, config["valid_size"], len(dataset))
train_collator = DataCollector(config["p_pronunciation"])
train_sampler = RandomSampler(train_dataset)
train_cargo = DataCargo(train_dataset,
train_collator,
batch_size=config["batch_size"],
sampler=train_sampler)
train_loader = DataLoader\
.from_generator(capacity=10, return_list=True)\
.set_batch_generator(train_cargo)
valid_dataset = SliceDataset(dataset, 0, config["valid_size"])
valid_collector = DataCollector(1.)
valid_sampler = SequentialSampler(valid_dataset)
valid_cargo = DataCargo(valid_dataset,
valid_collector,
batch_size=1,
sampler=valid_sampler)
valid_loader = DataLoader\
.from_generator(capacity=2, return_list=True)\
.set_batch_generator(valid_cargo)
return train_loader, valid_loader
n_mels = data_config["n_mels"]
train_clip_seconds = data_config["train_clip_seconds"]
transform = Transform(sample_rate, n_fft, win_length, hop_length, n_mels)
ljspeech = TransformDataset(ljspeech_meta, transform)
valid_size = data_config["valid_size"]
ljspeech_valid = SliceDataset(ljspeech, 0, valid_size)
ljspeech_train = SliceDataset(ljspeech, valid_size, len(ljspeech))
model_config = config["model"]
n_loop = model_config["n_loop"]
n_layer = model_config["n_layer"]
filter_size = model_config["filter_size"]
context_size = 1 + n_layer * sum([filter_size**i for i in range(n_loop)])
print("context size is {} samples".format(context_size))
train_batch_fn = DataCollector(context_size, sample_rate, hop_length,
train_clip_seconds)
valid_batch_fn = DataCollector(context_size,
sample_rate,
hop_length,
train_clip_seconds,
valid=True)
batch_size = data_config["batch_size"]
train_cargo = DataCargo(ljspeech_train,
train_batch_fn,
batch_size,
sampler=RandomSampler(ljspeech_train))
# only batch=1 for validation is enabled
valid_cargo = DataCargo(ljspeech_valid,
valid_batch_fn,
def __init__(self, network_file_path, configs):
self.configs = configs
self.file_path = network_file_path
# Initialize the event manager
self.mgr = EventManager()
self.collector = DataCollector()
# Read network in from file.
with open(file_path, 'r') as yaml_file:
network = yaml.load(yaml_file)
# Initialize the network components
self.hosts = {}
for host_name in network['hosts']:
self.hosts[host_name] = Host(host_name, self, self.collector)
self.routers = {}
if 'routers' in network:
for router_name in network['routers']:
self.routers[router_name] = Router(router_name, self)
self.flows = {}
for flow_dict in network['flows']:
flow_name = flow_dict.keys()[0]
flow_desc = flow_dict[flow_name]
self.flows[flow_name] = Flow(
flow_name,
self,
self.collector,
self.hosts[flow_desc['source']] ,
self.hosts[flow_desc['destination']],
flow_desc['amount'],
flow_desc['start']
)
self.links = {}
host_prefixes = ['H', 'S', 'T']
for link_dict in network['links']:
link_name = link_dict.keys()[0]
link_desc = link_dict[link_name]
aname = link_desc['anode']
bname = link_desc['bnode']
ahost = self.hosts[aname] \
if aname[0] in host_prefixes else self.routers[aname]
bhost = self.hosts[bname] \
if bname[0] in host_prefixes else self.routers[bname]
link = Link(
link_name,
self,
self.collector,
link_desc['rate'] / 8.0,
link_desc['delay'],
link_desc['buffer'],
link_desc['buffer'],
ahost,
bhost
)
self.links[link_name] = link
ahost.set_link(link)
bhost.set_link(link)
self.mgr.add_event(0.0, link.calculate_throughput, [])
# Add initial events for each of the flows
for flow in self.flows.values():
self.mgr.add_event(flow.start_t, flow.send_packet, [])
if not configs['tcpreno']:
self.mgr.add_event(flow.start_t, flow.tcp_fast_update, [])
# Initialize the routing tables
for router in self.routers.values():
router.init_routing_table()
if len(self.routers) > 0:
self.mgr.add_event(0.0, self.routers['R1'].send_rout_packet, [])
self.mgr.add_event(self.mgr.t + configs['dynamic_routing_step'],
self.start_dynamic_routing, [])
clip_norm)
ljspeech = TransformDataset(meta, transform)
# =========================dataiterator=========================
# use meta data's text length as a sort key for the sampler
train_config = config["train"]
batch_size = train_config["batch_size"]
text_lengths = [len(example[2]) for example in meta]
sampler = PartialyRandomizedSimilarTimeLengthSampler(text_lengths,
batch_size)
# some hyperparameters affect how we process data, so create a data collector!
model_config = config["model"]
downsample_factor = model_config["downsample_factor"]
r = model_config["outputs_per_step"]
collector = DataCollector(downsample_factor=downsample_factor, r=r)
ljspeech_loader = DataCargo(
ljspeech, batch_fn=collector, batch_size=batch_size, sampler=sampler)
# =========================model=========================
if args.device == -1:
place = fluid.CPUPlace()
else:
place = fluid.CUDAPlace(args.device)
with dg.guard(place):
# =========================model=========================
n_speakers = model_config["n_speakers"]
speaker_dim = model_config["speaker_embed_dim"]
speaker_embed_std = model_config["speaker_embedding_weight_std"]
n_vocab = en.n_vocab
class Simulator(object):
def __init__(self, network_file_path, configs):
self.configs = configs
self.file_path = network_file_path
# Initialize the event manager
self.mgr = EventManager()
self.collector = DataCollector()
# Read network in from file.
with open(file_path, 'r') as yaml_file:
network = yaml.load(yaml_file)
# Initialize the network components
self.hosts = {}
for host_name in network['hosts']:
self.hosts[host_name] = Host(host_name, self, self.collector)
self.routers = {}
if 'routers' in network:
for router_name in network['routers']:
self.routers[router_name] = Router(router_name, self)
self.flows = {}
for flow_dict in network['flows']:
flow_name = flow_dict.keys()[0]
flow_desc = flow_dict[flow_name]
self.flows[flow_name] = Flow(
flow_name,
self,
self.collector,
self.hosts[flow_desc['source']] ,
self.hosts[flow_desc['destination']],
flow_desc['amount'],
flow_desc['start']
)
self.links = {}
host_prefixes = ['H', 'S', 'T']
for link_dict in network['links']:
link_name = link_dict.keys()[0]
link_desc = link_dict[link_name]
aname = link_desc['anode']
bname = link_desc['bnode']
ahost = self.hosts[aname] \
if aname[0] in host_prefixes else self.routers[aname]
bhost = self.hosts[bname] \
if bname[0] in host_prefixes else self.routers[bname]
link = Link(
link_name,
self,
self.collector,
link_desc['rate'] / 8.0,
link_desc['delay'],
link_desc['buffer'],
link_desc['buffer'],
ahost,
bhost
)
self.links[link_name] = link
ahost.set_link(link)
bhost.set_link(link)
self.mgr.add_event(0.0, link.calculate_throughput, [])
# Add initial events for each of the flows
for flow in self.flows.values():
self.mgr.add_event(flow.start_t, flow.send_packet, [])
if not configs['tcpreno']:
self.mgr.add_event(flow.start_t, flow.tcp_fast_update, [])
# Initialize the routing tables
for router in self.routers.values():
router.init_routing_table()
if len(self.routers) > 0:
self.mgr.add_event(0.0, self.routers['R1'].send_rout_packet, [])
self.mgr.add_event(self.mgr.t + configs['dynamic_routing_step'],
self.start_dynamic_routing, [])
def get_data(self):
"""Telling all of the objects to report their data."""
for flow in self.flows.values():
self.mgr.add_event(0.0, flow.report_rates, [0.1, 0, 0])
self.mgr.add_event(0.0, flow.report_link_rates, [0.2, {}])
for link in self.links.values():
self.mgr.add_event(0.0, link.report_occupancy, [0.1])
self.mgr.add_event(0.0, link.report_dropped, [0.1])
self.mgr.add_event(0.0, link.report_throughput, [0.5])
for host in self.hosts.values():
self.mgr.add_event(0.0, host.report_rates, [0.1, 0, 0])
def start_dynamic_routing(self):
# Initialize the routing tables and send out routing packets
for router in self.routers.values():
router.init_routing_table(temp_table=True)
router.start_dynamic_routing()
router.send_rout_packet(True)
self.mgr.add_event(self.mgr.t + self.configs['dynamic_routing_step'],
self.start_dynamic_routing, [])
def check_done_dynamic_routing(self):
time_step = 0.01
num_not_done = 0
for router in self.routers.values():
if not router.done_d_routing:
num_not_done += 1
if num_not_done == 0:
for router in self.routers.values():
router.finish_dynamic_routing()
for link in self.links.values():
link.time_through_link_update()
def run(self):
# Starting data collection
self.get_data()
while self.mgr.has_events():
(t, event, args) = self.mgr.pop_event()
self.mgr.t = t
event(*args)
# Check if all our flows have terminated
flows_not_done = len(self.flows)
for flow in self.flows.values():
if flow.last_acknowledged == flow.max_seq:
flows_not_done -= 1
if flows_not_done == 0:
break
print 'Done with all events at time: {}'.format(self.mgr.t)
def plot(self):
# Get only the file_name
file_name = self.file_path.split("/")[-1].rstrip(".yml")
# Print the plots to pdf
plot_dir = "plots/"
# Whether or not its reno or fast
if self.configs['tcpreno']:
mode = '_reno'
else:
mode = '_fast'
# Pdf page
pp = PdfPages(plot_dir + file_name + mode + '.pdf')
print 'Plotting to {}'.format(plot_dir + file_name + mode + '.pdf')
collecs = self.collector.collection.keys()
collecs.sort()
for g in collecs:
self.collector.graph_data(pp, g, g)
if file_name =='testcase0':
# The congestion window for the flow
self.collector.graph_data(pp, 'F1 Congestion Window', 'F1_cwnd')
elif file_name == 'testcase1':
# The congestion window for the flow
self.collector.graph_data(pp, 'F1 Congestion Window', 'F1_cwnd')
# The link rates of L1 and L2
self.collector.combine_graphs(
pp, 'Test Case 1 Link Rates',
['F1_L1_rate', 'F1_L2_rate'],
labels=['L1', 'L2']
)
# The link rates of L3 and L4
self.collector.combine_graphs(
pp,
'Test Case 1 Link Rates',
['F1_L3_rate', 'F1_L4_rate'],
labels=['L3', 'L4']
)
elif file_name == 'testcase2':
self.collector.combine_graphs(
pp,
'Test Case 2 Flow Congestion Windows',
['F1_cwnd', 'F2_cwnd', 'F3_cwnd'],
labels=['F1', 'F2', 'F3']
)
self.collector.combine_graphs(
pp,
'Test Case 2 Buffer Occupancy',
['L1_a_occupancy', 'L2_a_occupancy', 'L3_a_occupancy'],
labels=['L1a', 'L2a', 'L3a']
)
pp.close()
