def run(self):
conf = Configuration(self.conf_path)
xml = XMLParser(conf)
if conf.hier:
self.distributer = HierSSSDistribute(xml)
else:
self.distributer = SSSDistribute(xml)
self.conf = self.distributer.returnConf()
self.event_handler = EventHandler
self.distributer.start()
events_handled = 0
events = EventQueue()
if self.conf.system_upgrade:
for info in self.conf.system_upgrade_infos:
if info[0] == 1:
upgrade_start_times = self.addSystemUpgrade(info, self.conf.total_time)
if info[-1] is not None:
self.addUpgradeCheckEvents(events, upgrade_start_times, info[-1])
if self.conf.correlated_failures:
for info in self.conf.correlated_failures_infos:
for i in xrange(10):
cf_info = deepcopy(list(info))
cf_info[0] += i * 8760
print "correlated_failures info:", cf_info
self.addCorrelatedFailures(cf_info)
if self.conf.system_scaling:
for info in self.conf.system_scaling_infos:
self.addSystemScaling(info)
info_logger.info("disk usage is: " + str(self.distributer.diskUsage()*100) + "%\n")
self.distributer.getRoot().printAll()
root = self.distributer.getRoot()
root.generateEvents(events, 0, self.conf.total_time, True)
for ts in self.conf.upgrade_ts:
full_system_check_event = Event(Event.EventType.UpgradeCheck, ts, root, 6)
events.addEvent(full_system_check_event)
if self.conf.event_file != None:
events_file = self.conf.event_file + '-' + self.ts
events.printAll(events_file, "Iteration number: "+str(self.iteration_times))
self.iteration_times += 1
handler = self.event_handler(self.distributer)
print "total slices:", handler.total_slices
e = events.removeFirst()
while e is not None:
handler.handleEvent(e, events)
e = events.removeFirst()
events_handled += 1
self.total_events_handled += events_handled
result = handler.end()
info_logger.info(result.toString())
return result
def __init__(self, conf_path):
self.iteration_times = 1
self.ts = strftime("%Y%m%d.%H.%M.%S")
self.total_events_handled = 0
self.conf = Configuration(conf_path)
xml = XMLParser(self.conf)
if self.conf.hier:
self.distributer = HierSSSDistribute(xml)
else:
self.distributer = SSSDistribute(xml)
def run(self):
conf = Configuration(self.conf_path)
xml = XMLParser(conf)
distributer_class = returnDistributer(conf.data_placement,
conf.hierarchical)
self.distributer = distributer_class(xml)
self.conf = self.distributer.returnConf()
if self.conf.rafi_recovery:
self.event_handler = RAFIEventHandler
else:
self.event_handler = EventHandler
self.distributer.start()
# self.distributer.printGroupsToFile()
info_logger.info("disk usage is: " +
str(self.distributer.diskUsage() * 100) + "%\n")
self.distributer.getRoot().printAll()
events_handled = 0
events = EventQueue()
root = self.distributer.getRoot()
root.generateEvents(events, 0, self.conf.total_time, True)
# if False:
if self.conf.event_file != None:
events_file = self.conf.event_file + '-' + self.ts
events.printAll(events_file,
"Iteration number: " + str(self.iteration_times))
self.iteration_times += 1
handler = self.event_handler(self.distributer)
print "total slices:", handler.total_slices
e = events.removeFirst()
while e is not None:
handler.handleEvent(e, events)
e = events.removeFirst()
events_handled += 1
self.total_events_handled += events_handled
result = handler.end()
info_logger.info(result.toString())
return result
CopySet(Cidon et al. 2013)
"""
pass
class RandomDistributeSameRack(RandomDistribute):
"""
HDFS: three replicas, two of them on different machines of same rack, the third one on a
different rack
QFS: n blocks, several of them on same rack
"""
class RandomDistributeDRC(RandomDistribute):
"""
DRC: Double Regenerating Codes.(Hu et al. 2017)
"""
pass
if __name__ == "__main__":
conf = Configuration()
xml = XMLParser(conf)
sss = RandomDistributeSSS(xml)
sss.start()
sss.printTest()
sss.printToFile()
sss.systemScaling(1000, 0.1, 20000, 3, 9000, True)
sss.printTest()
sss.printToFile()