def __init__(self, name, parent, parameters):
self.my_id = Machine.id_counter
Machine.id_counter += 1
super(Machine, self).__init__(name, parent, parameters)
# recovery generator for permanent machine failure
self.recovery_generator2 = None
# amount of time after which a machine failure is treated as permanent,
# and eager disk recovery is begun, if eager_recovery_enabled is True.
self.fail_timeout = -1
if self.fail_timeout == -1:
# Fraction of machine failures that are permanent.
Machine.fail_fraction = float(
parameters.get("fail_fraction", 0.008))
self.fail_timeout = float(parameters.get("fail_timeout", 0.25))
# If True, machine failure and recovery durations will be generated
# but ignored.
self.fast_forward = bool(parameters.get("fast_forward"))
self.eager_recovery_enabled = bool(
parameters.get("eager_recovery_enabled"))
self.fail_durations = []
conf = Configuration()
self.machine_repair_time = conf.node_repair_time
def run(self):
conf = Configuration(self.conf_path)
xml = XMLParser(conf)
if conf.hier:
self.distributer = HierSSSDistribute(xml)
else:
self.distributer = SSSDistribute(xml)
self.conf = self.distributer.returnConf()
self.event_handler = EventHandler
self.distributer.start()
events_handled = 0
events = EventQueue()
if self.conf.system_upgrade:
for info in self.conf.system_upgrade_infos:
if info[0] == 1:
upgrade_start_times = self.addSystemUpgrade(info, self.conf.total_time)
if info[-1] is not None:
self.addUpgradeCheckEvents(events, upgrade_start_times, info[-1])
if self.conf.correlated_failures:
for info in self.conf.correlated_failures_infos:
for i in xrange(10):
cf_info = deepcopy(list(info))
cf_info[0] += i * 8760
print "correlated_failures info:", cf_info
self.addCorrelatedFailures(cf_info)
if self.conf.system_scaling:
for info in self.conf.system_scaling_infos:
self.addSystemScaling(info)
info_logger.info("disk usage is: " + str(self.distributer.diskUsage()*100) + "%\n")
self.distributer.getRoot().printAll()
root = self.distributer.getRoot()
root.generateEvents(events, 0, self.conf.total_time, True)
for ts in self.conf.upgrade_ts:
full_system_check_event = Event(Event.EventType.UpgradeCheck, ts, root, 6)
events.addEvent(full_system_check_event)
if self.conf.event_file != None:
events_file = self.conf.event_file + '-' + self.ts
events.printAll(events_file, "Iteration number: "+str(self.iteration_times))
self.iteration_times += 1
handler = self.event_handler(self.distributer)
print "total slices:", handler.total_slices
e = events.removeFirst()
while e is not None:
handler.handleEvent(e, events)
e = events.removeFirst()
events_handled += 1
self.total_events_handled += events_handled
result = handler.end()
info_logger.info(result.toString())
return result
def __init__(self, name, parent, parameters):
super(Disk, self).__init__(name, parent, parameters)
conf = Configuration()
self.disk_capacity = conf.disk_capacity
self.disk_repair_time = conf.disk_repair_time
self.chunk_repair_time = conf.chunk_repair_time
self.slices_hit_by_LSE = []
self.latent_error_generator = None
self.scrub_generator = None
def __init__(self, conf_path):
self.iteration_times = 1
self.ts = strftime("%Y%m%d.%H.%M.%S")
self.total_events_handled = 0
self.conf = Configuration(conf_path)
xml = XMLParser(self.conf)
if self.conf.hier:
self.distributer = HierSSSDistribute(xml)
else:
self.distributer = SSSDistribute(xml)
def run(self):
conf = Configuration(self.conf_path)
xml = XMLParser(conf)
distributer_class = returnDistributer(conf.data_placement,
conf.hierarchical)
self.distributer = distributer_class(xml)
self.conf = self.distributer.returnConf()
if self.conf.rafi_recovery:
self.event_handler = RAFIEventHandler
else:
self.event_handler = EventHandler
self.distributer.start()
# self.distributer.printGroupsToFile()
info_logger.info("disk usage is: " +
str(self.distributer.diskUsage() * 100) + "%\n")
self.distributer.getRoot().printAll()
events_handled = 0
events = EventQueue()
root = self.distributer.getRoot()
root.generateEvents(events, 0, self.conf.total_time, True)
# if False:
if self.conf.event_file != None:
events_file = self.conf.event_file + '-' + self.ts
events.printAll(events_file,
"Iteration number: " + str(self.iteration_times))
self.iteration_times += 1
handler = self.event_handler(self.distributer)
print "total slices:", handler.total_slices
e = events.removeFirst()
while e is not None:
handler.handleEvent(e, events)
e = events.removeFirst()
events_handled += 1
self.total_events_handled += events_handled
result = handler.end()
info_logger.info(result.toString())
return result
def __init__(self, durations):
self.durations = durations
self.conf = Configuration()
self.drs_handler = self.conf.getDRSHandler()
self.isMDS = self.drs_handler.isMDS
ft = self.drs_handler.n - self.drs_handler.k
if self.isMDS:
self.ft = ft
else:
# We only consider LRC with l = 2, so we
# need to consider failures more than n-k-1
self.ft = ft - 1
self.concurrent_count = 0
self.lost_concurrent_count = 0
self.total_failure_slice_count = 0
def returnLayerArch(layer):
conf = Configuration()
dcs = conf.datacenters
racks = conf.racks
machines = conf.machines_per_rack
disks = conf.disks_per_machine
layer_tree = Tree("SYS")
for dc_id in xrange(dcs):
dc_node = layer_tree.addChild("DC" + str(dc_id))
for rack_id in xrange(racks):
rack_node = dc_node.addChild("R" + str(rack_id))
for machine_id in xrange(machines):
machine_node = rack_node.addChild("M" + str(machine_id))
# Haven't consider the medium is SSD.
for disk_id in xrange(disks):
machine_node.addChild("H" + str(disk_id))
return layer_tree
CopySet(Cidon et al. 2013)
"""
pass
class RandomDistributeSameRack(RandomDistribute):
"""
HDFS: three replicas, two of them on different machines of same rack, the third one on a
different rack
QFS: n blocks, several of them on same rack
"""
class RandomDistributeDRC(RandomDistribute):
"""
DRC: Double Regenerating Codes.(Hu et al. 2017)
"""
pass
if __name__ == "__main__":
conf = Configuration()
xml = XMLParser(conf)
sss = RandomDistributeSSS(xml)
sss.start()
sss.printTest()
sss.printToFile()
sss.systemScaling(1000, 0.1, 20000, 3, 9000, True)
sss.printTest()
sss.printToFile()
def returnLayers():
layers = []
conf = Configuration()
for i in xrange(1, conf.tier_num + 1):
layers.append(Layer(i))
return layers
def __init__(self, name, parent, parameters):
super(Disk, self).__init__(name, parent, parameters)
conf = Configuration()
self.disk_capacity = conf.disk_capacity
self.disk_repair_time = conf.disk_repair_time
self.slices_hit_by_LSE = []
machines_set = [[0 for i in xrange(len(machines[0]))]
for j in xrange(len(machines))]
for i, copy_set in enumerate(copy_sets):
format_output = "copy set " + str(i) + ": "
for machine in copy_set:
format_output += " " + machine.toString()
for j, rack_machines in enumerate(machines):
if machine in rack_machines:
machine_index = rack_machines.index(machine)
machines_set[j][machine_index] += 1
return copy_sets
if __name__ == "__main__":
conf = Configuration("/root/CR-SIM/conf/cr-sim.conf")
xml = XMLParser(conf)
distribute = COPYSETDistribute(xml)
machines = distribute.getAllMachines()
distribute.distributeSlices(distribute.getRoot(),
distribute.conf.total_slices)
distribute.printToFile()
"""
total_slices = 419431
distribute.distributeSlices(distribute.getRoot(), total_slices)
for i in xrange(total_slices):
format_output = "slice index " + str(i) + ": "
for disk in distribute.slice_locations[i]:
format_output += " " + disk.toString()
print format_output"""
def __init__(self, layer_id):
layer_path = CONF_PATH + os.sep + "layer_" + str(layer_id) + ".xml"
self.tree = ET.parse(layer_path)
self.root = self.tree.getroot()
self.conf = Configuration()