def __init__(self, name, parent, parameters): self.my_id = Machine.id_counter Machine.id_counter += 1 super(Machine, self).__init__(name, parent, parameters) # recovery generator for permanent machine failure self.recovery_generator2 = None # amount of time after which a machine failure is treated as permanent, # and eager disk recovery is begun, if eager_recovery_enabled is True. self.fail_timeout = -1 if self.fail_timeout == -1: # Fraction of machine failures that are permanent. Machine.fail_fraction = float( parameters.get("fail_fraction", 0.008)) self.fail_timeout = float(parameters.get("fail_timeout", 0.25)) # If True, machine failure and recovery durations will be generated # but ignored. self.fast_forward = bool(parameters.get("fast_forward")) self.eager_recovery_enabled = bool( parameters.get("eager_recovery_enabled")) self.fail_durations = [] conf = Configuration() self.machine_repair_time = conf.node_repair_time
def run(self): conf = Configuration(self.conf_path) xml = XMLParser(conf) if conf.hier: self.distributer = HierSSSDistribute(xml) else: self.distributer = SSSDistribute(xml) self.conf = self.distributer.returnConf() self.event_handler = EventHandler self.distributer.start() events_handled = 0 events = EventQueue() if self.conf.system_upgrade: for info in self.conf.system_upgrade_infos: if info[0] == 1: upgrade_start_times = self.addSystemUpgrade(info, self.conf.total_time) if info[-1] is not None: self.addUpgradeCheckEvents(events, upgrade_start_times, info[-1]) if self.conf.correlated_failures: for info in self.conf.correlated_failures_infos: for i in xrange(10): cf_info = deepcopy(list(info)) cf_info[0] += i * 8760 print "correlated_failures info:", cf_info self.addCorrelatedFailures(cf_info) if self.conf.system_scaling: for info in self.conf.system_scaling_infos: self.addSystemScaling(info) info_logger.info("disk usage is: " + str(self.distributer.diskUsage()*100) + "%\n") self.distributer.getRoot().printAll() root = self.distributer.getRoot() root.generateEvents(events, 0, self.conf.total_time, True) for ts in self.conf.upgrade_ts: full_system_check_event = Event(Event.EventType.UpgradeCheck, ts, root, 6) events.addEvent(full_system_check_event) if self.conf.event_file != None: events_file = self.conf.event_file + '-' + self.ts events.printAll(events_file, "Iteration number: "+str(self.iteration_times)) self.iteration_times += 1 handler = self.event_handler(self.distributer) print "total slices:", handler.total_slices e = events.removeFirst() while e is not None: handler.handleEvent(e, events) e = events.removeFirst() events_handled += 1 self.total_events_handled += events_handled result = handler.end() info_logger.info(result.toString()) return result
def __init__(self, name, parent, parameters): super(Disk, self).__init__(name, parent, parameters) conf = Configuration() self.disk_capacity = conf.disk_capacity self.disk_repair_time = conf.disk_repair_time self.chunk_repair_time = conf.chunk_repair_time self.slices_hit_by_LSE = [] self.latent_error_generator = None self.scrub_generator = None
def __init__(self, conf_path): self.iteration_times = 1 self.ts = strftime("%Y%m%d.%H.%M.%S") self.total_events_handled = 0 self.conf = Configuration(conf_path) xml = XMLParser(self.conf) if self.conf.hier: self.distributer = HierSSSDistribute(xml) else: self.distributer = SSSDistribute(xml)
def run(self): conf = Configuration(self.conf_path) xml = XMLParser(conf) distributer_class = returnDistributer(conf.data_placement, conf.hierarchical) self.distributer = distributer_class(xml) self.conf = self.distributer.returnConf() if self.conf.rafi_recovery: self.event_handler = RAFIEventHandler else: self.event_handler = EventHandler self.distributer.start() # self.distributer.printGroupsToFile() info_logger.info("disk usage is: " + str(self.distributer.diskUsage() * 100) + "%\n") self.distributer.getRoot().printAll() events_handled = 0 events = EventQueue() root = self.distributer.getRoot() root.generateEvents(events, 0, self.conf.total_time, True) # if False: if self.conf.event_file != None: events_file = self.conf.event_file + '-' + self.ts events.printAll(events_file, "Iteration number: " + str(self.iteration_times)) self.iteration_times += 1 handler = self.event_handler(self.distributer) print "total slices:", handler.total_slices e = events.removeFirst() while e is not None: handler.handleEvent(e, events) e = events.removeFirst() events_handled += 1 self.total_events_handled += events_handled result = handler.end() info_logger.info(result.toString()) return result
def __init__(self, durations): self.durations = durations self.conf = Configuration() self.drs_handler = self.conf.getDRSHandler() self.isMDS = self.drs_handler.isMDS ft = self.drs_handler.n - self.drs_handler.k if self.isMDS: self.ft = ft else: # We only consider LRC with l = 2, so we # need to consider failures more than n-k-1 self.ft = ft - 1 self.concurrent_count = 0 self.lost_concurrent_count = 0 self.total_failure_slice_count = 0
def returnLayerArch(layer): conf = Configuration() dcs = conf.datacenters racks = conf.racks machines = conf.machines_per_rack disks = conf.disks_per_machine layer_tree = Tree("SYS") for dc_id in xrange(dcs): dc_node = layer_tree.addChild("DC" + str(dc_id)) for rack_id in xrange(racks): rack_node = dc_node.addChild("R" + str(rack_id)) for machine_id in xrange(machines): machine_node = rack_node.addChild("M" + str(machine_id)) # Haven't consider the medium is SSD. for disk_id in xrange(disks): machine_node.addChild("H" + str(disk_id)) return layer_tree
CopySet(Cidon et al. 2013) """ pass class RandomDistributeSameRack(RandomDistribute): """ HDFS: three replicas, two of them on different machines of same rack, the third one on a different rack QFS: n blocks, several of them on same rack """ class RandomDistributeDRC(RandomDistribute): """ DRC: Double Regenerating Codes.(Hu et al. 2017) """ pass if __name__ == "__main__": conf = Configuration() xml = XMLParser(conf) sss = RandomDistributeSSS(xml) sss.start() sss.printTest() sss.printToFile() sss.systemScaling(1000, 0.1, 20000, 3, 9000, True) sss.printTest() sss.printToFile()
def returnLayers(): layers = [] conf = Configuration() for i in xrange(1, conf.tier_num + 1): layers.append(Layer(i)) return layers
def __init__(self, name, parent, parameters): super(Disk, self).__init__(name, parent, parameters) conf = Configuration() self.disk_capacity = conf.disk_capacity self.disk_repair_time = conf.disk_repair_time self.slices_hit_by_LSE = []
machines_set = [[0 for i in xrange(len(machines[0]))] for j in xrange(len(machines))] for i, copy_set in enumerate(copy_sets): format_output = "copy set " + str(i) + ": " for machine in copy_set: format_output += " " + machine.toString() for j, rack_machines in enumerate(machines): if machine in rack_machines: machine_index = rack_machines.index(machine) machines_set[j][machine_index] += 1 return copy_sets if __name__ == "__main__": conf = Configuration("/root/CR-SIM/conf/cr-sim.conf") xml = XMLParser(conf) distribute = COPYSETDistribute(xml) machines = distribute.getAllMachines() distribute.distributeSlices(distribute.getRoot(), distribute.conf.total_slices) distribute.printToFile() """ total_slices = 419431 distribute.distributeSlices(distribute.getRoot(), total_slices) for i in xrange(total_slices): format_output = "slice index " + str(i) + ": " for disk in distribute.slice_locations[i]: format_output += " " + disk.toString() print format_output"""
def __init__(self, layer_id): layer_path = CONF_PATH + os.sep + "layer_" + str(layer_id) + ".xml" self.tree = ET.parse(layer_path) self.root = self.tree.getroot() self.conf = Configuration()