Example #1
0
class UnifBFBSimulation(Simulation):
    ##
    # __init__() from Simulation
    #

    ##
    # Initialize UnifBFBSimulation
    #
    def init(self):
        self.logger = logging.getLogger(__name__)
        # self.logger.setLevel(logging.ERROR)
        self.logger.setLevel(logging.INFO)
        # self.logger.setLevel(logging.DEBUG)
        self.logger.addHandler(console)
        self.logger.propagate = False

        # Failure biasing prob
        self.fb_prob = float(self.is_parms.fb_prob)
        # Arrival rate of homogeneous Poisson process, beta
        self.poisson_rate = float(self.is_parms.beta)
        # Likelihood ratio
        self.lr = float(1.)

        self.logger.debug(
            "UnifBFBSimulation init() - fb_prob = %.6f, poisson_rate = %.6f",
            self.fb_prob, self.poisson_rate)

    ##
    # Reset the simulator
    #
    def reset(self):
        # Reset clocks and state for each disk
        for disk in self.disks:
            disk.init_clock(0)
            disk.init_state()

        # Reset clocks and state for each node
        for node in self.nodes:
            node.init_clock(0)
            node.init_state()

        # Reset clocks and state for each rack
        for rack in self.racks:
            rack.init_state()

        # Reset system state
        self.state = State(self.num_disks, self.num_nodes)

        # Rest repair queue
        self.repair_queue = []

        # Regenerate new placement
        self.placement = Placement(self.num_racks, self.nodes_per_rack,
                                   self.disks_per_node, self.capacity_per_disk,
                                   self.num_stripes, self.chunk_size,
                                   self.code_type, self.n, self.k,
                                   self.place_type, self.chunk_rack_config,
                                   self.l)
        # Reset LR
        self.lr = float(1.)

        self.total_failure_rate = 0.
        self.total_failrue_rate_cnt = 0
        self.total_repair_rate = 0.
        self.total_repair_rate_cnt = 0

    ##
    # Get failure rate
    #
    def get_failure_rate(self):
        fail_rate = float(0)

        for disk in self.disks:
            fail_rate += disk.curr_disk_fail_rate()

        for node in self.nodes:
            fail_rate += node.curr_node_fail_rate()

        # self.logger.debug("get_failure_rate(): fail_rate = %.6f", fail_rate)
        # print("get_failure_rate(): fail_rate = %.6f" % fail_rate)
        return fail_rate

    ##
    # Get the probability of node failure
    # To decide whether a failure event is node failure or disk failure
    #
    def get_node_failure_prob(self):
        comp_fail_rate = float(0)
        node_fail_rate = float(0)
        for disk in self.disks:
            comp_fail_rate += disk.curr_disk_fail_rate()
        for node in self.nodes:
            node_fail_rate += node.curr_node_fail_rate()

        return node_fail_rate / (node_fail_rate + comp_fail_rate)

    ##
    # Calculate the repair time for a failed component
    # The repair time = the amount of cross_rack data to download / cross_rack bandwidth
    #
    def get_disk_repair_duration(self, disk_idx):
        if not self.use_network:
            # get the repair time from a pre-defined repair distribution
            return self.disk_repair_dists.draw()
        else:
            # repair time = cross-rack repair traffic / available cross-rack bandwidth
            rack_id = disk_idx / (self.nodes_per_rack * self.disks_per_node)
            cross_rack_download = 0
            stripes_to_repair = self.placement.get_stripes_to_repair(disk_idx)
            # self.num_stripes_repaired += len(stripes_to_repair)
            # stripes_to_delay = []

            # print("len(stripes_to_repair) = %d" % len(stripes_to_repair))
            # for each stripe to repair
            for stripe_id in stripes_to_repair:
                num_failed_chunk = 0
                num_alive_chunk_same_rack = 0
                idx = 0
                fail_idx = 0
                alive_chunk_same_rack = []

                # check the status of each chunk in the stripe
                for disk_id in self.placement.get_stripe_location(stripe_id):

                    # for RS, DRC
                    if self.placement.code_type != Placement.CODE_TYPE_LRC:
                        if self.disks[disk_id].get_curr_state(
                        ) == Disk.STATE_CRASHED:
                            num_failed_chunk += 1
                        elif (disk_id / (self.nodes_per_rack *
                                         self.disks_per_node)) == rack_id:
                            num_alive_chunk_same_rack += 1
                    # for LRC
                    else:
                        if self.disks[disk_id].get_curr_state(
                        ) == Disk.STATE_CRASHED:
                            num_failed_chunk += 1
                            if disk_idx == disk_id:
                                fail_idx = idx
                        elif (disk_id / (self.nodes_per_rack *
                                         self.disks_per_node)) == rack_id:
                            num_alive_chunk_same_rack += 1
                            alive_chunk_same_rack.append(idx)
                        idx += 1

                # # this is a single-chunk repair
                # if num_failed_chunk == 1:
                #     self.num_stripes_repaired_single_chunk += 1

                # RS
                if self.placement.code_type == Placement.CODE_TYPE_RS:
                    if num_alive_chunk_same_rack < self.k:
                        cross_rack_download += (self.k -
                                                num_alive_chunk_same_rack)
                # LRC
                elif self.placement.code_type == Placement.CODE_TYPE_LRC:
                    if num_failed_chunk == 1:
                        # global parity
                        if fail_idx in self.placement.lrc_global_parity:
                            if num_alive_chunk_same_rack < self.k:
                                cross_rack_download += self.k - num_alive_chunk_same_rack
                        # data chunk or local parity
                        else:
                            # find which group that the failed chunk is in
                            fail_gid = 0
                            for gid in xrange(self.l):
                                if fail_idx in self.placement.lrc_data_group[gid] or \
                                        fail_idx == self.placement.lrc_local_parity[gid]:
                                    fail_gid = gid
                                    break
                            # find how many chunk in the same rack can be used for repair
                            num_alive_chunk_same_rack = 0
                            for each in alive_chunk_same_rack:
                                if each in self.placement.lrc_data_group[fail_gid] or \
                                        each == self.placement.lrc_data_group[fail_gid]:
                                    num_alive_chunk_same_rack += 1
                            if num_alive_chunk_same_rack < self.k / self.l:
                                cross_rack_download += self.k / self.l - num_alive_chunk_same_rack
                    else:
                        if num_alive_chunk_same_rack < self.k:
                            cross_rack_download += (self.k -
                                                    num_alive_chunk_same_rack)
                # DRC
                elif self.placement.code_type == Placement.CODE_TYPE_DRC:
                    if num_failed_chunk == 1:
                        if self.k == 5 and self.n == 9:
                            cross_rack_download += 1.0
                        elif self.k == 6 and self.n == 9:
                            cross_rack_download += 2.0
                        else:
                            print "Only support DRC - (9,6,3), (9,5,3)"
                    else:
                        if num_alive_chunk_same_rack < self.k:
                            cross_rack_download += (self.k -
                                                    num_alive_chunk_same_rack)
                else:
                    print "Not correct code type in set_disk_repair()!"

            repair_duration = cross_rack_download * self.chunk_size / \
                              float(self.network.get_avail_cross_rack_repair_bwth()) # seconds
            # print "repair_time = %.1f" % (repair_duration / 3600.)
            # print("repair_duration = %.10f, cross_rack_download=%d" % \
            #        (repair_duration / 3600., cross_rack_download))

            if repair_duration != 0:
                self.total_repair_rate += 3600. / repair_duration
                self.total_repair_rate_cnt += 1

            return repair_duration / 3600.  # hours

    def get_earliest_repair_time(self, curr_time):
        earliest_repair_time = curr_time
        if len(self.repair_queue) > 0:
            for repair_event in self.repair_queue:
                repair_event_time = repair_event[0]
                if repair_event_time > earliest_repair_time:
                    earliest_repair_time = repair_event_time

        return earliest_repair_time

    ##
    # Set next repair time for disk indexed with disk_index
    #
    def set_disk_repair(self, disk_idx, curr_time):
        heappush(self.repair_queue, (self.get_disk_repair_duration(disk_idx) +
                                     self.get_earliest_repair_time(curr_time),
                                     Disk.EVENT_DISK_REPAIR, disk_idx))

    ##
    # Set new node repair time for node node_idx
    #
    def set_node_repair(self, node_idx, curr_time):
        node_repair_duration = 0
        # Get the repair duration of each disk on this node
        for i in xrange(self.disks_per_node):
            disk_idx = self.disks_per_node * node_idx + i
            node_repair_duration += self.get_disk_repair_duration(disk_idx)

        heappush(
            self.repair_queue,
            (node_repair_duration + self.get_earliest_repair_time(curr_time),
             Node.EVENT_NODE_REPAIR, node_idx))

    ##
    # Get the next event in UnifBFBSimulation
    #
    def get_next_event(self, curr_time):
        # Update clock for each disk
        for disk in self.disks:
            disk.update_clock(curr_time)

        # Update clock for each node
        for node in self.nodes:
            node.update_clock(curr_time)

        # If not in a failed state, then draw for next failure
        if self.state.get_sys_state() == self.state.CURR_STATE_OK:
            failure_queue = []

            for each_disk in range(self.num_disks):
                failure_queue.append(
                    (self.disks[each_disk].disk_fail_distr.
                     draw_inverse_transform(self.disks[each_disk].read_clock())
                     + curr_time, Disk.EVENT_DISK_FAIL, each_disk))

            for each_node in range(self.num_nodes):
                failure_queue.append(
                    (self.nodes[each_node].node_fail_distr.
                     draw_inverse_transform(self.nodes[each_node].read_clock())
                     + curr_time, Node.EVENT_NODE_FAIL, each_node))

            heapify(failure_queue)
            (next_event_time, next_event_type,
             next_event_subsystem) = heappop(failure_queue)

            if next_event_type == Disk.EVENT_DISK_FAIL:
                self.disks[next_event_subsystem].fail_disk(next_event_time)
                self.set_disk_repair(next_event_subsystem, next_event_time)
            elif next_event_type == Node.EVENT_NODE_FAIL:
                self.nodes[next_event_subsystem].fail_node(next_event_time)
                for each_disk_on_this_node in range(
                        next_event_subsystem * self.disks_per_node,
                    (next_event_subsystem + 1) * self.disks_per_node):
                    self.disks[each_disk_on_this_node].fail_disk(
                        next_event_time)
                self.set_node_repair(next_event_subsystem, next_event_time)
            else:
                self.logger.error(
                    "UnifBFBSimulation - get_next_event(): wrong next_event_type!"
                )

            return (next_event_time, next_event_type, next_event_subsystem)

        elif self.state.get_sys_state() == self.state.CURR_STATE_DEGRADED:
            if not self.repair_queue:
                self.logger.error(
                    "UnifBFBSimulation - get_next_event(): repair_queue is empty!"
                )
                sys.exit(2)

            (repair_time, repair_event, subsystem_idx) = self.repair_queue[0]
            next_event_time = nprandom.exponential(
                self.poisson_rate) + curr_time

            if repair_time <= next_event_time:
                heappop(self.repair_queue)
                if repair_event == Disk.EVENT_DISK_REPAIR:
                    self.disks[subsystem_idx].repair_disk(repair_time)
                    return (repair_time, Disk.EVENT_DISK_REPAIR, subsystem_idx)
                elif repair_event == Node.EVENT_NODE_REPAIR:
                    self.nodes[subsystem_idx].repair_node()
                    for i in range(self.disks_per_node):
                        disk_idx = subsystem_idx * self.disks_per_node + i
                        self.disks[disk_idx].repair_disk(repair_time)
                    return (repair_time, Node.EVENT_NODE_REPAIR, subsystem_idx)
                else:
                    self.logger.error(
                        "UnifBFBSimulation - get_next_event(): wrong repair_event!"
                    )

            for disk in self.disks:
                disk.update_clock(next_event_time)
            for node in self.nodes:
                node.update_clock(next_event_time)

            self.total_failure_rate += self.get_failure_rate()
            self.total_failrue_rate_cnt += 1

            draw = nprandom.uniform()
            # Determine whether it is a "real" event or "pseudo" event
            if draw > self.fb_prob:
                # It is a pseudo event
                old_lr = self.lr
                self.lr *= (1. - self.get_failure_rate() /
                            self.poisson_rate) / (1. - self.fb_prob)
                self.logger.debug(
                    "get_next_event(): pseudo event - old_lr = %.10f, update, lr = %.10f",
                    old_lr, self.lr)
                # Return nothing because we are staying in the current state
                return (next_event_time, None, None)

            else:
                # Randomly fail a disk or node
                # prob_node_failure = self.get_node_failure_prob()
                if nprandom.uniform() > self.get_node_failure_prob():
                    # disk failure
                    avail_disks = self.state.get_avail_disks()
                    fail_disk_idx = avail_disks[random.randint(
                        0,
                        len(avail_disks) - 1)]

                    old_lr = self.lr
                    # self.lr *= (self.disks[fail_disk_idx].curr_disk_fail_rate() / self.poisson_rate) \
                    #            / (self.fb_prob * (1 - prob_node_failure) / len(avail_disks))
                    # The above equation equals to the following
                    self.lr *= (self.get_failure_rate() /
                                self.poisson_rate) / self.fb_prob
                    self.logger.debug(
                        "get_next_event(): disk failure event, lr = %.10f, update, lr = %.10f",
                        old_lr, self.lr)

                    self.disks[fail_disk_idx].fail_disk(next_event_time)
                    self.set_disk_repair(fail_disk_idx, next_event_time)

                    return (next_event_time, Disk.EVENT_DISK_FAIL,
                            fail_disk_idx)

                else:
                    avail_nodes = self.state.get_avail_nodes()
                    fail_node_idx = avail_nodes[random.randint(
                        0,
                        len(avail_nodes) - 1)]

                    old_lr = self.lr
                    # self.lr *= (self.nodes[fail_node_idx].curr_node_fail_rate() / self.poisson_rate) \
                    #            / (self.fb_prob * prob_node_failure / len(avail_nodes))
                    # The above equation equals to the following
                    self.lr *= (self.get_failure_rate() /
                                self.poisson_rate) / self.fb_prob
                    self.logger.debug(
                        "get_next_event(): node failure event - old_lr = %.10f, update, lr = %.10f",
                        old_lr, self.lr)

                    # Update internal node state
                    self.nodes[fail_node_idx].fail_node(next_event_time)
                    for each_disk_on_failed_node in range(
                            fail_node_idx * self.disks_per_node,
                        (fail_node_idx + 1) * self.disks_per_node):
                        self.disks[each_disk_on_failed_node].fail_disk(
                            next_event_time)

                    # Schedule repair for the failed node
                    self.set_node_repair(fail_node_idx, next_event_time)

                    return (next_event_time, Node.EVENT_NODE_FAIL,
                            fail_node_idx)

    ##
    # Run an iteration in UnifBFBSimulation
    #
    def run_iteration(self, ite=0):
        self.reset()
        curr_time = 0
        self.logger.info(
            "UnifBFBSimulator: begin an iteration %d, num_failed_disks = %d, "
            "avail_cross_rack_bwth = %d" %
            (ite, len(self.state.get_failed_disks()),
             self.network.get_avail_cross_rack_repair_bwth()))

        while True:
            (event_time, event_type,
             subsystem_idx) = self.get_next_event(curr_time)
            curr_time = event_time

            if event_time > self.mission_time:
                break

            if event_type != None:
                self.logger.debug(
                    "Time: %.3f, event = %s, subsystem = %d, "
                    "number_failed_disks = %d, number_failed_nodes = %d" %
                    (event_time, event_type, subsystem_idx,
                     self.state.get_num_failed_disks(),
                     self.state.get_num_failed_nodes()))

                if not self.state.update_state_unifbfb(event_type,
                                                       subsystem_idx):
                    self.logger.error('Update_state_unifbfb failed!')

            # Check durability when disk failure or node failure happens
            if event_type == Disk.EVENT_DISK_FAIL or event_type == Node.EVENT_NODE_FAIL:
                failed_disks = self.state.get_failed_disks()
                if self.placement.check_data_loss(failed_disks):
                    self.logger.debug(
                        "===== END of one iteration, self.lr = %.10f",
                        min(self.lr, 1))
                    (num_failed_stripes, num_lost_chunks
                     ) = self.placement.get_num_failed_status(failed_disks)
                    self.logger.info("avg_failure_rate = %.6f" %
                                     (self.total_failure_rate /
                                      self.total_failrue_rate_cnt))
                    self.logger.info(
                        "avg_repair_rate = %.6f" %
                        (self.total_repair_rate / self.total_repair_rate_cnt))
                    return (min(self.lr, 1), "(%d, %d, 0, 0)" %
                            (num_failed_stripes, num_lost_chunks))

        # No data loss
        self.logger.debug(
            "END of one iteration, self.lr = 0 because no data loss")
        return (0, "(0, 0, 0, 0)")
Example #2
0
class RegularSimulation(Simulation):
    ##
    # __init__() from Simulation
    #

    ##
    # Initialize the simulation
    #
    def init(self):
        # Initialize the state of the system
        self.state = State(self.num_disks)

        # Employ priority queue to keep all the failures and repairs
        # The element in the queue is (event_time, event_type, device_id)
        self.events_queue = []

        # Keep failed disks awaiting repair
        self.wait_repair_queue = []

        # Keep delayed stripes due to unavailable nodes
        # Key is the disk_idx delayed, value is the list of delayed stripes
        self.delayed_repair_dict = dict()

        self.enable_transient_failures = False

        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.ERROR)
        # self.logger.setLevel(logging.INFO)
        self.logger.addHandler(console)
        self.logger.propagate = False

    ##
    # Reset the simulation
    #
    def reset(self, ite=0):
        # Generate node transient and permanent failure events from trace
        if self.use_trace:
            for i in xrange(self.num_nodes):
                self.nodes[i] = Node(None, None, None,
                                     Trace(self.trace_id, i, 'p'),
                                     Trace(self.trace_id, i, 't'),
                                     Trace(self.trace_id, i, 'r'))

        self.state = State(self.num_disks)

        for disk in self.disks:
            disk.init_clock(0)
            disk.init_state()
        for node in self.nodes:
            node.init_state()
        for rack in self.racks:
            rack.init_state()

        self.events_queue = []
        self.wait_repair_queue = []
        self.delayed_repair_dict = dict()

        # generate disk failures and put them into events_queue
        for disk_id in xrange(len(self.disks)):
            disk_fail_time = self.disk_fail_dists.draw()
            if disk_fail_time <= self.mission_time:
                self.events_queue.append(
                    (disk_fail_time, Disk.EVENT_DISK_FAIL, disk_id))
        # generate node failures and push them into events_queue
        for node_id in xrange(self.num_nodes):
            if not self.use_trace:
                self.events_queue.append((self.node_fail_dists.draw(),
                                          Node.EVENT_NODE_FAIL, node_id))
                if self.enable_transient_failures:
                    self.events_queue.append(
                        (self.node_transient_fail_dists.draw(),
                         Node.EVENT_NODE_TRANSIENT_FAIL, node_id))
            else:
                for node_failure_time in self.nodes[
                        node_id].node_fail_trace.get_trace_ls():
                    # push node failure event to event_queue
                    self.events_queue.append(
                        (node_failure_time, Node.EVENT_NODE_FAIL, node_id))
                node_transient_failure_ls = self.nodes[
                    node_id].node_transient_fail_trace.get_trace_ls()
                node_transient_repair_ls = self.nodes[
                    node_id].node_transient_repair_trace.get_trace_ls()
                for ls_idx in xrange(len(node_transient_failure_ls)):
                    node_transient_failure_time = node_transient_failure_ls[
                        ls_idx]
                    node_transient_repair_time = node_transient_repair_ls[
                        ls_idx]
                    self.events_queue.append(
                        (node_transient_failure_time,
                         Node.EVENT_NODE_TRANSIENT_FAIL, node_id))
                    self.events_queue.append(
                        (node_transient_failure_time +
                         node_transient_repair_time,
                         Node.EVENT_NODE_TRANSIENT_REPAIR, node_id))

        # generate rack failures and push them into events_queue
        if not self.use_power_outage and self.enable_transient_failures:
            for rack_id in xrange(len(self.racks)):
                self.events_queue.append((self.rack_fail_dists.draw(),
                                          Rack.EVENT_RACK_FAIL, rack_id))

        # correlated failures caused by power outage
        if (not self.use_trace) and self.use_power_outage:
            for rack_id in xrange(self.num_racks):
                occur_time = float(0) + self.power_outage_dist.draw()
                while occur_time < self.mission_time:
                    self.events_queue.append(
                        (occur_time, Rack.EVENT_RACK_FAIL, rack_id))
                    occur_time += random.expovariate(
                        (1 / float(self.power_outage_duration)))
                    self.events_queue.append(
                        (occur_time, Rack.EVENT_RACK_REPAIR, rack_id))
                    for i in xrange(self.nodes_per_rack):
                        # draw a bernoulli distribution
                        if nprandom.binomial(1, 0.01):
                            self.events_queue.append(
                                (occur_time, Node.EVENT_NODE_FAIL,
                                 (self.nodes_per_rack * rack_id + i)))
                    occur_time += self.power_outage_dist.draw()

        heapify(self.events_queue)
        self.placement = Placement(self.num_racks, self.nodes_per_rack,
                                   self.disks_per_node, self.capacity_per_disk,
                                   self.num_stripes, self.chunk_size,
                                   self.code_type, self.n, self.k,
                                   self.place_type, self.chunk_rack_config,
                                   self.l)

        self.network = Network(self.num_racks, self.nodes_per_rack,
                               self.network_setting)

        self.num_stripes_repaired = 0
        self.num_stripes_repaired_single_chunk = 0
        self.num_stripes_delayed = 0

    ##
    # Generate permanent disk failure event
    #
    def set_disk_fail(self, disk_idx, curr_time):
        heappush(self.events_queue, (self.disk_fail_dists.draw() + curr_time,
                                     Disk.EVENT_DISK_FAIL, disk_idx))

    ##
    # Generate repair event for permanent disk failure
    #
    def set_disk_repair(self, disk_idx, curr_time):
        if not self.use_network:
            # get the repair time from a pre-defined repair distribution
            heappush(self.events_queue,
                     (self.disk_repair_dists.draw() + curr_time,
                      Disk.EVENT_DISK_REPAIR, disk_idx))
        else:
            # repair time = cross-rack repair traffic / available cross-rack bandwidth
            rack_id = disk_idx / (self.nodes_per_rack * self.disks_per_node)

            # If there is no available bandwidth or the rack is under transient failure
            if self.network.get_avail_cross_rack_repair_bwth() == 0 or \
                self.racks[rack_id].get_curr_state() != Rack.STATE_RACK_NORMAL:
                heappush(self.wait_repair_queue, (curr_time, disk_idx))
            else:
                cross_rack_download = 0
                stripes_to_repair = self.placement.get_stripes_to_repair(
                    disk_idx)
                self.num_stripes_repaired += len(stripes_to_repair)
                stripes_to_delay = []

                # for each stripe to repair
                for stripe_id in stripes_to_repair:
                    num_failed_chunk = 0
                    num_alive_chunk_same_rack = 0
                    num_unavail_chunk = 0
                    idx = 0
                    fail_idx = 0
                    alive_chunk_same_rack = []

                    # check the status of each chunk in the stripe
                    for disk_id in self.placement.get_stripe_location(
                            stripe_id):
                        # get the total number of unavailable chunk (due to permanent/transient failures) in this stripe
                        if self.disks[disk_id].state != Disk.STATE_NORMAL:
                            num_unavail_chunk += 1

                        # for RS, DRC
                        if self.placement.code_type != Placement.CODE_TYPE_LRC:
                            if self.disks[disk_id].get_curr_state(
                            ) == Disk.STATE_CRASHED:
                                num_failed_chunk += 1
                            elif (disk_id / (self.nodes_per_rack *
                                             self.disks_per_node)) == rack_id:
                                num_alive_chunk_same_rack += 1
                        # for LRC
                        else:
                            if self.disks[disk_id].get_curr_state(
                            ) == Disk.STATE_CRASHED:
                                num_failed_chunk += 1
                                if disk_idx == disk_id:
                                    fail_idx = idx
                            elif (disk_id / (self.nodes_per_rack *
                                             self.disks_per_node)) == rack_id:
                                num_alive_chunk_same_rack += 1
                                alive_chunk_same_rack.append(idx)
                            idx += 1

                    # this is a single-chunk repair
                    if num_failed_chunk == 1:
                        self.num_stripes_repaired_single_chunk += 1
                    # the repair for this stripe is delayed
                    if num_unavail_chunk > (self.n - self.k):
                        stripes_to_delay.append(stripe_id)

                    # RS
                    if self.placement.code_type == Placement.CODE_TYPE_RS:
                        if num_alive_chunk_same_rack < self.k:
                            cross_rack_download += (self.k -
                                                    num_alive_chunk_same_rack)
                    # LRC
                    elif self.placement.code_type == Placement.CODE_TYPE_LRC:
                        if num_failed_chunk == 1:
                            # global parity
                            if fail_idx in self.placement.lrc_global_parity:
                                if num_alive_chunk_same_rack < self.k:
                                    cross_rack_download += self.k - num_alive_chunk_same_rack
                            # data chunk or local parity
                            else:
                                # find which group that the failed chunk is in
                                fail_gid = 0
                                for gid in xrange(self.l):
                                    if fail_idx in self.placement.lrc_data_group[gid] or \
                                        fail_idx == self.placement.lrc_local_parity[gid]:
                                        fail_gid = gid
                                        break
                                # find how many chunk in the same rack can be used for repair
                                num_alive_chunk_same_rack = 0
                                for each in alive_chunk_same_rack:
                                    if each in self.placement.lrc_data_group[fail_gid] or \
                                        each == self.placement.lrc_data_group[fail_gid]:
                                        num_alive_chunk_same_rack += 1
                                if num_alive_chunk_same_rack < self.k / self.l:
                                    cross_rack_download += self.k / self.l - num_alive_chunk_same_rack
                        else:
                            if num_alive_chunk_same_rack < self.k:
                                cross_rack_download += (
                                    self.k - num_alive_chunk_same_rack)
                    # DRC
                    elif self.placement.code_type == Placement.CODE_TYPE_DRC:
                        if num_failed_chunk == 1:
                            if self.k == 5 and self.n == 9:
                                cross_rack_download += 1.0
                            elif self.k == 6 and self.n == 9:
                                cross_rack_download += 2.0
                            else:
                                print "Only support DRC - (9,6,3), (9,5,3)"
                        else:
                            if num_alive_chunk_same_rack < self.k:
                                cross_rack_download += (
                                    self.k - num_alive_chunk_same_rack)
                    else:
                        print "Not correct code type in set_disk_repair()!"

                repair_bwth = self.network.get_avail_cross_rack_repair_bwth()
                self.network.update_avail_cross_rack_repair_bwth(0)
                repair_time = cross_rack_download * self.chunk_size / float(
                    repair_bwth)  # seconds
                repair_time /= float(3600)  # hours

                if len(stripes_to_delay) != 0:
                    self.num_stripes_delayed += len(stripes_to_delay)
                    self.delayed_repair_dict[disk_idx] = stripes_to_delay

                self.logger.debug("repair_time = %d, repair_bwth = %d" %
                                  (repair_time, repair_bwth))
                heappush(self.events_queue,
                         (repair_time + curr_time, Disk.EVENT_DISK_REPAIR,
                          disk_idx, repair_bwth))

    ##
    # Generate permanent node failure event
    #
    def set_node_fail(self, node_idx, curr_time):
        heappush(self.events_queue, (self.node_fail_dists.draw() + curr_time,
                                     Node.EVENT_NODE_FAIL, node_idx))

    ##
    # Generate repair event for permanent node failure
    # The repair for the failed node is conducted by the repair for the failed disks on that node
    #
    def set_node_repair(self, node_idx, curr_time):
        for i in xrange(self.disks_per_node):
            disk_idx = node_idx * self.disks_per_node + i
            self.set_disk_repair(disk_idx, curr_time)

    ##
    # Generate transient node failure event
    #
    def set_node_transient_fail(self, node_idx, curr_time):
        heappush(self.events_queue,
                 (self.nodes[node_idx].node_transient_fail_distr.draw() +
                  curr_time, Node.EVENT_NODE_TRANSIENT_FAIL, node_idx))

    ##
    # Generate repair event for transient node failure
    #
    def set_node_transient_repair(self, node_idx, curr_time):
        heappush(self.events_queue,
                 (self.nodes[node_idx].node_transient_repair_distr.draw() +
                  curr_time, Node.EVENT_NODE_TRANSIENT_REPAIR, node_idx))

    ##
    # Generate transient rack failure
    #
    def set_rack_fail(self, rack_idx, curr_time):
        heappush(self.events_queue, (self.rack_fail_dists.draw() + curr_time,
                                     Rack.EVENT_RACK_FAIL, rack_idx))

    ##
    # Generate repair for transient rack failure
    #
    def set_rack_repair(self, rack_idx, curr_time):
        heappush(self.events_queue, (self.rack_repair_dists.draw() + curr_time,
                                     Rack.EVENT_RACK_REPAIR, rack_idx))

    ##
    # Get the next event from the event queue
    #
    def get_next_event(self, curr_time):
        self.logger.debug(
            "len(delayed_repair_dict) = %d, len(wait_repair_queue) = %d" %
            (len(self.delayed_repair_dict), len(self.wait_repair_queue)))
        # If there are some stripes delayed
        if len(self.delayed_repair_dict) != 0:
            items_to_remove = []  # keep the key of the items to remove
            for key in self.delayed_repair_dict:
                tmp_dict_value = []
                for stripe_id in self.delayed_repair_dict[key]:
                    repair_delay = False
                    num_unavail_chunk = 0
                    for disk_idx in self.placement.get_stripe_location(
                            stripe_id):
                        if self.disks[disk_idx].state != Disk.STATE_NORMAL:
                            num_unavail_chunk += 1
                        if num_unavail_chunk > (self.n - self.k):
                            repair_delay = True
                            break
                    if repair_delay:  # stripe whose repair is delayed
                        tmp_dict_value.append(stripe_id)
                if len(tmp_dict_value) == 0:
                    items_to_remove.append(key)
                else:
                    self.delayed_repair_dict[key] = tmp_dict_value
            for key in items_to_remove:
                self.delayed_repair_dict.pop(key)

        # If there are some failed disks awaiting repair
        if len(self.wait_repair_queue) != 0:
            disk_id = self.wait_repair_queue[0][1]
            rack_id = disk_id / (self.nodes_per_rack * self.disks_per_node)
            if self.use_network and self.network.get_avail_cross_rack_repair_bwth() != 0 and \
                self.network.get_avail_intra_rack_repair_bwth(rack_id) != 0 and \
                self.racks[rack_id].get_curr_state() == Rack.STATE_RACK_NORMAL:
                heappop(self.wait_repair_queue)
                self.set_disk_repair(disk_id, curr_time)

        next_event = heappop(self.events_queue)
        next_event_time = next_event[0]
        next_event_type = next_event[1]
        if next_event_time > self.mission_time:
            return (next_event_time, None, None)

        device_idx_set = []
        device_idx_set.append(next_event[2])
        repair_bwth_set = []
        # If use network bandwidth to calculate repair_time
        if self.use_network and next_event_type == Disk.EVENT_DISK_REPAIR:
            repair_bwth_set.append(next_event[3])

        # Gather the events with the same occurring time and event type
        while self.events_queue[0][0] == next_event_time and self.events_queue[
                0][1] == next_event_type:
            next_event = heappop(self.events_queue)
            device_idx_set.append(next_event[2])
            if self.use_network and next_event_type == Disk.EVENT_DISK_REPAIR:
                repair_bwth_set.append(next_event[3])

        # disk permanent failure
        if next_event_type == Disk.EVENT_DISK_FAIL:
            fail_time = next_event_time
            for device_idx in device_idx_set:
                # avoid the case that this disk is under repair
                if self.disks[device_idx].get_curr_state(
                ) != Disk.STATE_CRASHED:
                    if self.delayed_repair_dict.has_key(device_idx):
                        self.delayed_repair_dict.pop(device_idx)
                    # update the state of the disk
                    self.disks[device_idx].fail_disk(fail_time)
                    # generate the repair event
                    self.set_disk_repair(device_idx, fail_time)
            return (fail_time, Disk.EVENT_DISK_FAIL, device_idx_set)

        # node permanent failure
        elif next_event_type == Node.EVENT_NODE_FAIL:
            failed_disks_set = set([])
            fail_time = next_event_time
            for device_idx in device_idx_set:
                # avoid the case that the node is under repair
                if self.nodes[device_idx].get_curr_state(
                ) != Node.STATE_NODE_CRASHED:
                    # update the state of node
                    self.nodes[device_idx].fail_node(fail_time)
                    for i in xrange(self.disks_per_node):
                        disk_idx = device_idx * self.disks_per_node + i
                        failed_disks_set.add(disk_idx)
                        # avoid the case that the disk is under repair
                        if self.disks[disk_idx].get_curr_state(
                        ) != Disk.STATE_CRASHED:
                            if self.delayed_repair_dict.has_key(device_idx):
                                self.delayed_repair_dict.pop(device_idx)
                            # update the state of the disk
                            self.disks[disk_idx].fail_disk(fail_time)
                            # generate the repair event
                            self.set_disk_repair(disk_idx, fail_time)
            return (fail_time, Node.EVENT_NODE_FAIL, failed_disks_set)

        # node transient failure
        elif next_event_type == Node.EVENT_NODE_TRANSIENT_FAIL:
            fail_time = next_event_time
            for device_idx in device_idx_set:
                if self.nodes[device_idx].get_curr_state(
                ) == Node.STATE_NODE_NORMAL:
                    # update the state of node
                    self.nodes[device_idx].offline_node()
                    for i in xrange(self.disks_per_node):
                        disk_id = device_idx * self.disks_per_node + i
                        if self.disks[disk_id].get_curr_state(
                        ) == Disk.STATE_NORMAL:
                            # update the state of disk
                            self.disks[disk_id].offline_disk(fail_time)
                # generate the repair event
                if not self.use_trace:
                    self.set_node_transient_repair(device_idx, fail_time)

            return (fail_time, Node.EVENT_NODE_TRANSIENT_FAIL, None)

        # transient rack failure
        elif next_event_type == Rack.EVENT_RACK_FAIL:
            fail_time = next_event_time
            for device_idx in device_idx_set:
                if self.racks[device_idx].get_curr_state(
                ) == Rack.STATE_RACK_NORMAL:
                    # update the state of the rack
                    self.racks[device_idx].fail_rack(fail_time)
                    for i in xrange(self.nodes_per_rack):
                        # update the state of the node
                        node_idx = device_idx * self.nodes_per_rack + i
                        if self.nodes[node_idx].get_curr_state(
                        ) == Node.STATE_NODE_NORMAL:
                            self.nodes[node_idx].offline_node()
                            for j in xrange(self.disks_per_node):
                                # update the state of the disk
                                disk_idx = node_idx * self.disks_per_node + j
                                if self.disks[disk_idx].get_curr_state(
                                ) == Disk.STATE_NORMAL:
                                    self.disks[disk_idx].offline_disk(
                                        fail_time)
                # generate the repair event
                if not self.use_power_outage:
                    self.set_rack_repair(device_idx, fail_time)

            return (fail_time, Rack.EVENT_RACK_FAIL, None)

        # repair for permanent disk failure
        elif next_event_type == Disk.EVENT_DISK_REPAIR:
            repair_time = next_event_time
            for repair_disk_idx in device_idx_set:
                if self.disks[repair_disk_idx].get_curr_state(
                ) == Disk.STATE_CRASHED:
                    # update the state of the disk
                    self.disks[repair_disk_idx].repair_disk(repair_time)
                    # generate next permanent disk failure
                    self.set_disk_fail(repair_disk_idx, repair_time)

                # if the repair event is caused by permanent node failure
                node_idx = repair_disk_idx / self.disks_per_node
                if self.nodes[node_idx].get_curr_state(
                ) == Node.STATE_NODE_CRASHED:
                    all_disk_ok = True
                    for i in xrange(self.disks_per_node):
                        disk = self.disks[node_idx * self.disks_per_node + i]
                        if disk.get_curr_state() != disk.STATE_NORMAL:
                            all_disk_ok = False
                            break
                    if all_disk_ok:
                        # update the state of the node
                        self.nodes[node_idx].repair_node()
                        # generate next permanent node failure
                        if not self.use_trace:
                            self.set_node_fail(node_idx, repair_time)
            # update the network status
            if self.use_network:
                idx = 0
                for repair_disk_idx in device_idx_set:
                    repair_bwth = repair_bwth_set[idx]
                    self.network.update_avail_cross_rack_repair_bwth(
                        self.network.get_avail_cross_rack_repair_bwth() +
                        repair_bwth)
                    idx += 1

            # return the set of repaired disks
            return (repair_time, Disk.EVENT_DISK_REPAIR, device_idx_set)

        # repair for node transient failure
        elif next_event_type == Node.EVENT_NODE_TRANSIENT_REPAIR:
            repair_time = next_event_time
            for repair_node_idx in device_idx_set:
                # update the state of the node
                if self.nodes[repair_node_idx].get_curr_state(
                ) == Node.STATE_NODE_UNAVAILABLE:
                    self.nodes[repair_node_idx].online_node()
                    # update the state of the disk
                    for i in xrange(self.disks_per_node):
                        disk_id = repair_node_idx * self.disks_per_node + i
                        if self.disks[disk_id].get_curr_state(
                        ) == Disk.STATE_UNAVAILABLE:
                            self.disks[disk_id].online_disk(repair_time)
                # generate the next transient node failure
                if not self.use_trace:
                    self.set_node_transient_fail(repair_node_idx, repair_time)
            return (repair_time, Node.EVENT_NODE_TRANSIENT_REPAIR, None)

        # repair for rack transient failure
        elif next_event_type == Rack.EVENT_RACK_REPAIR:
            repair_time = next_event_time
            for repair_rack_idx in device_idx_set:
                if self.racks[repair_rack_idx].get_curr_state(
                ) == Rack.STATE_RACK_UNAVAILABLE:
                    # update the state of the rack
                    self.racks[repair_rack_idx].repair_rack()
                    for i in xrange(self.nodes_per_rack):
                        node_idx = repair_rack_idx * self.nodes_per_rack + i
                        # update the state of the node
                        if self.nodes[node_idx].get_curr_state(
                        ) == Node.STATE_NODE_UNAVAILABLE:
                            self.nodes[node_idx].online_node()
                            for j in xrange(self.disks_per_node):
                                disk_idx = node_idx * self.disks_per_node + j
                                # update the state of the disk
                                if self.disks[disk_idx].get_curr_state(
                                ) == Disk.STATE_UNAVAILABLE:
                                    self.disks[disk_idx].online_disk(
                                        repair_time)
                # generate the next transient rack failure
                if not self.use_power_outage:
                    self.set_rack_fail(repair_rack_idx, repair_time)

            return (repair_time, Rack.EVENT_RACK_REPAIR, None)

        else:
            self.logger.error('Wrong type of next_event in get_next_event()!')
            return None

    ##
    # Run an iteration of the simulator
    #
    def run_iteration(self, ite=0):
        self.reset()
        curr_time = 0

        self.logger.info(
            "Regular Simulator: begin an iteration %d, num_failed_disks = %d, "
            "avail_cross_rack_bwth = %d" %
            (ite, len(self.state.get_failed_disks()),
             self.network.get_avail_cross_rack_repair_bwth()))

        while True:
            (event_time, event_type,
             disk_id_set) = self.get_next_event(curr_time)
            curr_time = event_time
            if curr_time > self.mission_time:
                break
            # update the whole status
            if not self.state.update_state(event_type, disk_id_set):
                self.logger.error('update_state failed!')
            if event_type != None:
                self.logger.debug(
                    "Time %s, Event type: %s, Number of failed disks: %s\n" %
                    (event_time, event_type,
                     self.state.get_num_failed_disks()))

            # Check durability when disk_failure/node_failure happens
            if event_type == Disk.EVENT_DISK_FAIL or event_type == Node.EVENT_NODE_FAIL:
                if ite == 1:
                    self.logger.info(
                        "Time %s, Event type: %s, Number of failed disks: %s\n"
                        % (event_time, event_type,
                           self.state.get_num_failed_disks()))
                failed_disks = self.state.get_failed_disks()
                if self.placement.check_data_loss(failed_disks):
                    # the number of failed stripes and the number of lost chunks
                    (num_failed_stripes, num_lost_chunks
                     ) = self.placement.get_num_failed_status(failed_disks)
                    # Count in the delayed stripes
                    if len(self.delayed_repair_dict) != 0:
                        for key in self.delayed_repair_dict:
                            num_failed_stripes += len(
                                self.delayed_repair_dict[key])
                            num_lost_chunks += len(
                                self.delayed_repair_dict[key])
                    # Calculate blocked ratio
                    sum_unavail_time = 0
                    for disk_id in xrange(self.num_disks):
                        sum_unavail_time += self.disks[disk_id].get_unavail_time(curr_time) * \
                                            self.placement.get_num_chunks_per_disk(disk_id)
                    blocked_ratio = sum_unavail_time / (
                        self.placement.num_chunks * curr_time)
                    # Calculate the single-chunk repair ratio
                    single_chunk_repair_ratio = 0
                    self.logger.info(
                        "num_stripes_repaired_single_chunk = %d, num_stripes_repaired = %d"
                        % (self.num_stripes_repaired_single_chunk,
                           self.num_stripes_repaired))

                    if self.num_stripes_repaired != 0:
                        single_chunk_repair_ratio = float(self.num_stripes_repaired_single_chunk) / \
                                                    float(self.num_stripes_repaired)

                    return (1, "(%d, %d, %f, %f)" %
                            (num_failed_stripes, num_lost_chunks,
                             blocked_ratio, single_chunk_repair_ratio))

        # No data loss
        # Calculate blocked ratio
        sum_unavail_time = 0
        for disk_id in xrange(self.num_disks):
            sum_unavail_time += self.disks[disk_id].get_unavail_time(self.mission_time) * \
                                self.placement.get_num_chunks_per_disk(disk_id)
        blocked_ratio = sum_unavail_time / (self.placement.num_chunks *
                                            self.mission_time)
        # Calculate the single-chunk repair ratio
        single_chunk_repair_ratio = 0
        if self.num_stripes_repaired != 0:
            single_chunk_repair_ratio = float(self.num_stripes_repaired_single_chunk) / \
                                        float(self.num_stripes_repaired)

        return (0,
                "(0, 0, %f, %f)" % (blocked_ratio, single_chunk_repair_ratio))