def addBandwidthStat(self, r):
if self.max_bw < r.bandwidht():
self.max_bw = r.bandwidht()
error_logger.info("Max bw now is:" + str(self.max_bw))
self.putBandwidthList(r.start, r)
self.putBandwidthList(r.end, r)
def diskUsage(self):
if self.total_slices == 0:
error_logger.info("slices have not distributed, disks are empty!")
return 0
disk_cap_in_MBs = self.returnDiskCapacity() * pow(10, 12) / pow(2, 20)
system_cap = self.returnTotalDiskCount() * disk_cap_in_MBs
return self.returnTotalSliceCount(
) * self.n * self.conf.chunk_size / system_cap
def distributeSlices(self, root, increase_slices):
full_disk_count = 0
full_machine_count = 0
self.total_slices += increase_slices
disks_per_machine = self.conf.disks_per_machine
machines_per_rack = self.conf.machines_per_rack
rack_count = self.conf.rack_count
machines = self.getAllMachines()
disk_indexes = [[[a for a in xrange(disks_per_machine)] for i in xrange(machines_per_rack)] for j in xrange(rack_count)]
machine_indexes = [[i for i in xrange(machines_per_rack)] for j in xrange(rack_count)]
rack_indexes = [i for i in xrange(rack_count)]
for slice_index in xrange(self.total_slices - increase_slices, self.total_slices):
retry_count = 0
location = []
# (rack_index, machine_index)
rack_machine_indexes = []
while retry_count <= 100:
flag = False
chosen_rack_indexes = sample(rack_indexes, self.r)
for i, item in enumerate(chosen_rack_indexes):
if len(machine_indexes[item]) < self.slices_chunks_on_racks[i]:
flag = True
if flag:
retry_count += 1
continue
for i, rack_index in enumerate(chosen_rack_indexes):
chosen_machine_indexes = sample(machine_indexes[rack_index], self.slices_chunks_on_racks[i])
for m_index in chosen_machine_indexes:
rack_machine_indexes.append((rack_index, m_index))
break
for rack_index, m_index in rack_machine_indexes:
disk_index = choice(disk_indexes[rack_index][m_index])
disk = machines[rack_index][m_index].getChildren()[disk_index]
disk.addChild(slice_index)
location.append(disk)
if len(disk.getChildren()) >= self.conf.max_chunks_per_disk:
full_disk_count += 1
error_logger.info("One disk is completely full " + str(disk.toString()))
disk_indexes[rack_index][m_index].remove(disk_index)
if disk_indexes[rack_index][m_index] == []:
machine_indexes[rack_index].remove(m_index)
if machine_indexes[rack_index] == []:
rack_indexes.remove(rack_index)
self.slice_locations.append(location)
self._my_assert(len(self.slice_locations[slice_index]) == self.n)
self._my_assert(len(self.slice_locations) == self.total_slices)
def distributeSliceToDisk(self, slice_index, machines):
retry_count = 0
full_machine_count = 0
locations = []
if len(machines) < self.r:
raise Exception("No enough racks left")
retry_flag = True
while retry_flag:
racks_for_slice = sample(machines, self.r)
retry_flag = False
for i, rack in enumerate(racks_for_slice):
if len(rack) < self.slices_chunks_on_racks[i]:
retry_flag = True
retry_count += 1
if retry_count > 100:
error_logger.error("Unable to distribute slice " +
str(slice_index))
raise Exception("Data distribution failed")
else:
break
# choose machines from the right rack
for i, rack in enumerate(racks_for_slice):
machines_for_slice = sample(rack, self.slices_chunks_on_racks[i])
for machine in machines_for_slice:
disk = choice(machine.getChildren())
locations.append(disk)
disk.addChild(slice_index)
slice_count = len(disk.getChildren())
if slice_count >= self.conf.max_chunks_per_disk:
full_disk_count += 1
error_logger.info("One disk is completely full " +
str(disk.toString()))
rack.remove(machine)
if len(rack) == 0:
error_logger.error("One rack is completely full" +
str(machine.getParent().getID()))
machines.remove(rack)
# LZR
self.slice_locations[slice_index] = locations