def list_container_objects(self, container): #container is a list of containers if DEBUG: dmt = mtimer(self.__class__.name, "container_list", None) dmt.begin() ret = None obj_list = [] list_threads = [ list_container_objects_thread(container[i]) for i in range(self.m) ] for it in list_threads: it.start() for it in list_threads: it.join() if DEBUG: for it in list_threads: TestLogger.getInstance().log_sentence( "list_container.list_threads\t" + it.dmt.get_info() + "\t" + "None") if DEBUG: dmt.end() TestLogger.getInstance().log_sentence("list_conainer.list\t" + dmt.get_info() + "\t" + "None") for it in list_threads: obj_list.extend(it.obj_list) obj_name_list = [] for i in obj_list: if cmp(i.name[-5:], '.meta') == 0: obj_name_list.append(os.path.splitext(i.name)[0]) obj_name_set = list(set(obj_name_list)) name_suffix = str( get_cloud_provider(container[0].driver.__class__.name)) container_name = container[0].name[:len(container[0].name) - len(name_suffix)] # ret = [self.get_object(container_name, obj_name_set[i]) # for i in range(len(obj_name_set)) ] # return ret ret = [ mObject(i, None, None, {}, None, None, container_name, self, False) for i in obj_name_set ] print("list_container_object complete") return ret
def download_stripes(file_name, container_name, stripes, k, curdir, keys_dict): #stripes is a list of ( provider, i) ,i means ith stripe stripe_threads = [createThread(container_name, file_name + '.' + str(stripes[i][1]), get_cloud_provider(stripes[i][0]), stripes[i][0], keys_dict[stripes[i][0]][0], keys_dict[stripes[i][0]][1], curdir ) for i in range(k) ] for it in stripe_threads : it.start() for it in stripe_threads : it.join()
def download_meta(meta_name, container_name, provider_list, keys_dict, provider_dirs) : #download .meta file #provider_list is a list of (str(provider_name)) meta_threads = [ createThread(container_name, meta_name, get_cloud_provider(provider_list[i]), provider_list[i], keys_dict[provider_list[i]][0], keys_dict[provider_list[i]][1], provider_dirs[i], timing = True) for i in range(len(provider_list)) ] for it in meta_threads: it.start() for it in meta_threads: it.join() temp = {it.name : it.time for it in meta_threads } return temp print("download .meta complete")
def delete_object(self, mobj): mobj = self.integrate(mobj) #delete .meta file for d in mobj.driver.drivers: try: d.delete_object( d.get_object( mobj.container_name + str(get_cloud_provider(d.__class__.name)), mobj.name + '.meta')) except LibcloudError: None for i in range(len(mobj.driver.drivers)): print(mobj.objs[i]) #delete file stripes print(len(mobj.driver.drivers)) for i in range(len(mobj.driver.drivers)): print(mobj.objs[i]) ret = [] ret.extend([0] * len(mobj.driver.drivers)) for i in range(len(mobj.driver.drivers)): try: ret[i] = mobj.driver.drivers[i].delete_object(mobj.objs[i]) except LibcloudError: ret[i] = False for it in ret: if it: return True raise ObjectDoesNotExistError(value=None, driver=self, object_name=mobj.name) return False
def fec_upload(file_name, container_name, block_size, k, m, stripe_location, keys_dict): #fec_file file = open(file_name, "r") streams = fec_file(file, block_size, k, m) write_streams_to_file(streams, file_name) print("fec file complete") #generate .meta file #note: the size of stripe_location should be equal with m #the content of stripe_location shoud be Storage providers meta = FileMeta() meta.set_name(file_name) meta.set_size(path.getsize(file_name)) meta.set_blocksize(block_size) meta.set_k(k) meta.set_m(m) for i in range(m): meta.set_stripe_location("s" + str(i), stripe_location[i]) #generate md5 for stripes and .meta for i in range(m): file_it = open(file_name + '.' + str(i)) meta.set_md5("c" + str(i), md5.new(file_it.read()).hexdigest()) meta.set_md5("cmeta", meta.cal_md5()) meta.save_to_file() print("save meta complete") #threading upload #different Sotrage Providers should have different mlibcloudid and mlibcloudkey if EXP: EXP_timer = mtimer("[EXP] upload_stripes :") EXP_timer.begin() threads = [ createThread(file_name + '.' + str(i), container_name, get_cloud_provider(stripe_location[i]), keys_dict[stripe_location[i]][0], keys_dict[stripe_location[i]][1]) for i in range(m) ] for it in threads: it.start() for it in threads: it.join() if EXP: EXP_timer.end() EXP_timer.record_data() #upload .meta to cloud meta_location = set(stripe_location) meta_threads = [ createThread(file_name + '.meta', container_name, get_cloud_provider(i), keys_dict[i][0], keys_dict[i][1]) for i in meta_location ] for it in meta_threads: it.start() for it in meta_threads: it.join()
def get_object(self, container_name, obj_name): meta_name = obj_name + '.meta' container_name_suffix = [str(i) for i in self.get_provider_list()] #get .meta objects if DEBUG: dmt = mtimer(self.__class__.name, container_name, obj_name) dmt.begin() meta_objs = [ self.drivers[i].get_object( container_name + container_name_suffix[i], meta_name) for i in range(len(self.drivers)) ] if DEBUG: dmt.end() TestLogger.getInstance().log_sentence("get_object.get_metas" + "\t" + dmt.get_info() + '\t' + "True") #dwonload .meta file if DEBUG: dmt.begin() if not os.path.exists('./temp'): os.mkdir('./temp') dest_path = './temp/' meta_threads = [ download_object_thread(self.drivers[i], meta_objs[i], dest_path + meta_name + '.' + str(i), overwrite_existing=True, timing=True) for i in range(len(self.drivers)) ] for it in meta_threads: it.start() for it in meta_threads: it.join() if DEBUG: for it in meta_threads: TestLogger.getInstance().log_sentence( "get_objects.download_metas_thread\t" + it.dmt.get_info() + '\t' + str(it.ret)) if DEBUG: dmt.end() TestLogger.getInstance().log_sentence( "get_objects.download_metas\t" + dmt.get_info() + '\t' + "None") #driver_times {driver_name : driver_time } # driver_times = {it.name : it.time for it in meta_threads } driver_times = {} for it in meta_threads: driver_times[it.name] = it.time #check meta files if DEBUG: dmt.begin() meta = None for i in range(len(self.drivers)): if not self.check_meta(dest_path + meta_name + '.' + str(i)): print(" %d th meta md5 check False " % i) else: print(" %d th meta md5 check True " % i) meta = FileMeta() meta.load_from_file(dest_path + meta_name + '.' + str(i)) break if meta == None: print("There is no valid meta file ") raise LibcloudError("No valid meta file Error", driver=self) return None k, m, file_size, block_size, stripe_location = self.get_info_from_meta( meta) self.k = k self.m = m self.block_size = block_size #delete meta files for i in range(m): os.remove('./temp/' + meta_name + '.' + str(i)) if DEBUG: dmt.end() TestLogger.getInstance().log_sentence( "get_object.check_getinfo_from_meta\t" + dmt.get_info() + "\t" + "None") if DEBUG: dmt.begin() #temp is a list of (provider , i, cost_time) sorted by cost_time ,i means the ith stripe temp = [(stripe_location['s' + str(i)], i, driver_times[stripe_location['s' + str(i)]]) for i in range(m)] temp = sorted(temp, key=lambda t: t[2]) #stripes_to_download is a list of ( driver, i) , i means the ith stripes sorted_drivers = [(self.driver_dict[temp[i][0]], temp[i][1]) for i in range(m)] obj_names = [obj_name + '.' + str(i[1]) for i in sorted_drivers] obj_drivers = [i[0] for i in sorted_drivers] # parts = [ sorted_drivers[i][1] for i in range(m) ] container_name_suffix = [ str(get_cloud_provider(obj_drivers[i].__class__.name)) for i in range(m) ] obj_list = [ obj_drivers[i].get_object( container_name + container_name_suffix[i], obj_names[i]) for i in range(m) ] ret = mObject(obj_name, file_size, None, {}, meta, obj_list, container_name, self, True) if DEBUG: dmt.end() TestLogger.getInstance().log_sentence("get_object.strategy\t" + dmt.get_info() + "\t" + "True") return ret
def get_provider_list(self): provider_list = [ get_cloud_provider(d.__class__.name) for d in self.drivers ] return provider_list