def list_container_objects(self, container):
#container is a list of containers
if DEBUG:
dmt = mtimer(self.__class__.name, "container_list", None)
dmt.begin()
ret = None
obj_list = []
list_threads = [
list_container_objects_thread(container[i]) for i in range(self.m)
]
for it in list_threads:
it.start()
for it in list_threads:
it.join()
if DEBUG:
for it in list_threads:
TestLogger.getInstance().log_sentence(
"list_container.list_threads\t" + it.dmt.get_info() +
"\t" + "None")
if DEBUG:
dmt.end()
TestLogger.getInstance().log_sentence("list_conainer.list\t" +
dmt.get_info() + "\t" +
"None")
for it in list_threads:
obj_list.extend(it.obj_list)
obj_name_list = []
for i in obj_list:
if cmp(i.name[-5:], '.meta') == 0:
obj_name_list.append(os.path.splitext(i.name)[0])
obj_name_set = list(set(obj_name_list))
name_suffix = str(
get_cloud_provider(container[0].driver.__class__.name))
container_name = container[0].name[:len(container[0].name) -
len(name_suffix)]
# ret = [self.get_object(container_name, obj_name_set[i])
# for i in range(len(obj_name_set)) ]
# return ret
ret = [
mObject(i, None, None, {}, None, None, container_name, self, False)
for i in obj_name_set
]
print("list_container_object complete")
return ret
def download_stripes(file_name, container_name, stripes, k, curdir, keys_dict): #stripes is a list of ( provider, i) ,i means ith stripe stripe_threads = [createThread(container_name, file_name + '.' + str(stripes[i][1]), get_cloud_provider(stripes[i][0]), stripes[i][0], keys_dict[stripes[i][0]][0], keys_dict[stripes[i][0]][1], curdir ) for i in range(k) ] for it in stripe_threads : it.start() for it in stripe_threads : it.join()
def download_meta(meta_name, container_name, provider_list, keys_dict, provider_dirs) :
#download .meta file
#provider_list is a list of (str(provider_name))
meta_threads = [ createThread(container_name,
meta_name,
get_cloud_provider(provider_list[i]),
provider_list[i],
keys_dict[provider_list[i]][0],
keys_dict[provider_list[i]][1],
provider_dirs[i],
timing = True)
for i in range(len(provider_list)) ]
for it in meta_threads:
it.start()
for it in meta_threads:
it.join()
temp = {it.name : it.time for it in meta_threads }
return temp
print("download .meta complete")
def delete_object(self, mobj):
mobj = self.integrate(mobj)
#delete .meta file
for d in mobj.driver.drivers:
try:
d.delete_object(
d.get_object(
mobj.container_name +
str(get_cloud_provider(d.__class__.name)),
mobj.name + '.meta'))
except LibcloudError:
None
for i in range(len(mobj.driver.drivers)):
print(mobj.objs[i])
#delete file stripes
print(len(mobj.driver.drivers))
for i in range(len(mobj.driver.drivers)):
print(mobj.objs[i])
ret = []
ret.extend([0] * len(mobj.driver.drivers))
for i in range(len(mobj.driver.drivers)):
try:
ret[i] = mobj.driver.drivers[i].delete_object(mobj.objs[i])
except LibcloudError:
ret[i] = False
for it in ret:
if it:
return True
raise ObjectDoesNotExistError(value=None,
driver=self,
object_name=mobj.name)
return False
def fec_upload(file_name, container_name, block_size, k, m, stripe_location,
keys_dict):
#fec_file
file = open(file_name, "r")
streams = fec_file(file, block_size, k, m)
write_streams_to_file(streams, file_name)
print("fec file complete")
#generate .meta file
#note: the size of stripe_location should be equal with m
#the content of stripe_location shoud be Storage providers
meta = FileMeta()
meta.set_name(file_name)
meta.set_size(path.getsize(file_name))
meta.set_blocksize(block_size)
meta.set_k(k)
meta.set_m(m)
for i in range(m):
meta.set_stripe_location("s" + str(i), stripe_location[i])
#generate md5 for stripes and .meta
for i in range(m):
file_it = open(file_name + '.' + str(i))
meta.set_md5("c" + str(i), md5.new(file_it.read()).hexdigest())
meta.set_md5("cmeta", meta.cal_md5())
meta.save_to_file()
print("save meta complete")
#threading upload
#different Sotrage Providers should have different mlibcloudid and mlibcloudkey
if EXP:
EXP_timer = mtimer("[EXP] upload_stripes :")
EXP_timer.begin()
threads = [
createThread(file_name + '.' + str(i), container_name,
get_cloud_provider(stripe_location[i]),
keys_dict[stripe_location[i]][0],
keys_dict[stripe_location[i]][1]) for i in range(m)
]
for it in threads:
it.start()
for it in threads:
it.join()
if EXP:
EXP_timer.end()
EXP_timer.record_data()
#upload .meta to cloud
meta_location = set(stripe_location)
meta_threads = [
createThread(file_name + '.meta', container_name,
get_cloud_provider(i), keys_dict[i][0], keys_dict[i][1])
for i in meta_location
]
for it in meta_threads:
it.start()
for it in meta_threads:
it.join()
def get_object(self, container_name, obj_name):
meta_name = obj_name + '.meta'
container_name_suffix = [str(i) for i in self.get_provider_list()]
#get .meta objects
if DEBUG:
dmt = mtimer(self.__class__.name, container_name, obj_name)
dmt.begin()
meta_objs = [
self.drivers[i].get_object(
container_name + container_name_suffix[i], meta_name)
for i in range(len(self.drivers))
]
if DEBUG:
dmt.end()
TestLogger.getInstance().log_sentence("get_object.get_metas" +
"\t" + dmt.get_info() +
'\t' + "True")
#dwonload .meta file
if DEBUG:
dmt.begin()
if not os.path.exists('./temp'):
os.mkdir('./temp')
dest_path = './temp/'
meta_threads = [
download_object_thread(self.drivers[i],
meta_objs[i],
dest_path + meta_name + '.' + str(i),
overwrite_existing=True,
timing=True)
for i in range(len(self.drivers))
]
for it in meta_threads:
it.start()
for it in meta_threads:
it.join()
if DEBUG:
for it in meta_threads:
TestLogger.getInstance().log_sentence(
"get_objects.download_metas_thread\t" + it.dmt.get_info() +
'\t' + str(it.ret))
if DEBUG:
dmt.end()
TestLogger.getInstance().log_sentence(
"get_objects.download_metas\t" + dmt.get_info() + '\t' +
"None")
#driver_times {driver_name : driver_time }
# driver_times = {it.name : it.time for it in meta_threads }
driver_times = {}
for it in meta_threads:
driver_times[it.name] = it.time
#check meta files
if DEBUG:
dmt.begin()
meta = None
for i in range(len(self.drivers)):
if not self.check_meta(dest_path + meta_name + '.' + str(i)):
print(" %d th meta md5 check False " % i)
else:
print(" %d th meta md5 check True " % i)
meta = FileMeta()
meta.load_from_file(dest_path + meta_name + '.' + str(i))
break
if meta == None:
print("There is no valid meta file ")
raise LibcloudError("No valid meta file Error", driver=self)
return None
k, m, file_size, block_size, stripe_location = self.get_info_from_meta(
meta)
self.k = k
self.m = m
self.block_size = block_size
#delete meta files
for i in range(m):
os.remove('./temp/' + meta_name + '.' + str(i))
if DEBUG:
dmt.end()
TestLogger.getInstance().log_sentence(
"get_object.check_getinfo_from_meta\t" + dmt.get_info() +
"\t" + "None")
if DEBUG:
dmt.begin()
#temp is a list of (provider , i, cost_time) sorted by cost_time ,i means the ith stripe
temp = [(stripe_location['s' + str(i)], i,
driver_times[stripe_location['s' + str(i)]])
for i in range(m)]
temp = sorted(temp, key=lambda t: t[2])
#stripes_to_download is a list of ( driver, i) , i means the ith stripes
sorted_drivers = [(self.driver_dict[temp[i][0]], temp[i][1])
for i in range(m)]
obj_names = [obj_name + '.' + str(i[1]) for i in sorted_drivers]
obj_drivers = [i[0] for i in sorted_drivers]
# parts = [ sorted_drivers[i][1] for i in range(m) ]
container_name_suffix = [
str(get_cloud_provider(obj_drivers[i].__class__.name))
for i in range(m)
]
obj_list = [
obj_drivers[i].get_object(
container_name + container_name_suffix[i], obj_names[i])
for i in range(m)
]
ret = mObject(obj_name, file_size, None, {}, meta, obj_list,
container_name, self, True)
if DEBUG:
dmt.end()
TestLogger.getInstance().log_sentence("get_object.strategy\t" +
dmt.get_info() + "\t" +
"True")
return ret
def get_provider_list(self):
provider_list = [
get_cloud_provider(d.__class__.name) for d in self.drivers
]
return provider_list