def test_sid(metasync, opts):
import services
allset = set()
for srv, doc in services.backends():
if (srv != "disk"):
instance = services.factory(srv)
assert instance is not None
sid = instance.sid()
print(sid, instance)
assert sid is not None
assert sid not in allset
allset.add(sid)
def test_sid(metasync, opts):
import services
allset = set()
for srv, doc in services.backends():
if(srv != "disk"):
instance = services.factory(srv)
assert instance is not None
sid = instance.sid()
print(sid, instance)
assert sid is not None
assert sid not in allset
allset.add(sid)
def _load(self):
if not self.check_sanity():
return
if (not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)
# load config
self.config = util.load_config(self.path_conf)
self.namespace = self.config.get("core", "namespace")
self.clientid = self.config.get("core", "clientid")
# load services from config
self.srvmap = {}
for tok in self.config.get("backend", "services").split(","):
srv = services.factory(tok)
self.srvmap[srv.sid()] = srv
self.nreplicas = int(self.config.get("backend", "nreplicas"))
nthreads = self.options.nthreads if self.options is not None else 2
self.scheduler = Scheduler(self.services,
(nthreads + 1) * len(self.srvmap))
# load translator pipe
if self.is_encypted():
self.translators.append(translators.TrEncrypt(self))
# TODO. for integrity option
# if self.is_signed():
# self.translators.append(TrSigned(self))
beg = time.time()
if (os.path.exists(self.get_path("mapping.pcl"))):
with open(self.get_path("mapping.pcl")) as f:
self.mapping = pickle.load(f)
else:
mapconfig = []
for srv in self.services:
mapconfig.append((srv.sid(), srv.info_storage() / GB))
hspacesum = sum(map(lambda x: x[1], mapconfig))
hspace = max(hspacesum + 1, 1024)
self.mapping = DetMap2(mapconfig,
hspace=hspace,
replica=self.nreplicas)
self.mapping.pack()
with open(self.get_path("mapping.pcl"), "w") as f:
pickle.dump(self.mapping, f)
end = time.time()
dbg.time("mapping init %s" % (end - beg))
dbg.dbg("head: %s", self.get_head_name())
def _load(self):
if not self.check_sanity():
return
if(not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)
# load config
self.config = util.load_config(self.path_conf)
self.namespace = self.config.get("core", "namespace")
self.clientid = self.config.get("core", "clientid")
# load services from config
self.srvmap = {}
for tok in self.config.get("backend", "services").split(","):
srv = services.factory(tok)
self.srvmap[srv.sid()] = srv
self.nreplicas = int(self.config.get("backend", "nreplicas"))
nthreads = self.options.nthreads if self.options is not None else 2
self.scheduler = Scheduler(self.services, (nthreads+1)*len(self.srvmap))
# load translator pipe
if self.is_encypted():
self.translators.append(translators.TrEncrypt(self))
# TODO. for integrity option
# if self.is_signed():
# self.translators.append(TrSigned(self))
beg = time.time()
if(os.path.exists(self.get_path("mapping.pcl"))):
with open(self.get_path("mapping.pcl")) as f:
self.mapping = pickle.load(f)
else:
mapconfig = []
for srv in self.services:
mapconfig.append((srv.sid(), srv.info_storage()/GB))
hspacesum = sum(map(lambda x:x[1], mapconfig))
hspace = max(hspacesum+1, 1024)
self.mapping = DetMap2(mapconfig, hspace=hspace, replica=self.nreplicas)
self.mapping.pack()
with open(self.get_path("mapping.pcl"), "w") as f:
pickle.dump(self.mapping, f)
end = time.time()
dbg.time("mapping init %s" % (end-beg))
dbg.dbg("head: %s", self.get_head_name())
def test_bench_latency(metasync, opts):
import services
allset = set()
path = '%d' % time.time()
with open("/dev/urandom") as ifd:
content = ifd.read(128)
for srv, doc in services.backends():
if (srv != "disk"):
instance = services.factory(srv)
beg = time.time()
instance.put(path, content)
end = time.time()
print(srv + " up " + str(end - beg))
beg = time.time()
ct = instance.get(path)
end = time.time()
print(srv + " dn " + str(end - beg))
def test_bench_latency(metasync, opts):
import services
allset = set()
path = '%d' % time.time()
with open("/dev/urandom") as ifd:
content = ifd.read(128)
for srv, doc in services.backends():
if(srv != "disk"):
instance = services.factory(srv)
beg = time.time()
instance.put(path,content)
end = time.time()
print(srv + " up " + str(end-beg))
beg = time.time()
ct = instance.get(path)
end = time.time()
print(srv + " dn " + str(end-beg))
def test_service_auth(metasync, opts):
dropbox = services.factory('dropbox')
google = services.factory('google')
box = services.factory('box')
else:
files.append((item['name'], item['created_time']))
return dirs, files
def walk(srv, path, listdir):
toexpand = [path]
while toexpand:
path = toexpand.pop(0)
dirs, files = listdir(srv, path)
yield path, dirs, files
toexpand.extend([path+'/'+d for d in dirs])
cls = sys.argv[1].lower()
path = util.format_path(sys.argv[2])
srv = services.factory(cls)
method = globals().get(cls + '_listdir', None)
beg = None
beg_file = None
end = None
end_file = None
for root, dirs, files in walk(srv, path, method):
#print root, dirs, files
for f in files:
#print f
ts = util.convert_time(f[1])
if beg == None or ts < beg:
beg = ts
beg_file = f
if end == None or ts > end:
def test_service_auth(metasync, opts):
dropbox = services.factory('dropbox')
google = services.factory('google')
box = services.factory('box')
def cmd_reconfigure(self, backends, replica):
srvmap = {}
for tok in backends.split(","):
srv = services.factory(tok)
srvmap[srv.sid()] = srv
lst_services = srvmap.values()
mapconfig = []
lock_dic = {}
for srv in lst_services:
mapconfig.append((srv.sid(), srv.info_storage() / GB))
for srv in srvmap:
lock_dic[srv] = threading.Lock()
if srv not in self.srvmap:
srvmap[srv].putdir(self.get_remote_path("objects"))
for srv in self.srvmap:
if srv not in lock_dic:
lock_dic[srv] = threading.Lock()
beg = time.time()
self.mapping.reconfig(mapconfig, eval(replica))
end = time.time()
dbg.info("remap: %.3fs" % (end - beg))
beg = time.time()
lst_objs = self.blobstore.list()
added, removed = self.mapping.get_remapping(lst_objs)
nthreads = self.options.nthreads if self.options is not None else 2
#REFACTOR
def __put_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if (len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
with open(self.get_local_obj_path(next), "rb") as f:
blob = f.read()
for tr in self.translators:
blob = tr.put(blob)
# XXX HACK
backoff = 0.5
remote_path = self.get_remote_obj_path(next)
while not srv.put(remote_path, blob):
time.sleep(backoff)
backoff *= 2
def __rm_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if (len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
remote_path = self.get_remote_obj_path(next)
srv.rm(remote_path)
cnt_added = 0
for srv in added:
if (len(added[srv]) == 0): continue
cnt_added += len(added[srv])
for i in range(nthreads):
self.scheduler.submit(srvmap[srv], False, __put_next,
added[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap put: %.3fs" % (end - beg))
beg = time.time()
cnt_removed = 0
for srv in removed:
if (len(removed[srv]) == 0): continue
cnt_removed += len(removed[srv])
for i in range(nthreads):
self.scheduler.submit(self.srvmap[srv], False, __rm_next,
removed[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap rm: %.3fs" % (end - beg))
dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
def cmd_clone(self, namespace, backend=None, encrypt_key=None):
# if wrong target
if self.check_sanity():
return False
# reset all the path by including the namespace
self.path_root = os.path.join(self.path_root, namespace)
self.path_meta = os.path.join(self.path_root, META_DIR)
self.path_conf = self.get_path("config")
self.path_objs = self.get_path("objects")
#self.path_head_history = self.get_path("head_history")
if os.path.exists(self.path_root):
dbg.err("%s already exists." % self.path_root)
return False
if backend is None:
print "input one of the storage backends, (e.g., dropbox,google,box)"
print " for testing, use disk@/path (e.g., disk@/tmp)"
backend = raw_input("> ")
srv = services.factory(backend)
self.namespace = namespace
# create repo directory
os.mkdir(self.path_root)
os.mkdir(self.path_meta)
os.mkdir(self.path_objs)
curmaster = self.get_uptodate_master(False, srv)
sp = curmaster.split(".")
master = sp[0]
seed = sp[1]
seed = srv.get(self.get_remote_path("configs/%s" % seed))
conf = util.loads_config(seed)
# setup client specific info
conf.set('core', 'clientid', util.gen_uuid())
conf.set('core', 'encryptkey', _get_conf_encryptkey(encrypt_key))
with open(self.path_conf, "w") as fd:
conf.write(fd)
self._load()
beg = time.time()
self.bstore_download()
self._join()
with open(self.get_head(), "w") as f:
f.write(curmaster)
with open(self.get_prev(), "w") as f:
f.write(curmaster)
# send my head to remote
self._put_all(self.get_head(),
self.get_remote_path(self.get_head_name()))
self._put_all(self.get_prev(),
self.get_remote_path(self.get_prev_name()))
self._join()
if (master):
ret = self.restore_from_master()
end = time.time()
dbg.dbg("clone: %ss" % (end - beg))
return True
def cmd_clone(self, namespace, backend=None, encrypt_key=None):
# if wrong target
if self.check_sanity():
return False
# reset all the path by including the namespace
self.path_root = os.path.join(self.path_root, namespace)
self.path_meta = os.path.join(self.path_root, META_DIR)
self.path_conf = self.get_path("config")
self.path_objs = self.get_path("objects")
#self.path_head_history = self.get_path("head_history")
if os.path.exists(self.path_root):
dbg.err("%s already exists." % self.path_root)
return False
if backend is None:
print "input one of the storage backends, (e.g., dropbox,google,box)"
print " for testing, use disk@/path (e.g., disk@/tmp)"
backend = raw_input("> ")
srv = services.factory(backend)
self.namespace = namespace
# create repo directory
os.mkdir(self.path_root)
os.mkdir(self.path_meta)
os.mkdir(self.path_objs)
curmaster = self.get_uptodate_master(False, srv)
sp = curmaster.split(".")
master = sp[0]
seed = sp[1]
seed = srv.get(self.get_remote_path("configs/%s" % seed))
conf = util.loads_config(seed)
# setup client specific info
conf.set('core', 'clientid' , util.gen_uuid())
conf.set('core', 'encryptkey', _get_conf_encryptkey(encrypt_key))
with open(self.path_conf, "w") as fd:
conf.write(fd)
self._load()
beg = time.time()
self.bstore_download()
self._join()
with open(self.get_head(), "w") as f:
f.write(curmaster)
with open(self.get_prev(), "w") as f:
f.write(curmaster)
# send my head to remote
self._put_all(self.get_head(), self.get_remote_path(self.get_head_name()))
self._put_all(self.get_prev(), self.get_remote_path(self.get_prev_name()))
self._join()
if (master):
ret = self.restore_from_master()
end = time.time()
dbg.dbg("clone: %ss" % (end-beg))
return True
def cmd_reconfigure(self, backends, replica):
srvmap = {}
for tok in backends.split(","):
srv = services.factory(tok)
srvmap[srv.sid()] = srv
lst_services = srvmap.values()
mapconfig = []
lock_dic = {}
for srv in lst_services:
mapconfig.append((srv.sid(), srv.info_storage()/GB))
for srv in srvmap:
lock_dic[srv] = threading.Lock()
if srv not in self.srvmap:
srvmap[srv].putdir(self.get_remote_path("objects"))
for srv in self.srvmap:
if srv not in lock_dic:
lock_dic[srv] = threading.Lock()
beg = time.time()
self.mapping.reconfig(mapconfig, eval(replica))
end = time.time()
dbg.info("remap: %.3fs" % (end-beg))
beg = time.time()
lst_objs = self.blobstore.list()
added, removed = self.mapping.get_remapping(lst_objs)
nthreads = self.options.nthreads if self.options is not None else 2
#REFACTOR
def __put_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if(len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
with open(self.get_local_obj_path(next), "rb") as f:
blob = f.read()
for tr in self.translators:
blob = tr.put(blob)
# XXX HACK
backoff = 0.5
remote_path = self.get_remote_obj_path(next)
while not srv.put(remote_path, blob):
time.sleep(backoff)
backoff *= 2
def __rm_next(srv, lst, lock):
dbg.job("submitted to: %s" % srv)
while True:
lock.acquire()
if(len(lst) == 0):
lock.release()
break
next = lst.pop()
lock.release()
if next is not None:
remote_path = self.get_remote_obj_path(next)
srv.rm(remote_path)
cnt_added = 0
for srv in added:
if(len(added[srv]) == 0): continue
cnt_added += len(added[srv])
for i in range(nthreads):
self.scheduler.submit(srvmap[srv], False, __put_next, added[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap put: %.3fs" % (end-beg))
beg = time.time()
cnt_removed = 0
for srv in removed:
if(len(removed[srv]) == 0): continue
cnt_removed += len(removed[srv])
for i in range(nthreads):
self.scheduler.submit(self.srvmap[srv], False, __rm_next, removed[srv], lock_dic[srv])
self._join()
end = time.time()
dbg.info("remap rm: %.3fs" % (end-beg))
dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
return dirs, files
def walk(srv, path, listdir):
toexpand = [path]
while toexpand:
path = toexpand.pop(0)
dirs, files = listdir(srv, path)
yield path, dirs, files
toexpand.extend([path + '/' + d for d in dirs])
cls = sys.argv[1].lower()
path = util.format_path(sys.argv[2])
srv = services.factory(cls)
method = globals().get(cls + '_listdir', None)
beg = None
beg_file = None
end = None
end_file = None
for root, dirs, files in walk(srv, path, method):
#print root, dirs, files
for f in files:
#print f
ts = util.convert_time(f[1])
if beg == None or ts < beg:
beg = ts
beg_file = f
if end == None or ts > end: