def cmd_mv(self, src_pn, dst_pn):
if not self.check_sanity():
dbg.err("it's not a metasync repo.")
return False
src_pn = os.path.abspath(src_pn)
dst_pn = os.path.abspath(dst_pn)
#TODO: check src_pn exists
beg = time.time()
try:
dirname = os.path.dirname(src_pn)
dirblob = self.blobstore.load_dir(dirname, False, dirty=True)
if (dirblob is None):
dbg.err("%s does not exist" % src_pn)
return False
except NotTrackedException as e:
dbg.err(str(e))
return False
fname = os.path.basename(src_pn)
if (not fname in dirblob):
dbg.err("%s does not exist" % pn)
return False
fblob = dirblob[fname]
dirblob.rm(fname)
dst_dirname = os.path.dirname(dst_pn)
if (dirname != dst_dirname):
dirblob = self.blobstore.load_dir(dirname, True, dirty=True)
assert dirblob is not None
dst_fname = os.path.basename(dst_pn)
dirblob.add(dst_fname, fblob, dirty=False)
root = self.get_root_blob()
root.store()
newblobs = self.blobstore.get_added_blobs()
util.write_file(self.get_head(), root.hv)
self.append_history(root.hv)
end = time.time()
dbg.time("local write: %f" % (end - beg))
# push new blobs remotely
self.bstore_sync(newblobs)
self._put_all(self.get_head(),
self.get_remote_path(self.get_head_name()))
end = time.time()
dbg.time("remote write: %f" % (end - beg))
# move the file
shutil.move(src_pn, dst_pn)
self._join()
return True
def cmd_mv(self, src_pn, dst_pn):
if not self.check_sanity():
dbg.err("it's not a metasync repo.")
return False
src_pn = os.path.abspath(src_pn)
dst_pn = os.path.abspath(dst_pn)
#TODO: check src_pn exists
beg = time.time()
try:
dirname = os.path.dirname(src_pn)
dirblob = self.blobstore.load_dir(dirname, False, dirty=True)
if(dirblob is None):
dbg.err("%s does not exist" % src_pn)
return False
except NotTrackedException as e:
dbg.err(str(e))
return False
fname = os.path.basename(src_pn)
if(not fname in dirblob):
dbg.err("%s does not exist" % pn)
return False
fblob = dirblob[fname]
dirblob.rm(fname)
dst_dirname = os.path.dirname(dst_pn)
if(dirname != dst_dirname):
dirblob = self.blobstore.load_dir(dirname, True, dirty=True)
assert dirblob is not None
dst_fname = os.path.basename(dst_pn)
dirblob.add(dst_fname, fblob, dirty=False)
root = self.get_root_blob()
root.store()
newblobs = self.blobstore.get_added_blobs()
util.write_file(self.get_head(), root.hv)
self.append_history(root.hv)
end = time.time()
dbg.time("local write: %f" % (end-beg))
# push new blobs remotely
self.bstore_sync(newblobs)
self._put_all(self.get_head(), self.get_remote_path(self.get_head_name()))
end = time.time()
dbg.time("remote write: %f" % (end-beg))
# move the file
shutil.move(src_pn, dst_pn)
self._join()
return True
def cmd_rm(self, pn):
if not self.check_sanity():
dbg.err("this is not a metasync repo")
return False
#TODO: check if the file exists
beg = time.time()
try:
dirname = os.path.dirname(pn)
dirblob = self.blobstore.load_dir(dirname, False)
if (dirblob is None):
dbg.err("%s does not exist" % pn)
return False
except NotTrackedException as e:
dbg.err(str(e))
return False
fname = os.path.basename(pn)
if (not fname in dirblob):
dbg.err("%s does not exist" % pn)
return False
dirblob.rm(fname)
root = self.get_root_blob()
root.store()
newblobs = self.blobstore.get_added_blobs()
# we may need to include pointer for previous version.
util.write_file(self.get_head(), root.hv)
self.append_history(root.hv)
end = time.time()
dbg.time("local write: %f" % (end - beg))
# push new blobs remotely
self.bstore_sync(newblobs)
self._put_all(self.get_head(),
self.get_remote_path(self.get_head_name()))
end = time.time()
dbg.time("remote write: %f" % (end - beg))
self._join()
# drop local copy
# TODO: rm only tracked files if removing file.
try:
os.unlink(pn)
except:
dbg.err("failed to rm %s" % pn)
return False
return True
def cmd_rm(self, pn):
if not self.check_sanity():
dbg.err("this is not a metasync repo")
return False
#TODO: check if the file exists
beg = time.time()
try:
dirname = os.path.dirname(pn)
dirblob = self.blobstore.load_dir(dirname, False)
if(dirblob is None):
dbg.err("%s does not exist" % pn)
return False
except NotTrackedException as e:
dbg.err(str(e))
return False
fname = os.path.basename(pn)
if(not fname in dirblob):
dbg.err("%s does not exist" % pn)
return False
dirblob.rm(fname)
root = self.get_root_blob()
root.store()
newblobs = self.blobstore.get_added_blobs()
# we may need to include pointer for previous version.
util.write_file(self.get_head(), root.hv)
self.append_history(root.hv)
end = time.time()
dbg.time("local write: %f" % (end-beg))
# push new blobs remotely
self.bstore_sync(newblobs)
self._put_all(self.get_head(), self.get_remote_path(self.get_head_name()))
end = time.time()
dbg.time("remote write: %f" % (end-beg))
self._join()
# drop local copy
# TODO: rm only tracked files if removing file.
try:
os.unlink(pn)
except:
dbg.err("failed to rm %s" % pn)
return False
return True
def _load(self):
if not self.check_sanity():
return
if (not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)
# load config
self.config = util.load_config(self.path_conf)
self.namespace = self.config.get("core", "namespace")
self.clientid = self.config.get("core", "clientid")
# load services from config
self.srvmap = {}
for tok in self.config.get("backend", "services").split(","):
srv = services.factory(tok)
self.srvmap[srv.sid()] = srv
self.nreplicas = int(self.config.get("backend", "nreplicas"))
nthreads = self.options.nthreads if self.options is not None else 2
self.scheduler = Scheduler(self.services,
(nthreads + 1) * len(self.srvmap))
# load translator pipe
if self.is_encypted():
self.translators.append(translators.TrEncrypt(self))
# TODO. for integrity option
# if self.is_signed():
# self.translators.append(TrSigned(self))
beg = time.time()
if (os.path.exists(self.get_path("mapping.pcl"))):
with open(self.get_path("mapping.pcl")) as f:
self.mapping = pickle.load(f)
else:
mapconfig = []
for srv in self.services:
mapconfig.append((srv.sid(), srv.info_storage() / GB))
hspacesum = sum(map(lambda x: x[1], mapconfig))
hspace = max(hspacesum + 1, 1024)
self.mapping = DetMap2(mapconfig,
hspace=hspace,
replica=self.nreplicas)
self.mapping.pack()
with open(self.get_path("mapping.pcl"), "w") as f:
pickle.dump(self.mapping, f)
end = time.time()
dbg.time("mapping init %s" % (end - beg))
dbg.dbg("head: %s", self.get_head_name())
def _load(self):
if not self.check_sanity():
return
if(not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)
# load config
self.config = util.load_config(self.path_conf)
self.namespace = self.config.get("core", "namespace")
self.clientid = self.config.get("core", "clientid")
# load services from config
self.srvmap = {}
for tok in self.config.get("backend", "services").split(","):
srv = services.factory(tok)
self.srvmap[srv.sid()] = srv
self.nreplicas = int(self.config.get("backend", "nreplicas"))
nthreads = self.options.nthreads if self.options is not None else 2
self.scheduler = Scheduler(self.services, (nthreads+1)*len(self.srvmap))
# load translator pipe
if self.is_encypted():
self.translators.append(translators.TrEncrypt(self))
# TODO. for integrity option
# if self.is_signed():
# self.translators.append(TrSigned(self))
beg = time.time()
if(os.path.exists(self.get_path("mapping.pcl"))):
with open(self.get_path("mapping.pcl")) as f:
self.mapping = pickle.load(f)
else:
mapconfig = []
for srv in self.services:
mapconfig.append((srv.sid(), srv.info_storage()/GB))
hspacesum = sum(map(lambda x:x[1], mapconfig))
hspace = max(hspacesum+1, 1024)
self.mapping = DetMap2(mapconfig, hspace=hspace, replica=self.nreplicas)
self.mapping.pack()
with open(self.get_path("mapping.pcl"), "w") as f:
pickle.dump(self.mapping, f)
end = time.time()
dbg.time("mapping init %s" % (end-beg))
dbg.dbg("head: %s", self.get_head_name())
def cmd_checkin(self, paths, unit=BLOB_UNIT, upload_only_first=False):
if not self.check_sanity():
dbg.err("this is not a metasync repo")
return False
if type(paths) != types.ListType:
paths = [paths]
for pn in paths:
if not os.path.exists(pn):
dbg.err("File %s doesn't exits." % pn)
return False
beg = time.time()
#XXX: considering mtime, check hash of chunks?
changed = False
for path in paths:
if (not os.path.isfile(path)):
changed = True
for root, dirs, files in os.walk(path):
fsizesum = 0
for fname in files:
fsizesum += os.stat(os.path.join(root, fname)).st_size
print(root + " " + str(fsizesum))
if (fsizesum < unit):
dirblob = self.blobstore.load_dir(root,
dirty=True,
merge=True)
for fname in files:
dirblob.add_file(fname, os.path.join(root, fname))
dirblob.done_adding()
else:
dirblob = self.blobstore.load_dir(root, dirty=True)
for fname in files:
fileblob = self.blobstore.load_file(
os.path.join(root, fname), unit)
if (fname in dirblob
and dirblob[fname].hv == fileblob.hv):
continue
dirblob.add(fname, fileblob)
else:
fileblob = self.blobstore.load_file(path, unit)
dirname = os.path.dirname(path)
if (dirname == ""): dirname = "."
dirblob = self.blobstore.load_dir(dirname, dirty=True)
fname = os.path.basename(path)
if (fname in dirblob and dirblob[fname].hv == fileblob.hv):
continue
changed = True
dirblob.add(fname, fileblob)
if (not changed): return True
root = self.get_root_blob()
root.store()
newblobs = self.blobstore.get_added_blobs()
util.write_file(
self.get_head(), "%s.%s.%d" %
(root.hv, self.get_config_hash(), self.get_next_version()))
end = time.time()
dbg.time("local write: %f" % (end - beg))
# push new blobs remotely
leftover = self.bstore_sync(newblobs)
self._update_all(self.get_head(),
self.get_remote_path(self.get_head_name()))
self._join()
end = time.time()
dbg.time("remote write for R1: %f" % (end - beg))
if (not upload_only_first):
self.bstore_sync_left(leftover)
end = time.time()
dbg.time("remote write for left: %f" % (end - beg))
return []
else:
return leftover
def cmd_checkin(self, paths, unit=BLOB_UNIT, upload_only_first=False):
if not self.check_sanity():
dbg.err("this is not a metasync repo")
return False
if type(paths) != types.ListType:
paths = [paths]
for pn in paths:
if not os.path.exists(pn):
dbg.err("File %s doesn't exits." % pn)
return False
beg = time.time()
#XXX: considering mtime, check hash of chunks?
changed = False
for path in paths:
if(not os.path.isfile(path)):
changed = True
for root, dirs, files in os.walk(path):
fsizesum = 0
for fname in files:
fsizesum += os.stat(os.path.join(root,fname)).st_size
print(root + " " + str(fsizesum))
if(fsizesum < unit):
dirblob = self.blobstore.load_dir(root, dirty=True, merge=True)
for fname in files:
dirblob.add_file(fname, os.path.join(root, fname))
dirblob.done_adding()
else:
dirblob = self.blobstore.load_dir(root, dirty=True)
for fname in files:
fileblob = self.blobstore.load_file(os.path.join(root, fname), unit)
if(fname in dirblob and dirblob[fname].hv == fileblob.hv):
continue
dirblob.add(fname, fileblob)
else:
fileblob = self.blobstore.load_file(path, unit)
dirname = os.path.dirname(path)
if(dirname == ""): dirname = "."
dirblob = self.blobstore.load_dir(dirname, dirty=True)
fname = os.path.basename(path)
if(fname in dirblob and dirblob[fname].hv == fileblob.hv):
continue
changed = True
dirblob.add(fname, fileblob)
if(not changed): return True
root = self.get_root_blob()
root.store()
newblobs = self.blobstore.get_added_blobs()
util.write_file(self.get_head(), "%s.%s.%d" % (root.hv, self.get_config_hash(), self.get_next_version()))
end = time.time()
dbg.time("local write: %f" % (end-beg))
# push new blobs remotely
leftover = self.bstore_sync(newblobs)
self._update_all(self.get_head(), self.get_remote_path(self.get_head_name()))
self._join()
end = time.time()
dbg.time("remote write for R1: %f" % (end-beg))
if(not upload_only_first):
self.bstore_sync_left(leftover)
end = time.time()
dbg.time("remote write for left: %f" % (end-beg))
return []
else:
return leftover