Example #1
0
    def cmd_mv(self, src_pn, dst_pn):
        if not self.check_sanity():
            dbg.err("it's not a metasync repo.")
            return False
        src_pn = os.path.abspath(src_pn)
        dst_pn = os.path.abspath(dst_pn)

        #TODO: check src_pn exists
        beg = time.time()
        try:
            dirname = os.path.dirname(src_pn)
            dirblob = self.blobstore.load_dir(dirname, False, dirty=True)
            if (dirblob is None):
                dbg.err("%s does not exist" % src_pn)
                return False
        except NotTrackedException as e:
            dbg.err(str(e))
            return False

        fname = os.path.basename(src_pn)
        if (not fname in dirblob):
            dbg.err("%s does not exist" % pn)
            return False
        fblob = dirblob[fname]
        dirblob.rm(fname)

        dst_dirname = os.path.dirname(dst_pn)
        if (dirname != dst_dirname):
            dirblob = self.blobstore.load_dir(dirname, True, dirty=True)
            assert dirblob is not None

        dst_fname = os.path.basename(dst_pn)
        dirblob.add(dst_fname, fblob, dirty=False)

        root = self.get_root_blob()
        root.store()
        newblobs = self.blobstore.get_added_blobs()

        util.write_file(self.get_head(), root.hv)
        self.append_history(root.hv)

        end = time.time()
        dbg.time("local write: %f" % (end - beg))

        # push new blobs remotely
        self.bstore_sync(newblobs)
        self._put_all(self.get_head(),
                      self.get_remote_path(self.get_head_name()))

        end = time.time()
        dbg.time("remote write: %f" % (end - beg))

        # move the file
        shutil.move(src_pn, dst_pn)
        self._join()

        return True
Example #2
0
    def cmd_mv(self, src_pn, dst_pn):
        if not self.check_sanity():
            dbg.err("it's not a metasync repo.")
            return False
        src_pn = os.path.abspath(src_pn)
        dst_pn = os.path.abspath(dst_pn)

        #TODO: check src_pn exists
        beg = time.time()
        try:
            dirname = os.path.dirname(src_pn)
            dirblob = self.blobstore.load_dir(dirname, False, dirty=True)
            if(dirblob is None):
                dbg.err("%s does not exist" % src_pn)
                return False
        except NotTrackedException as e:
            dbg.err(str(e))
            return False

        fname = os.path.basename(src_pn)
        if(not fname in dirblob): 
            dbg.err("%s does not exist" % pn)
            return False
        fblob = dirblob[fname]
        dirblob.rm(fname)

        dst_dirname = os.path.dirname(dst_pn)
        if(dirname != dst_dirname):
            dirblob = self.blobstore.load_dir(dirname, True, dirty=True)
            assert dirblob is not None

        dst_fname = os.path.basename(dst_pn)
        dirblob.add(dst_fname, fblob, dirty=False)

        root = self.get_root_blob()
        root.store()
        newblobs = self.blobstore.get_added_blobs()

        util.write_file(self.get_head(), root.hv)
        self.append_history(root.hv)

        end = time.time()
        dbg.time("local write: %f" % (end-beg))

        # push new blobs remotely
        self.bstore_sync(newblobs)
        self._put_all(self.get_head(), self.get_remote_path(self.get_head_name()))

        end = time.time()
        dbg.time("remote write: %f" % (end-beg))

        # move the file
        shutil.move(src_pn, dst_pn)
        self._join()

        return True 
Example #3
0
    def cmd_rm(self, pn):
        if not self.check_sanity():
            dbg.err("this is not a metasync repo")
            return False
        #TODO: check if the file exists

        beg = time.time()
        try:
            dirname = os.path.dirname(pn)
            dirblob = self.blobstore.load_dir(dirname, False)
            if (dirblob is None):
                dbg.err("%s does not exist" % pn)
                return False
        except NotTrackedException as e:
            dbg.err(str(e))
            return False

        fname = os.path.basename(pn)
        if (not fname in dirblob):
            dbg.err("%s does not exist" % pn)
            return False

        dirblob.rm(fname)
        root = self.get_root_blob()
        root.store()
        newblobs = self.blobstore.get_added_blobs()

        # we may need to include pointer for previous version.
        util.write_file(self.get_head(), root.hv)
        self.append_history(root.hv)

        end = time.time()
        dbg.time("local write: %f" % (end - beg))

        # push new blobs remotely
        self.bstore_sync(newblobs)
        self._put_all(self.get_head(),
                      self.get_remote_path(self.get_head_name()))

        end = time.time()
        dbg.time("remote write: %f" % (end - beg))
        self._join()

        # drop local copy
        # TODO: rm only tracked files if removing file.
        try:
            os.unlink(pn)
        except:
            dbg.err("failed to rm %s" % pn)
            return False

        return True
Example #4
0
    def cmd_rm(self, pn):
        if not self.check_sanity():
            dbg.err("this is not a metasync repo")
            return False
        #TODO: check if the file exists

        beg = time.time()
        try:
            dirname = os.path.dirname(pn)
            dirblob = self.blobstore.load_dir(dirname, False)
            if(dirblob is None):
                dbg.err("%s does not exist" % pn)
                return False
        except NotTrackedException as e:
            dbg.err(str(e))
            return False

        fname = os.path.basename(pn)
        if(not fname in dirblob): 
            dbg.err("%s does not exist" % pn)
            return False

        dirblob.rm(fname)
        root = self.get_root_blob()
        root.store()
        newblobs = self.blobstore.get_added_blobs()

        # we may need to include pointer for previous version.
        util.write_file(self.get_head(), root.hv)
        self.append_history(root.hv)

        end = time.time()
        dbg.time("local write: %f" % (end-beg))

        # push new blobs remotely
        self.bstore_sync(newblobs)
        self._put_all(self.get_head(), self.get_remote_path(self.get_head_name()))

        end = time.time()
        dbg.time("remote write: %f" % (end-beg))
        self._join()

        # drop local copy
        # TODO: rm only tracked files if removing file.
        try:
            os.unlink(pn)
        except:
            dbg.err("failed to rm %s" % pn)
            return False

        return True
Example #5
0
    def _load(self):
        if not self.check_sanity():
            return

        if (not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)

        # load config
        self.config = util.load_config(self.path_conf)
        self.namespace = self.config.get("core", "namespace")
        self.clientid = self.config.get("core", "clientid")

        # load services from config
        self.srvmap = {}
        for tok in self.config.get("backend", "services").split(","):
            srv = services.factory(tok)
            self.srvmap[srv.sid()] = srv

        self.nreplicas = int(self.config.get("backend", "nreplicas"))

        nthreads = self.options.nthreads if self.options is not None else 2
        self.scheduler = Scheduler(self.services,
                                   (nthreads + 1) * len(self.srvmap))

        # load translator pipe
        if self.is_encypted():
            self.translators.append(translators.TrEncrypt(self))

        # TODO. for integrity option
        # if self.is_signed():
        #     self.translators.append(TrSigned(self))

        beg = time.time()
        if (os.path.exists(self.get_path("mapping.pcl"))):
            with open(self.get_path("mapping.pcl")) as f:
                self.mapping = pickle.load(f)
        else:
            mapconfig = []
            for srv in self.services:
                mapconfig.append((srv.sid(), srv.info_storage() / GB))
            hspacesum = sum(map(lambda x: x[1], mapconfig))
            hspace = max(hspacesum + 1, 1024)
            self.mapping = DetMap2(mapconfig,
                                   hspace=hspace,
                                   replica=self.nreplicas)
            self.mapping.pack()
            with open(self.get_path("mapping.pcl"), "w") as f:
                pickle.dump(self.mapping, f)
        end = time.time()
        dbg.time("mapping init %s" % (end - beg))
        dbg.dbg("head: %s", self.get_head_name())
Example #6
0
    def _load(self):
        if not self.check_sanity():
            return

        if(not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)

        # load config
        self.config    = util.load_config(self.path_conf)
        self.namespace = self.config.get("core", "namespace")
        self.clientid  = self.config.get("core", "clientid")

        # load services from config
        self.srvmap = {}
        for tok in self.config.get("backend", "services").split(","):
            srv = services.factory(tok)
            self.srvmap[srv.sid()] = srv

        self.nreplicas = int(self.config.get("backend", "nreplicas"))
            
        nthreads = self.options.nthreads if self.options is not None else 2
        self.scheduler = Scheduler(self.services, (nthreads+1)*len(self.srvmap))

        # load translator pipe
        if self.is_encypted():
            self.translators.append(translators.TrEncrypt(self))

        # TODO. for integrity option
        # if self.is_signed():
        #     self.translators.append(TrSigned(self))

        beg = time.time()
        if(os.path.exists(self.get_path("mapping.pcl"))):
            with open(self.get_path("mapping.pcl")) as f:
                self.mapping = pickle.load(f)
        else:
            mapconfig = []
            for srv in self.services:
                mapconfig.append((srv.sid(), srv.info_storage()/GB))
            hspacesum = sum(map(lambda x:x[1], mapconfig))
            hspace = max(hspacesum+1, 1024)
            self.mapping = DetMap2(mapconfig, hspace=hspace, replica=self.nreplicas)
            self.mapping.pack()
            with open(self.get_path("mapping.pcl"), "w") as f:
                pickle.dump(self.mapping, f)
        end = time.time()
        dbg.time("mapping init %s" % (end-beg))
        dbg.dbg("head: %s", self.get_head_name())
Example #7
0
    def cmd_checkin(self, paths, unit=BLOB_UNIT, upload_only_first=False):
        if not self.check_sanity():
            dbg.err("this is not a metasync repo")
            return False
        if type(paths) != types.ListType:
            paths = [paths]
        for pn in paths:
            if not os.path.exists(pn):
                dbg.err("File %s doesn't exits." % pn)
                return False

        beg = time.time()
        #XXX: considering mtime, check hash of chunks?
        changed = False
        for path in paths:
            if (not os.path.isfile(path)):
                changed = True
                for root, dirs, files in os.walk(path):
                    fsizesum = 0
                    for fname in files:
                        fsizesum += os.stat(os.path.join(root, fname)).st_size
                    print(root + " " + str(fsizesum))
                    if (fsizesum < unit):
                        dirblob = self.blobstore.load_dir(root,
                                                          dirty=True,
                                                          merge=True)
                        for fname in files:
                            dirblob.add_file(fname, os.path.join(root, fname))
                        dirblob.done_adding()
                    else:
                        dirblob = self.blobstore.load_dir(root, dirty=True)
                        for fname in files:
                            fileblob = self.blobstore.load_file(
                                os.path.join(root, fname), unit)
                            if (fname in dirblob
                                    and dirblob[fname].hv == fileblob.hv):
                                continue
                            dirblob.add(fname, fileblob)
            else:
                fileblob = self.blobstore.load_file(path, unit)
                dirname = os.path.dirname(path)
                if (dirname == ""): dirname = "."
                dirblob = self.blobstore.load_dir(dirname, dirty=True)
                fname = os.path.basename(path)
                if (fname in dirblob and dirblob[fname].hv == fileblob.hv):
                    continue
                changed = True
                dirblob.add(fname, fileblob)
        if (not changed): return True
        root = self.get_root_blob()
        root.store()
        newblobs = self.blobstore.get_added_blobs()

        util.write_file(
            self.get_head(), "%s.%s.%d" %
            (root.hv, self.get_config_hash(), self.get_next_version()))

        end = time.time()
        dbg.time("local write: %f" % (end - beg))

        # push new blobs remotely
        leftover = self.bstore_sync(newblobs)
        self._update_all(self.get_head(),
                         self.get_remote_path(self.get_head_name()))

        self._join()
        end = time.time()
        dbg.time("remote write for R1: %f" % (end - beg))
        if (not upload_only_first):
            self.bstore_sync_left(leftover)
            end = time.time()
            dbg.time("remote write for left: %f" % (end - beg))
            return []
        else:
            return leftover
Example #8
0
    def cmd_checkin(self, paths, unit=BLOB_UNIT, upload_only_first=False):
        if not self.check_sanity():
            dbg.err("this is not a metasync repo")
            return False
        if type(paths) != types.ListType:
            paths = [paths] 
        for pn in paths: 
            if not os.path.exists(pn):
                dbg.err("File %s doesn't exits." % pn)
                return False
            
        beg = time.time()
        #XXX: considering mtime, check hash of chunks?
        changed = False
        for path in paths:
            if(not os.path.isfile(path)): 
                changed = True
                for root, dirs, files in os.walk(path):
                    fsizesum = 0
                    for fname in files:
                        fsizesum += os.stat(os.path.join(root,fname)).st_size
                    print(root + " " + str(fsizesum))
                    if(fsizesum < unit):
                        dirblob = self.blobstore.load_dir(root, dirty=True, merge=True)
                        for fname in files:
                            dirblob.add_file(fname, os.path.join(root, fname))
                        dirblob.done_adding()
                    else:
                        dirblob = self.blobstore.load_dir(root, dirty=True)
                        for fname in files:
                            fileblob = self.blobstore.load_file(os.path.join(root, fname), unit)
                            if(fname in dirblob and dirblob[fname].hv == fileblob.hv):
                                continue
                            dirblob.add(fname, fileblob)
            else:
                fileblob = self.blobstore.load_file(path, unit)
                dirname = os.path.dirname(path)
                if(dirname == ""): dirname = "."
                dirblob = self.blobstore.load_dir(dirname, dirty=True)
                fname = os.path.basename(path)
                if(fname in dirblob and dirblob[fname].hv == fileblob.hv):
                    continue
                changed = True
                dirblob.add(fname, fileblob)
        if(not changed): return True
        root = self.get_root_blob()
        root.store()
        newblobs = self.blobstore.get_added_blobs()

        util.write_file(self.get_head(), "%s.%s.%d" % (root.hv, self.get_config_hash(), self.get_next_version()))

        end = time.time()
        dbg.time("local write: %f" % (end-beg))

        # push new blobs remotely
        leftover = self.bstore_sync(newblobs)
        self._update_all(self.get_head(), self.get_remote_path(self.get_head_name()))

        self._join()
        end = time.time()
        dbg.time("remote write for R1: %f" % (end-beg))
        if(not upload_only_first):
            self.bstore_sync_left(leftover)
            end = time.time()
            dbg.time("remote write for left: %f" % (end-beg))
            return []
        else:
            return leftover