Esempio n. 1
0
def test_sid(metasync, opts):
    import services
    allset = set()
    for srv, doc in services.backends():
        if (srv != "disk"):
            instance = services.factory(srv)
            assert instance is not None
            sid = instance.sid()
            print(sid, instance)
            assert sid is not None
            assert sid not in allset
            allset.add(sid)
Esempio n. 2
0
def test_sid(metasync, opts):
    import services
    allset = set()
    for srv, doc in services.backends():
        if(srv != "disk"):
            instance = services.factory(srv)
            assert instance is not None
            sid = instance.sid()
            print(sid, instance)
            assert sid is not None
            assert sid not in allset
            allset.add(sid)
Esempio n. 3
0
    def _load(self):
        if not self.check_sanity():
            return

        if (not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)

        # load config
        self.config = util.load_config(self.path_conf)
        self.namespace = self.config.get("core", "namespace")
        self.clientid = self.config.get("core", "clientid")

        # load services from config
        self.srvmap = {}
        for tok in self.config.get("backend", "services").split(","):
            srv = services.factory(tok)
            self.srvmap[srv.sid()] = srv

        self.nreplicas = int(self.config.get("backend", "nreplicas"))

        nthreads = self.options.nthreads if self.options is not None else 2
        self.scheduler = Scheduler(self.services,
                                   (nthreads + 1) * len(self.srvmap))

        # load translator pipe
        if self.is_encypted():
            self.translators.append(translators.TrEncrypt(self))

        # TODO. for integrity option
        # if self.is_signed():
        #     self.translators.append(TrSigned(self))

        beg = time.time()
        if (os.path.exists(self.get_path("mapping.pcl"))):
            with open(self.get_path("mapping.pcl")) as f:
                self.mapping = pickle.load(f)
        else:
            mapconfig = []
            for srv in self.services:
                mapconfig.append((srv.sid(), srv.info_storage() / GB))
            hspacesum = sum(map(lambda x: x[1], mapconfig))
            hspace = max(hspacesum + 1, 1024)
            self.mapping = DetMap2(mapconfig,
                                   hspace=hspace,
                                   replica=self.nreplicas)
            self.mapping.pack()
            with open(self.get_path("mapping.pcl"), "w") as f:
                pickle.dump(self.mapping, f)
        end = time.time()
        dbg.time("mapping init %s" % (end - beg))
        dbg.dbg("head: %s", self.get_head_name())
Esempio n. 4
0
    def _load(self):
        if not self.check_sanity():
            return

        if(not os.path.exists(AUTH_DIR)): os.mkdir(AUTH_DIR)

        # load config
        self.config    = util.load_config(self.path_conf)
        self.namespace = self.config.get("core", "namespace")
        self.clientid  = self.config.get("core", "clientid")

        # load services from config
        self.srvmap = {}
        for tok in self.config.get("backend", "services").split(","):
            srv = services.factory(tok)
            self.srvmap[srv.sid()] = srv

        self.nreplicas = int(self.config.get("backend", "nreplicas"))
            
        nthreads = self.options.nthreads if self.options is not None else 2
        self.scheduler = Scheduler(self.services, (nthreads+1)*len(self.srvmap))

        # load translator pipe
        if self.is_encypted():
            self.translators.append(translators.TrEncrypt(self))

        # TODO. for integrity option
        # if self.is_signed():
        #     self.translators.append(TrSigned(self))

        beg = time.time()
        if(os.path.exists(self.get_path("mapping.pcl"))):
            with open(self.get_path("mapping.pcl")) as f:
                self.mapping = pickle.load(f)
        else:
            mapconfig = []
            for srv in self.services:
                mapconfig.append((srv.sid(), srv.info_storage()/GB))
            hspacesum = sum(map(lambda x:x[1], mapconfig))
            hspace = max(hspacesum+1, 1024)
            self.mapping = DetMap2(mapconfig, hspace=hspace, replica=self.nreplicas)
            self.mapping.pack()
            with open(self.get_path("mapping.pcl"), "w") as f:
                pickle.dump(self.mapping, f)
        end = time.time()
        dbg.time("mapping init %s" % (end-beg))
        dbg.dbg("head: %s", self.get_head_name())
Esempio n. 5
0
def test_bench_latency(metasync, opts):
    import services
    allset = set()
    path = '%d' % time.time()
    with open("/dev/urandom") as ifd:
        content = ifd.read(128)

    for srv, doc in services.backends():
        if (srv != "disk"):
            instance = services.factory(srv)
            beg = time.time()
            instance.put(path, content)
            end = time.time()
            print(srv + " up " + str(end - beg))
            beg = time.time()
            ct = instance.get(path)
            end = time.time()
            print(srv + " dn " + str(end - beg))
Esempio n. 6
0
def test_bench_latency(metasync, opts):
    import services
    allset = set()
    path = '%d' % time.time()
    with open("/dev/urandom") as ifd:
        content = ifd.read(128)

    for srv, doc in services.backends():
        if(srv != "disk"):
            instance = services.factory(srv)
            beg = time.time()
            instance.put(path,content)
            end = time.time()
            print(srv + " up " + str(end-beg))
            beg = time.time()
            ct = instance.get(path)
            end = time.time()
            print(srv + " dn " + str(end-beg))
Esempio n. 7
0
def test_service_auth(metasync, opts):
    dropbox = services.factory('dropbox')
    google = services.factory('google')
    box = services.factory('box')
Esempio n. 8
0
        else:
            files.append((item['name'], item['created_time']))
    return dirs, files

def walk(srv, path, listdir):
    toexpand = [path]
    while toexpand:
        path = toexpand.pop(0)
        dirs, files = listdir(srv, path)
        yield path, dirs, files
        toexpand.extend([path+'/'+d for d in dirs])

cls = sys.argv[1].lower()
path = util.format_path(sys.argv[2])

srv = services.factory(cls)
method = globals().get(cls + '_listdir', None)

beg = None
beg_file = None
end = None
end_file = None
for root, dirs, files in walk(srv, path, method):
    #print root, dirs, files
    for f in files:
        #print f
        ts = util.convert_time(f[1])
        if beg == None or ts < beg:
            beg = ts
            beg_file = f
        if end == None or ts > end:
Esempio n. 9
0
def test_service_auth(metasync, opts):
    dropbox = services.factory('dropbox')
    google = services.factory('google')
    box = services.factory('box')
Esempio n. 10
0
    def cmd_reconfigure(self, backends, replica):
        srvmap = {}
        for tok in backends.split(","):
            srv = services.factory(tok)
            srvmap[srv.sid()] = srv
        lst_services = srvmap.values()
        mapconfig = []
        lock_dic = {}
        for srv in lst_services:
            mapconfig.append((srv.sid(), srv.info_storage() / GB))
        for srv in srvmap:
            lock_dic[srv] = threading.Lock()
            if srv not in self.srvmap:
                srvmap[srv].putdir(self.get_remote_path("objects"))
        for srv in self.srvmap:
            if srv not in lock_dic:
                lock_dic[srv] = threading.Lock()

        beg = time.time()
        self.mapping.reconfig(mapconfig, eval(replica))
        end = time.time()
        dbg.info("remap: %.3fs" % (end - beg))
        beg = time.time()
        lst_objs = self.blobstore.list()
        added, removed = self.mapping.get_remapping(lst_objs)
        nthreads = self.options.nthreads if self.options is not None else 2

        #REFACTOR
        def __put_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if (len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    with open(self.get_local_obj_path(next), "rb") as f:
                        blob = f.read()
                        for tr in self.translators:
                            blob = tr.put(blob)
                        # XXX HACK
                        backoff = 0.5
                        remote_path = self.get_remote_obj_path(next)
                        while not srv.put(remote_path, blob):
                            time.sleep(backoff)
                            backoff *= 2

        def __rm_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if (len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    remote_path = self.get_remote_obj_path(next)
                    srv.rm(remote_path)

        cnt_added = 0
        for srv in added:
            if (len(added[srv]) == 0): continue
            cnt_added += len(added[srv])
            for i in range(nthreads):
                self.scheduler.submit(srvmap[srv], False, __put_next,
                                      added[srv], lock_dic[srv])
        self._join()

        end = time.time()
        dbg.info("remap put: %.3fs" % (end - beg))

        beg = time.time()
        cnt_removed = 0
        for srv in removed:
            if (len(removed[srv]) == 0): continue
            cnt_removed += len(removed[srv])
            for i in range(nthreads):
                self.scheduler.submit(self.srvmap[srv], False, __rm_next,
                                      removed[srv], lock_dic[srv])
        self._join()
        end = time.time()
        dbg.info("remap rm: %.3fs" % (end - beg))
        dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
Esempio n. 11
0
    def cmd_clone(self, namespace, backend=None, encrypt_key=None):
        # if wrong target
        if self.check_sanity():
            return False

        # reset all the path by including the namespace
        self.path_root = os.path.join(self.path_root, namespace)
        self.path_meta = os.path.join(self.path_root, META_DIR)
        self.path_conf = self.get_path("config")
        self.path_objs = self.get_path("objects")
        #self.path_head_history = self.get_path("head_history")

        if os.path.exists(self.path_root):
            dbg.err("%s already exists." % self.path_root)
            return False

        if backend is None:
            print "input one of the storage backends, (e.g., dropbox,google,box)"
            print "  for testing, use disk@/path (e.g., disk@/tmp)"
            backend = raw_input("> ")

        srv = services.factory(backend)
        self.namespace = namespace

        # create repo directory
        os.mkdir(self.path_root)
        os.mkdir(self.path_meta)
        os.mkdir(self.path_objs)

        curmaster = self.get_uptodate_master(False, srv)
        sp = curmaster.split(".")
        master = sp[0]
        seed = sp[1]
        seed = srv.get(self.get_remote_path("configs/%s" % seed))
        conf = util.loads_config(seed)

        # setup client specific info
        conf.set('core', 'clientid', util.gen_uuid())
        conf.set('core', 'encryptkey', _get_conf_encryptkey(encrypt_key))

        with open(self.path_conf, "w") as fd:
            conf.write(fd)

        self._load()
        beg = time.time()
        self.bstore_download()
        self._join()

        with open(self.get_head(), "w") as f:
            f.write(curmaster)
        with open(self.get_prev(), "w") as f:
            f.write(curmaster)

        # send my head to remote
        self._put_all(self.get_head(),
                      self.get_remote_path(self.get_head_name()))
        self._put_all(self.get_prev(),
                      self.get_remote_path(self.get_prev_name()))
        self._join()

        if (master):
            ret = self.restore_from_master()
        end = time.time()
        dbg.dbg("clone: %ss" % (end - beg))
        return True
Esempio n. 12
0
    def cmd_clone(self, namespace, backend=None, encrypt_key=None):
        # if wrong target
        if self.check_sanity():
            return False

        # reset all the path by including the namespace
        self.path_root   = os.path.join(self.path_root, namespace)
        self.path_meta   = os.path.join(self.path_root, META_DIR)
        self.path_conf   = self.get_path("config")
        self.path_objs   = self.get_path("objects")
        #self.path_head_history = self.get_path("head_history")

        if os.path.exists(self.path_root):
            dbg.err("%s already exists." % self.path_root)
            return False

        if backend is None:
            print "input one of the storage backends, (e.g., dropbox,google,box)"
            print "  for testing, use disk@/path (e.g., disk@/tmp)"
            backend = raw_input("> ")

        srv  = services.factory(backend)
        self.namespace = namespace

        # create repo directory
        os.mkdir(self.path_root)
        os.mkdir(self.path_meta)
        os.mkdir(self.path_objs)

        curmaster = self.get_uptodate_master(False, srv)
        sp = curmaster.split(".")
        master = sp[0]
        seed = sp[1]
        seed = srv.get(self.get_remote_path("configs/%s" % seed))
        conf = util.loads_config(seed)

        # setup client specific info
        conf.set('core', 'clientid'  , util.gen_uuid())
        conf.set('core', 'encryptkey', _get_conf_encryptkey(encrypt_key))

        with open(self.path_conf, "w") as fd:
            conf.write(fd)

        self._load()
        beg = time.time()
        self.bstore_download()
        self._join()

        with open(self.get_head(), "w") as f:
            f.write(curmaster)
        with open(self.get_prev(), "w") as f:
            f.write(curmaster)

        # send my head to remote
        self._put_all(self.get_head(), self.get_remote_path(self.get_head_name()))
        self._put_all(self.get_prev(), self.get_remote_path(self.get_prev_name()))
        self._join()

        if (master):
            ret = self.restore_from_master()
        end = time.time()
        dbg.dbg("clone: %ss" % (end-beg))
        return True
Esempio n. 13
0
    def cmd_reconfigure(self, backends, replica):
        srvmap = {}
        for tok in backends.split(","):
            srv = services.factory(tok)
            srvmap[srv.sid()] = srv
        lst_services = srvmap.values()
        mapconfig = []
        lock_dic = {}
        for srv in lst_services:
            mapconfig.append((srv.sid(), srv.info_storage()/GB))
        for srv in srvmap:
            lock_dic[srv] = threading.Lock()
            if srv not in self.srvmap:
                srvmap[srv].putdir(self.get_remote_path("objects"))
        for srv in self.srvmap:
            if srv not in lock_dic: 
                lock_dic[srv] = threading.Lock()

        beg = time.time()
        self.mapping.reconfig(mapconfig, eval(replica))
        end = time.time()
        dbg.info("remap: %.3fs" % (end-beg))
        beg = time.time()
        lst_objs = self.blobstore.list()
        added, removed = self.mapping.get_remapping(lst_objs)
        nthreads = self.options.nthreads if self.options is not None else 2

        #REFACTOR 
        def __put_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if(len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    with open(self.get_local_obj_path(next), "rb") as f:
                        blob = f.read()
                        for tr in self.translators:
                            blob = tr.put(blob)
                        # XXX HACK
                        backoff = 0.5
                        remote_path = self.get_remote_obj_path(next)
                        while not srv.put(remote_path, blob):
                            time.sleep(backoff)
                            backoff *= 2

        def __rm_next(srv, lst, lock):
            dbg.job("submitted to: %s" % srv)
            while True:
                lock.acquire()
                if(len(lst) == 0):
                    lock.release()
                    break
                next = lst.pop()
                lock.release()
                if next is not None:
                    remote_path = self.get_remote_obj_path(next)
                    srv.rm(remote_path)
        cnt_added = 0 
        for srv in added:
            if(len(added[srv]) == 0): continue
            cnt_added += len(added[srv])
            for i in range(nthreads):
                self.scheduler.submit(srvmap[srv], False, __put_next, added[srv], lock_dic[srv])   
        self._join()

        end = time.time()
        dbg.info("remap put: %.3fs" % (end-beg))

        beg = time.time()
        cnt_removed = 0
        for srv in removed:
            if(len(removed[srv]) == 0): continue
            cnt_removed += len(removed[srv])
            for i in range(nthreads):
                self.scheduler.submit(self.srvmap[srv], False, __rm_next, removed[srv], lock_dic[srv])   
        self._join()
        end = time.time()
        dbg.info("remap rm: %.3fs" % (end-beg))
        dbg.info("added %d, removed %d" % (cnt_added, cnt_removed))
Esempio n. 14
0
    return dirs, files


def walk(srv, path, listdir):
    toexpand = [path]
    while toexpand:
        path = toexpand.pop(0)
        dirs, files = listdir(srv, path)
        yield path, dirs, files
        toexpand.extend([path + '/' + d for d in dirs])


cls = sys.argv[1].lower()
path = util.format_path(sys.argv[2])

srv = services.factory(cls)
method = globals().get(cls + '_listdir', None)

beg = None
beg_file = None
end = None
end_file = None
for root, dirs, files in walk(srv, path, method):
    #print root, dirs, files
    for f in files:
        #print f
        ts = util.convert_time(f[1])
        if beg == None or ts < beg:
            beg = ts
            beg_file = f
        if end == None or ts > end: