Ejemplo n.º 1
0
    def sync_indexes(self):
        self.check_busy()
        conn = self.conn
        mkdirp(self.cachedir)
        # All cached idxs are extra until proven otherwise
        extra = set()
        for f in os.listdir(self.cachedir):
            debug1('%s\n' % f)
            if f.endswith('.idx'):
                extra.add(f)
        needed = set()
        conn.write('list-indexes\n')
        for line in linereader(conn):
            if not line:
                break
            assert(line.find('/') < 0)
            parts = line.split(' ')
            idx = parts[0]
            if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
                # If the server requests that we load an idx and we don't
                # already have a copy of it, it is needed
                needed.add(idx)
            # Any idx that the server has heard of is proven not extra
            extra.discard(idx)

        self.check_ok()
        debug1('client: removing extra indexes: %s\n' % extra)
        for idx in extra:
            os.unlink(os.path.join(self.cachedir, idx))
        debug1('client: server requested load of: %s\n' % needed)
        for idx in needed:
            self.sync_index(idx)
        git.auto_midx(self.cachedir)
Ejemplo n.º 2
0
 def _suggest_packs(self):
     ob = self._busy
     if ob:
         assert(ob == 'receive-objects-v2')
         self.conn.write('\xff\xff\xff\xff')  # suspend receive-objects-v2
     suggested = []
     for line in linereader(self.conn):
         if not line:
             break
         debug2('%s\n' % line)
         if line.startswith('index '):
             idx = line[6:]
             debug1('client: received index suggestion: %s\n'
                    % git.shorten_hash(idx))
             suggested.append(idx)
         else:
             assert(line.endswith('.idx'))
             debug1('client: completed writing pack, idx: %s\n'
                    % git.shorten_hash(line))
             suggested.append(line)
     self.check_ok()
     if ob:
         self._busy = None
     idx = None
     for idx in suggested:
         self.sync_index(idx)
     git.auto_midx(self.cachedir)
     if ob:
         self._busy = ob
         self.conn.write('%s\n' % ob)
     return idx
Ejemplo n.º 3
0
 def _suggest_packs(self):
     ob = self._busy
     if ob:
         assert (ob == 'receive-objects-v2')
         self.conn.write('\xff\xff\xff\xff')  # suspend receive-objects-v2
     suggested = []
     for line in linereader(self.conn):
         if not line:
             break
         debug2('%s\n' % line)
         if line.startswith('index '):
             idx = line[6:]
             debug1('client: received index suggestion: %s\n' %
                    git.shorten_hash(idx))
             suggested.append(idx)
         else:
             assert (line.endswith('.idx'))
             debug1('client: completed writing pack, idx: %s\n' %
                    git.shorten_hash(line))
             suggested.append(line)
     self.check_ok()
     if ob:
         self._busy = None
     idx = None
     for idx in suggested:
         self.sync_index(idx)
     git.auto_midx(self.cachedir)
     if ob:
         self._busy = ob
         self.conn.write('%s\n' % ob)
     return idx
Ejemplo n.º 4
0
    def sync_indexes(self):
        self.check_busy()
        conn = self.conn
        mkdirp(self.cachedir)
        # All cached idxs are extra until proven otherwise
        extra = set()
        for f in os.listdir(self.cachedir):
            debug1('%s\n' % f)
            if f.endswith('.idx'):
                extra.add(f)
        needed = set()
        conn.write('list-indexes\n')
        for line in linereader(conn):
            if not line:
                break
            assert (line.find('/') < 0)
            parts = line.split(' ')
            idx = parts[0]
            if len(parts) == 2 and parts[1] == 'load' and idx not in extra:
                # If the server requests that we load an idx and we don't
                # already have a copy of it, it is needed
                needed.add(idx)
            # Any idx that the server has heard of is proven not extra
            extra.discard(idx)

        self.check_ok()
        debug1('client: removing extra indexes: %s\n' % extra)
        for idx in extra:
            os.unlink(os.path.join(self.cachedir, idx))
        debug1('client: server requested load of: %s\n' % needed)
        for idx in needed:
            self.sync_index(idx)
        git.auto_midx(self.cachedir)
Ejemplo n.º 5
0
Archivo: client.py Proyecto: jmberg/bup
    def sync_indexes(self):
        conn = self.conn
        mkdirp(self.cachedir)
        # All cached idxs are extra until proven otherwise
        extra = set()
        for f in os.listdir(self.cachedir):
            debug1(path_msg(f) + '\n')
            if f.endswith(b'.idx'):
                extra.add(f)
        needed = set()
        for idx, load in self._list_indexes():
            if load:
                # If the server requests that we load an idx and we don't
                # already have a copy of it, it is needed
                needed.add(idx)
            # Any idx that the server has heard of is proven not extra
            extra.discard(idx)

        debug1('client: removing extra indexes: %s\n' % extra)
        for idx in extra:
            os.unlink(os.path.join(self.cachedir, idx))
        debug1('client: server requested load of: %s\n' % needed)
        for idx in needed:
            self.sync_index(idx)
        git.auto_midx(self.cachedir)
Ejemplo n.º 6
0
Archivo: tgit.py Proyecto: zzmjohn/bup
def test_midx_close():
    fddir = b'/proc/self/fd'
    try:
        os.listdir(fddir)
    except Exception:
        # not supported, not Linux, I guess
        return

    def openfiles():
        for fd in os.listdir(fddir):
            try:
                yield os.readlink(os.path.join(fddir, fd))
            except OSError:
                pass

    def force_midx(objdir):
        args = [path.exe(), b'midx', b'--auto', b'--dir', objdir]
        check_call(args)

    with no_lingering_errors(), \
         test_tempdir(b'bup-tgit-') as tmpdir:
        environ[b'BUP_DIR'] = bupdir = tmpdir + b'/bup'
        git.init_repo(bupdir)
        # create a few dummy idxes
        for i in range(10):
            _create_idx(tmpdir, i)
        git.auto_midx(tmpdir)
        l = git.PackIdxList(tmpdir)
        # this doesn't exist (yet)
        WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
        for i in range(10, 15):
            _create_idx(tmpdir, i)
        # delete the midx ...
        # TODO: why do we need to? git.auto_midx() below doesn't?!
        for fn in os.listdir(tmpdir):
            if fn.endswith(b'.midx'):
                os.unlink(os.path.join(tmpdir, fn))
        # and make a new one
        git.auto_midx(tmpdir)
        # check it still doesn't exist - we haven't refreshed
        WVPASSEQ(None, l.exists(struct.pack('18xBB', 10, 0)))
        # check that we still have the midx open, this really
        # just checks more for the kernel API ('deleted' string)
        for fn in openfiles():
            if not b'midx-' in fn:
                continue
            WVPASSEQ(True, b'deleted' in fn)
        # refresh the PackIdxList
        l.refresh()
        # and check that an object in pack 10 exists now
        WVPASSEQ(True, l.exists(struct.pack('18xBB', 10, 0)))
        for fn in openfiles():
            if not b'midx-' in fn:
                continue
            # check that we don't have it open anymore
            WVPASSEQ(False, b'deleted' in fn)
Ejemplo n.º 7
0
 def finish_writing(self, run_midx=True):
     if self.meta_writer != self.data_writer and self.meta_writer is not None:
         self._finish(self.meta_writer, self.meta_fakesha, meta=True)
         self.meta_writer = None
     if self.data_writer is not None:
         self._finish(self.data_writer, self.data_fakesha)
         if self.meta_writer == self.data_writer:
             self.meta_writer = None
         self.data_writer = None
     if run_midx:
         git.auto_midx(self.cachedir)
Ejemplo n.º 8
0
    def _synchronize_idxes(self):
        changes = False
        local_idxes = set(fnmatch.filter(os.listdir(self.cachedir), b'*.idx'))
        for remote_idx in self.storage.list(b'*.encidx'):
            local_idx = remote_idx.replace(b'.encidx', b'.idx')
            if local_idx in local_idxes:
                local_idxes.remove(local_idx)
            else:
                ec = self._open_read(remote_idx, Kind.IDX)
                with open(os.path.join(self.cachedir, local_idx), 'wb') as f:
                    f.write(ec.read()[1])
                changes = True
        for local_idx in local_idxes:
            changes = True
            os.unlink(os.path.join(self.cachedir, local_idx))

        if changes:
            git.auto_midx(self.cachedir)
Ejemplo n.º 9
0
Archivo: client.py Proyecto: leto/bup
 def sync_index(self, name):
     #debug1('requesting %r\n' % name)
     self.check_busy()
     mkdirp(self.cachedir)
     self.conn.write('send-index %s\n' % name)
     n = struct.unpack('!I', self.conn.read(4))[0]
     assert(n)
     fn = os.path.join(self.cachedir, name)
     f = open(fn + '.tmp', 'w')
     count = 0
     progress('Receiving index from server: %d/%d\r' % (count, n))
     for b in chunkyreader(self.conn, n):
         f.write(b)
         count += len(b)
         progress('Receiving index from server: %d/%d\r' % (count, n))
     progress('Receiving index from server: %d/%d, done.\n' % (count, n))
     self.check_ok()
     f.close()
     os.rename(fn + '.tmp', fn)
     git.auto_midx(self.cachedir)