예제 #1
0
 def _deleteFile(self, request, filekey, metakey, reqKu, reqID):
     fname = os.path.join(self.config.storedir, filekey)
     loggerdele.debug("reading file data from %s" % fname)
     if not os.path.exists(fname):
         # check for tarball for originator
         tarballs = []
         tarballbase = os.path.join(self.config.storedir, reqKu.id())+".tar"
         if os.path.exists(tarballbase+".gz"):
             tarballs.append((tarballbase+".gz", 'r:gz'))
         if os.path.exists(tarballbase):
             tarballs.append((tarballbase, 'r'))
         for tarball, openmode in tarballs:
             mfilekey = "%s.%s.meta" % (filekey, metakey)
             loggerdele.debug("opening %s, %s for delete..." 
                     % (tarball, openmode))
             ftype = os.popen('file %s' % tarball)
             loggerdele.debug("ftype of %s is %s" % (tarball, ftype.read()))
             ftype.close()
             tar = tarfile.open(tarball, openmode)
             mnames = [n for n in tar.getnames() 
                     if n[:len(filekey)] == filekey]
             tar.close()
             if len(mnames) > 2:
                 deleted = TarfileUtils.delete(tarball, mfilekey)
             else:
                 deleted = TarfileUtils.delete(tarball, [filekey, mfilekey])
             if deleted:
                 loggerdele.info("DELETED %s (from %s)" % (deleted, tarball))
             return ""
         request.setResponseCode(http.NOT_FOUND, "Not found: %s" % filekey)
         request.write("Not found: %s" % filekey)
     else:
         f = BlockFile.open(fname,"rb+")
         nID = int(reqID, 16)
         if f.hasNode(nID):
             # remove this node/metakey from owning this file block
             f.delNode(nID, metakey)
             if f.emptyNodes():
                 # if this was the only owning node, delete it
                 f.close()
                 os.remove(fname)
         f.close()
         loggerdele.debug("returning DELETE response")
         return ""
예제 #2
0
    def _sendFile(self, request, filekey, reqKu, returnMeta):
        fname = os.path.join(self.config.storedir,filekey)
        loggerretr.debug("reading file data from %s" % fname)
        # XXX: make sure requestor owns the file? 
        if returnMeta:
            loggerretr.debug("returnMeta = %s" % returnMeta)
            request.setHeader('Content-type', 'Multipart/Related')
            rand_bound = binascii.hexlify(generateRandom(13))
            request.setHeader('boundary', rand_bound)
        if not os.path.exists(fname):
            # check for tarball for originator
            tarball = os.path.join(self.config.storedir,reqKu.id()+".tar")
            tarballs = []
            if os.path.exists(tarball+'.gz'):
                tarballs.append((tarball+'.gz', 'r:gz'))
            if os.path.exists(tarball):
                tarballs.append((tarball, 'r'))
            loggerretr.debug("tarballs = %s" % tarballs)
            # XXX: does this work? does it close both tarballs if both got
            # opened?
            for tarball, openmode in tarballs:
                tar = tarfile.open(tarball, openmode)
                try:
                    tinfo = tar.getmember(filekey)
                    returnedMeta = False
                    if returnMeta:
                        loggerretr.debug("tar returnMeta %s" % filekey)
                        try:
                            metas = [f for f in tar.getnames()
                                        if f[:len(filekey)] == filekey 
                                        and f[-4:] == 'meta']
                            loggerretr.debug("tar returnMetas=%s" % metas)
                            for m in metas:
                                minfo = tar.getmember(m)
                                H = []
                                H.append("--%s" % rand_bound)
                                H.append("Content-Type: "
                                        "Application/octet-stream")
                                H.append("Content-ID: %s" % m)
                                H.append("Content-Length: %d" % minfo.size)
                                H.append("")
                                H = '\r\n'.join(H)
                                request.write(H)
                                request.write('\r\n')
                                tarm = tar.extractfile(minfo)
                                loggerretr.debug("successful metadata"
                                    " RETRIEVE (from %s)" % tarball)
                                # XXX: bad blocking stuff
                                while 1:
                                    buf = tarm.read()
                                    if buf == "":
                                        break
                                    request.write(buf)
                                    request.write('\r\n')
                                tarm.close()

                            H = []
                            H.append("--%s" % rand_bound)
                            H.append("Content-Type: Application/octet-stream")
                            H.append("Content-ID: %s" % filekey)
                            H.append("Content-Length: %d" % tinfo.size)
                            H.append("")
                            H = '\r\n'.join(H)
                            request.write(H)
                            request.write('\r\n')
                            returnedMeta = True
                        except:
                            # couldn't find any metadata, just return normal
                            # file
                            loggerretr.debug("no metadata found")
                            pass
                    # XXX: bad blocking stuff
                    tarf = tar.extractfile(tinfo)
                    # XXX: update timestamp on tarf in tarball
                    loggerretr.debug("successful RETRIEVE (from %s)" 
                            % tarball)
                    # XXX: bad blocking stuff
                    while 1:
                        buf = tarf.read()
                        if buf == "":
                            break
                        request.write(buf)
                    tarf.close()
                    tar.close()
                    if returnedMeta:
                        T = []
                        T.append("")
                        T.append("--%s--" % rand_bound)
                        T.append("")
                        T = '\r\n'.join(T)
                        request.write(T)
                    return ""
                except:
                    tar.close()
            request.setResponseCode(http.NOT_FOUND, "Not found: %s" % filekey)
            request.write("Not found: %s" % filekey)
        else:
            f = BlockFile.open(fname,"rb")
            loggerretr.log(logging.INFO, "successful RETRIEVE for %s" % filekey)
            meta = f.meta(int(reqKu.id(),16))
            if returnMeta and meta:
                loggerretr.debug("returnMeta %s" % filekey)
                loggerretr.debug("returnMetas=%s" % meta)
                for m in meta:
                    H = []
                    H.append("--%s" % rand_bound)
                    H.append("Content-Type: Application/octet-stream")
                    H.append("Content-ID: %s.%s.meta" % (filekey, m))
                    H.append("Content-Length: %d" % len(meta[m]))
                    H.append("")
                    H.append(meta[m])
                    H = '\r\n'.join(H)
                    request.write(H)
                    request.write('\r\n')

                H = []
                H.append("--%s" % rand_bound)
                H.append("Content-Type: Application/octet-stream")
                H.append("Content-ID: %s" % filekey)
                H.append("Content-Length: %d" % f.size())
                H.append("")
                H = '\r\n'.join(H)
                request.write(H)
                request.write('\r\n')
            # XXX: bad blocking stuff
            while 1:
                buf = f.read()
                if buf == "":
                    break
                request.write(buf)
            f.close()
            if returnMeta and meta:
                T = []
                T.append("")
                T.append("--%s--" % rand_bound)
                T.append("")
                request.write('\r\n'.join(T))
            return ""
예제 #3
0
    def _sendVerify(self, request, filekey, offset, length, reqKu, nodeID, 
            meta):
        fname = os.path.join(self.config.storedir,filekey)
        loggervrfy.debug("request for %s" % fname)
        if os.path.exists(fname):
            loggervrfy.debug("looking in regular blockfile for %s" % fname)
            if meta:
                f = BlockFile.open(fname, 'rb+')
            else:
                f = BlockFile.open(fname, 'rb')
        else:
            # check for tarball for originator
            loggervrfy.debug("checking tarball for %s" % fname)
            tarballs = []
            tarballbase = os.path.join(self.config.storedir, reqKu.id())+".tar"
            if os.path.exists(tarballbase+".gz"):
                tarballs.append((tarballbase+".gz", 'r:gz'))
            if os.path.exists(tarballbase):
                tarballs.append((tarballbase, 'r'))
            loggervrfy.debug("tarballs is %s" % tarballs)
            for tarball, openmode in tarballs:
                loggervrfy.debug("looking in tarball %s..." % tarball)
                tar = tarfile.open(tarball, openmode)
                try:
                    tarf = tar.extractfile(filekey)
                    tari = tar.getmember(filekey)
                    # XXX: update timestamp on tarf in tarball
                    fsize = tari.size
                    if offset > fsize or (offset+length) > fsize:
                        # XXX: should limit length
                        loggervrfy.debug("VERIFY response failed (from %s):"
                                " bad offset/length" % tarball)
                        msg = "Bad request: bad offset/length in VERIFY"
                        request.setResponseCode(http.BAD_REQUEST, msg) 
                        return msg
                    # XXX: could avoid seek/read if length == 0
                    tarf.seek(offset)
                    # XXX: bad blocking read
                    data = tarf.read(length)
                    tarf.close()
                    if meta:
                        mfname = "%s.%s.meta" % (filekey, meta[0])
                        loggervrfy.debug("looking for %s" % mfname)
                        if mfname in tar.getnames():
                            # make sure that the data is the same, if not,
                            # remove it and re-add it
                            tarmf = tar.extractfile(mfname)
                            # XXX: bad blocking read
                            stored_meta = tarmf.read()
                            tarmf.close()
                            if meta[1] != stored_meta:
                                loggervrfy.debug("updating tarball"
                                        " metadata for %s.%s" 
                                        % (filekey, meta[0]))
                                tar.close()
                                TarfileUtils.delete(tarball, mfname)
                                if openmode == 'r:gz':
                                    tarball = TarfileUtils.gunzipTarball(
                                            tarball)
                                tar = tarfile.open(tarball, 'a')
                                metaio = StringIO(meta[1])
                                tinfo = tarfile.TarInfo(mfname)
                                tinfo.size = len(meta[1])
                                tar.addfile(tinfo, metaio)
                                tar.close()
                                if openmode == 'r:gz':
                                    tarball = TarfileUtils.gzipTarball(
                                            tarball)
                            else:
                                loggervrfy.debug("no need to update tarball"
                                        " metadata for %s.%s" 
                                        % (filekey, meta[0]))
                        else:
                            # add it
                            loggervrfy.debug("adding tarball metadata"
                                    " for %s.%s" % (filekey, meta[0]))
                            tar.close()
                            if openmode == 'r:gz':
                                tarball = TarfileUtils.gunzipTarball(tarball)
                            tar = tarfile.open(tarball, 'a')
                            metaio = StringIO(meta[1])
                            tinfo = tarfile.TarInfo(mfname)
                            tinfo.size = len(meta[1])
                            tar.addfile(tinfo, metaio)
                            tar.close()
                            if openmode == 'r:gz':
                                tarball = TarfileUtils.gzipTarball(
                                        tarball)
                        
                    tar.close()
                    hash = hashstring(data)
                    loggervrfy.info("successful VERIFY (from %s)" % tarball)
                    return hash
                except:
                    tar.close()
            loggervrfy.debug("requested file %s doesn't exist" % fname)
            msg = "Not found: not storing %s" % filekey
            request.setResponseCode(http.NOT_FOUND, msg)
            return msg

        # make sure request is reasonable   
        fsize = os.stat(fname)[stat.ST_SIZE]
        if offset > fsize or (offset+length) > fsize:
            # XXX: should limit length
            loggervrfy.debug("VERIFY response failed (bad offset/length)")
            msg = "Bad request: bad offset/length in VERIFY"
            request.setResponseCode(http.BAD_REQUEST, msg) 
            return msg
        else:
            # XXX: blocking
            # XXX: could avoid seek/read if length == 0 (noop for meta update)
            f.seek(offset)
            data = f.read(length)
            if meta:
                loggervrfy.debug("adding metadata for %s.%s" 
                        % (fname, meta[0]))
                f.addNode(int(nodeID, 16) , {meta[0]: meta[1]})
            # XXX: blocking
            f.close()
            hash = hashstring(data)
            loggervrfy.debug("returning VERIFY")
            return hash
예제 #4
0
    def _storeFile(self, request, filekey, reqKu, nodeID):
        # [XXX: memory management is not happy here.  might want to look at
        # request.registerProducer().  Otherwise, might have to scrap
        # using the STORE(ROOT(RESOURCE)) deal in favor of
        # producer/consumer model for STORE ops
        # (http://itamarst.org/writings/OSCON03/twisted_internet-108.html).
        # Another option might include subclassing web.resource.Resource
        # and making this derive from that...  Or might be web.Site that
        # needs to be subclassed... Or maybe web.site.Request -
        # web.site.Request.process()?  Request seems doubly-bad: perhaps a
        # copy is made somewhere, because memory mushrooms to 2x big
        # upload, then goes back down to around 1x.
        # [update: This should be fixable in twisted.web2, but I am informed
        # that in the current version, there is no workaround]

        # get the data to a tmp file
        loggerstor.debug("writing store data to tmpfile")
        tmpfile = tempfile.mktemp(dir=self.config.storedir)

        tarball = os.path.join(self.config.storedir,reqKu.id()+".tar")

        # rename and/or prepend the data appropriately
        tmpTarMode = None
        if filekey[-4:] == ".tar":
            tmpfile = tmpfile+".tar"
            tmpTarMode = 'r'
            targetTar = tarball
        elif filekey[-7:] == ".tar.gz":
            tmpfile = tmpfile+".tar.gz"
            tmpTarMode = 'r:gz'
            targetTar = tarball+".gz"
        loggerstor.debug("tmpfile is %s" % tmpfile)

        # XXX: if the server supports both .tar and tar.gz, this is wrong; we'd
        # need to check *both* for already existing dudes instead of just
        # choosing one
        if os.path.exists(tarball+'.gz'):
            tarball = (tarball+'.gz', 'r:gz')
        elif os.path.exists(tarball):
            tarball = (tarball, 'r')
        else:
            tarball = None
        loggerstor.debug("tarball is %s" % str(tarball))

        data = request.args.get('filename')[0]  # XXX: file in mem! need web2.
        # XXX: bad blocking stuff here
        f = open(tmpfile, 'wb')
        f.write(data)
        f.close()
        ftype = os.popen('file %s' % tmpfile)
        loggerstor.debug("ftype of %s is %s" % (tmpfile, ftype.read()))
        ftype.close()

        if tmpTarMode:
            # client sent a tarball
            loggerstor.debug("about to chksum %s" % tmpfile)
            digests = TarfileUtils.verifyHashes(tmpfile, '.meta')
            loggerstor.debug("chksum returned %s" % digests)
            ftype = os.popen('file %s' % tmpfile)
            loggerstor.debug("ftype of %s is %s" % (tmpfile, ftype.read()))
            ftype.close()
            if not digests:
                msg = "Attempted to use non-CAS storage key(s) for" \
                        " STORE tarball"
                loggerstor.debug(msg)
                os.remove(tmpfile)
                request.setResponseCode(http.CONFLICT, msg) 
                return msg
            # XXX: add digests to a db of already stored files (for quick 
            # lookup)
            if tarball:
                tarname, tarnameMode = tarball
                loggerstor.debug("concatenating tarfiles %s and %s" 
                        % (tarname, tmpfile))
                f1 = tarfile.open(tarname, tarnameMode)
                f2 = tarfile.open(tmpfile, tmpTarMode)
                f1names = f1.getnames()
                f2names = f2.getnames()
                f1.close()
                f2.close()
                dupes = [f for f in f1names if f in f2names]
                TarfileUtils.delete(tmpfile, dupes)
                ftype = os.popen('file %s' % tarname)
                loggerstor.debug("ftype of %s is %s" % (tarname, ftype.read()))
                ftype.close()
                TarfileUtils.concatenate(tarname, tmpfile)
                ftype = os.popen('file %s' % tarname)
                loggerstor.debug("ftype of %s is %s" % (tarname, ftype.read()))
                ftype.close()
            else:
                loggerstor.debug("saving %s as tarfile %s" % (tmpfile, 
                    targetTar))
                os.rename(tmpfile, targetTar)
        else:
            # client sent regular file
            h = hashfile(tmpfile)
            if request.args.has_key('meta') and request.args.has_key('metakey'):
                metakey = request.args.get('metakey')[0]
                meta = request.args.get('meta')[0]  # XXX: file in mem! 
            else:
                metakey = None
                meta = None
            if fencode(long(h, 16)) != filekey:
                msg = "Attempted to use non-CAS storage key for STORE data "
                msg += "(%s != %s)" % (filekey, fencode(long(h, 16)))
                os.remove(tmpfile)
                request.setResponseCode(http.CONFLICT, msg) 
                return msg
            fname = os.path.join(self.config.storedir, filekey)
            if os.path.exists(fname):
                loggerstor.debug("adding metadata to %s" % fname)
                f = BlockFile.open(fname,'rb+')
                if not f.hasNode(nodeID):
                    f.addNode(int(nodeID,16), {metakey: meta})
                    f.close()
                os.remove(tmpfile)
            else:
                if os.path.exists(nodeID+".tar"):
                    # XXX: need to do something with metadata!
                    print "XXX: need to do something with metadata for tar!"
                    tarball = tarfile.open(tarname, 'r')
                    if fname in tarball.getnames():
                        loggerstor.debug("%s already stored in tarball" % fname)
                        # if the file is already in the corresponding tarball,
                        # update its timestamp and return success.
                        loggerstor.debug("%s already stored" % filekey)
                        # XXX: update timestamp for filekey in tarball
                        return "Successful STORE"
                    else:
                        loggerstor.debug("tarball for %s, but %s not in tarball"
                                % (nodeID,fname))
                if len(data) < 8192 and fname != tarname: #XXX: magic # (blk sz)
                    # If the file is small, move it into the appropriate
                    # tarball.  Note that this code is unlikely to ever be
                    # executed if the client is an official flud client, as
                    # they do the tarball aggregation thing already, and all
                    # tarballs will be > 8192.  This is, then, really just
                    # defensive coding -- clients aren't required to implement
                    # that tarball aggregation strategy.  And it is really only
                    # useful for filesystems with inefficient small file 
                    # storage.
                    loggerstor.debug("moving small file '%s' into tarball" 
                            % fname)
                    if not os.path.exists(tarname):
                        tarball = tarfile.open(tarname, 'w')
                    else:
                        tarball = tarfile.open(tarname, 'a')
                    # XXX: more bad blocking stuff
                    tarball.add(tmpfile, os.path.basename(fname))
                    if meta:
                        metafilename = "%s.%s.meta" % (os.path.basename(fname), 
                                metakey)
                        loggerstor.debug("adding metadata file to tarball %s" 
                                % metafilename)
                        metaio = StringIO(meta)
                        tinfo = tarfile.TarInfo(metafilename)
                        tinfo.size = len(meta)
                        tarball.addfile(tinfo, metaio)
                    tarball.close()
                    os.remove(tmpfile)
                else:
                    # store the file 
                    loggerstor.debug("storing %s" % fname)
                    os.rename(tmpfile, fname)
                    BlockFile.convert(fname, (int(nodeID,16), {metakey: meta}))

        loggerstor.debug("successful STORE for %s" % filekey)
        return "Successful STORE"