Example #1
0
 def test_readall(self):
     tstr = '1234567890'
     with TemporaryFile() as f:
         f.write(tstr)
         f.flush()
         f.seek(0)
         self.assertEqual(utils.readall(f, 5), tstr[:5])
         self.assertEqual(utils.readall(f, 10), tstr[5:])
         self.assertEqual(utils.readall(f, 1), '')
         self.assertRaises(IOError, utils.readall, f, 1, 0)
Example #2
0
 def test_readall(self):
     tstr = '1234567890'
     with TemporaryFile() as f:
         f.write(tstr)
         f.flush()
         f.seek(0)
         self.assertEqual(utils.readall(f, 5), tstr[:5])
         self.assertEqual(utils.readall(f, 10), tstr[5:])
         self.assertEqual(utils.readall(f, 1), '')
         self.assertRaises(IOError, utils.readall, f, 1, 0)
Example #3
0
    def upload_object_unchunked(
            self, obj, f,
            withHashFile=False,
            size=None,
            etag=None,
            content_encoding=None,
            content_disposition=None,
            content_type=None,
            sharing=None,
            public=None):
        """
        :param obj: (str) remote object path

        :param f: open file descriptor

        :param withHashFile: (bool)

        :param size: (int) size of data to upload

        :param etag: (str)

        :param content_encoding: (str)

        :param content_disposition: (str)

        :param content_type: (str)

        :param sharing: {'read':[user and/or grp names],
            'write':[usr and/or grp names]}

        :param public: (bool)

        :returns: (dict) created object metadata
        """
        self._assert_container()

        if withHashFile:
            data = f.read()
            try:
                import json
                data = json.dumps(json.loads(data))
            except ValueError:
                raise ClientError('"%s" is not json-formated' % f.name, 1)
            except SyntaxError:
                msg = '"%s" is not a valid hashmap file' % f.name
                raise ClientError(msg, 1)
            f = StringIO(data)
        else:
            data = readall(f, size) if size else f.read()
        r = self.object_put(
            obj,
            data=data,
            etag=etag,
            content_encoding=content_encoding,
            content_disposition=content_disposition,
            content_type=content_type,
            permissions=sharing,
            public=public,
            success=201)
        return r.headers
Example #4
0
    def _calculate_blocks_for_upload(self, blocksize, blockhash, size, nblocks, hashes, hmap, fileobj, hash_cb=None):
        offset = 0
        if hash_cb:
            hash_gen = hash_cb(nblocks)
            hash_gen.next()

        for i in xrange(nblocks):
            block = readall(fileobj, min(blocksize, size - offset))
            bytes = len(block)
            if bytes <= 0:
                break
            hash = _pithos_hash(block, blockhash)
            hashes.append(hash)
            hmap[hash] = (offset, bytes)
            offset += bytes
            if hash_cb:
                hash_gen.next()
        msg = "Failed to calculate uploading blocks: " "read bytes(%s) != requested size (%s)" % (offset, size)
        assert offset == size, msg
Example #5
0
    def _upload_missing_blocks(self, missing, hmap, fileobj, upload_gen=None):
        """upload missing blocks asynchronously"""

        self._init_thread_limit()

        flying = []
        failures = []
        for hash in missing:
            offset, bytes = hmap[hash]
            fileobj.seek(offset)
            data = readall(fileobj, bytes)
            r = self._put_block_async(data, hash)
            flying.append(r)
            unfinished = self._watch_thread_limit(flying)
            for thread in set(flying).difference(unfinished):
                if thread.exception:
                    failures.append(thread)
                    if isinstance(
                            thread.exception,
                            ClientError) and thread.exception.status == 502:
                        self.POOLSIZE = self._thread_limit
                elif thread.isAlive():
                    flying.append(thread)
                elif upload_gen:
                    try:
                        upload_gen.next()
                    except:
                        pass
            flying = unfinished

        for thread in flying:
            thread.join()
            if thread.exception:
                failures.append(thread)
            elif upload_gen:
                try:
                    upload_gen.next()
                except:
                    pass

        return [failure.kwargs['hash'] for failure in failures]
Example #6
0
 def _hash_from_file(self, fp, start, size, blockhash):
     fp.seek(start)
     block = readall(fp, size)
     h = newhashlib(blockhash)
     h.update(block.strip('\x00'))
     return hexlify(h.digest())