Ejemplo n.º 1
0
 def account_update(self, account, metadata, to_delete=None, headers=None):
     uri = "/v1.0/account/update"
     account_id = utils.quote(account, '')
     params = {'id': account_id}
     data = json.dumps({"metadata": metadata, "to_delete": to_delete})
     resp, resp_body = self._account_request('POST', uri, params=params,
                                             data=data, headers=headers)
Ejemplo n.º 2
0
 def _connect_put(chunk):
     raw_url = chunk["url"]
     parsed = urlparse(raw_url)
     try:
         chunk_path = parsed.path.split('/')[-1]
         hdrs = {}
         hdrs["transfer-encoding"] = "chunked"
         hdrs[chunk_headers["content_id"]] = sysmeta['id']
         hdrs[chunk_headers["content_version"]] = sysmeta['version']
         hdrs[chunk_headers["content_path"]] = utils.quote(obj_name)
         hdrs[chunk_headers["content_size"]] = sysmeta['content_length']
         hdrs[chunk_headers["content_chunkmethod"]] = \
             sysmeta['chunk_method']
         hdrs[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
         hdrs[chunk_headers["content_policy"]] = sysmeta['policy']
         hdrs[chunk_headers["content_chunksnb"]] = len(chunks)
         hdrs[chunk_headers["container_id"]] = \
             utils.name2cid(account, container)
         hdrs[chunk_headers["chunk_pos"]] = chunk["pos"]
         hdrs[chunk_headers["chunk_id"]] = chunk_path
         with ConnectionTimeout(CONNECTION_TIMEOUT):
             conn = http_connect(
                 parsed.netloc, 'PUT', parsed.path, hdrs)
             conn.chunk = chunk
         return conn
     except (Exception, Timeout):
         pass
Ejemplo n.º 3
0
 def account_show(self, account, headers=None):
     uri = "/v1.0/account/show"
     account_id = utils.quote(account, '')
     params = {'id': account_id}
     resp, resp_body = self._account_request('GET', uri, params=params,
                                             headers=headers)
     return resp_body
Ejemplo n.º 4
0
 def _connect_put(chunk):
     raw_url = chunk["url"]
     parsed = urlparse(raw_url)
     try:
         chunk_path = parsed.path.split('/')[-1]
         h = {}
         h["transfer-encoding"] = "chunked"
         h[chunk_headers["content_id"]] = self.sysmeta['id']
         h[chunk_headers["content_version"]] = self.sysmeta['version']
         h[chunk_headers["content_path"]] = \
             utils.quote(self.sysmeta['content_path'])
         h[chunk_headers["content_chunkmethod"]] = \
             self.sysmeta['chunk_method']
         h[chunk_headers["content_policy"]] = self.sysmeta['policy']
         h[chunk_headers["container_id"]] = self.sysmeta['container_id']
         h[chunk_headers["chunk_pos"]] = chunk["pos"]
         h[chunk_headers["chunk_id"]] = chunk_path
         with ConnectionTimeout(io.CONNECTION_TIMEOUT):
             conn = io.http_connect(
                 parsed.netloc, 'PUT', parsed.path, h)
             conn.chunk = chunk
         return conn, chunk
     except (Exception, Timeout) as e:
         msg = str(e)
         logger.error("Failed to connect to %s (%s)", chunk, msg)
         chunk['error'] = msg
         return None, chunk
Ejemplo n.º 5
0
 def account_create(self, account, headers=None):
     uri = '/v1.0/account/create'
     account_id = utils.quote(account, '')
     params = {'id': account_id}
     resp, resp_body = self._account_request('PUT', uri, params=params,
                                             headers=headers)
     created = (resp.status_code == 201)
     return created
Ejemplo n.º 6
0
    def container_list(self, account, limit=None, marker=None,
                       end_marker=None, prefix=None, delimiter=None,
                       headers=None):
        uri = "v1.0/account/containers"
        account_id = utils.quote(account, '')
        params = {"id": account_id, "limit": limit, "marker": marker,
                  "delimiter": delimiter, "prefix": prefix,
                  "end_marker": end_marker}

        resp, resp_body = self._account_request(
            'GET', uri, params=params, headers=headers)
        listing = resp_body['listing']
        del resp_body['listing']
        return listing, resp_body
Ejemplo n.º 7
0
    def _put_stream_rain(self, account, container, obj_name, src, sysmeta,
                         chunks, headers=None):
        global_checksum = hashlib.md5()
        total_bytes_transferred = 0
        content_chunks = []
        content_length = sysmeta['content_length']

        def _encode_rawxlist(chunks_at_pos):
            res_chunks = []
            for subpos, c in chunks_at_pos.iteritems():
                host = c['url'].split('/')[2]
                chunk_id = c['url'].split('/')[-1]
                res_chunks.append("%s/%s" % (host, chunk_id))
            return '|'.join(res_chunks)

        def _limit_stream(stream, size):
            read_size = 0
            while read_size < size:
                to_read = size - read_size
                if to_read > WRITE_CHUNK_SIZE:
                    to_read = WRITE_CHUNK_SIZE
                data = stream.read(to_read)
                global_checksum.update(data)
                read_size += to_read
                yield data

        def _decode_chunklist(chunklist):
            res = []
            for c in chunklist.split(';'):
                pos, url, size, hash = c.split('|')
                res.append({
                    "url": "http://%s" % url,
                    "pos": pos,
                    "size": int(size),
                    "hash": hash
                })
            return res

        for pos in xrange(len(chunks)):
            rainx_url = self._get_service_url("rainx")

            chunk_size = chunks[pos][str(0)]['size']
            remaining_bytes = content_length - total_bytes_transferred
            if chunk_size > remaining_bytes:
                chunk_size = remaining_bytes

            headers = {}
            headers["X-oio-chunk-meta-content-storage-policy"] = \
                sysmeta['policy']
            headers["X-oio-chunk-meta-rawxlist"] = \
                _encode_rawxlist(chunks[pos])
            headers[chunk_headers["content_id"]] = sysmeta['id']
            headers[chunk_headers["content_version"]] = sysmeta['version']
            headers[chunk_headers["content_path"]] = utils.quote(obj_name)
            headers[chunk_headers["content_size"]] = sysmeta['content_length']
            headers[chunk_headers["content_chunksnb"]] = len(chunks)
            headers[chunk_headers["container_id"]] = \
                utils.name2cid(account, container)
            headers[chunk_headers["chunk_pos"]] = pos
            headers[chunk_headers["chunk_size"]] = chunk_size
            headers[chunk_headers["content_chunkmethod"]] = \
                sysmeta['chunk_method']
            headers[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
            headers[chunk_headers["content_policy"]] = sysmeta['policy']

            resp = self.session.put(rainx_url,
                                    data=_limit_stream(src, chunk_size),
                                    headers=headers)
            resp.raise_for_status()

            content_chunks.extend(_decode_chunklist(resp.headers['chunklist']))

            total_bytes_transferred += chunk_size

        content_checksum = global_checksum.hexdigest()

        return content_chunks, total_bytes_transferred, content_checksum
Ejemplo n.º 8
0
 def account_delete(self, account, headers=None):
     uri = '/v1.0/account/delete'
     account_id = utils.quote(account, '')
     params = {'id': account_id}
     resp, resp_body = self._account_request('POST', uri, params=params,
                                             headers=headers)