def test_http_proxy(http_server, monkeypatch, test_port):
    test_host = 'www.foobarz.invalid'
    test_path = '/someurl?barf'

    get_path = None
    def do_GET(self):
        nonlocal get_path
        get_path = self.path
        self.send_response(200)
        self.send_header("Content-Type", 'application/octet-stream')
        self.send_header("Content-Length", '0')
        self.end_headers()
    monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)

    conn = HTTPConnection(test_host, test_port,
                          proxy=(http_server.host, http_server.port))
    try:
        conn.send_request('GET', test_path)
        resp = conn.read_response()
        assert resp.status == 200
        conn.discard()
    finally:
        conn.disconnect()

    if test_port is None:
        exp_path = 'http://%s%s' % (test_host, test_path)
    else:
        exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path)

    assert get_path == exp_path
def test_http_proxy(http_server, monkeypatch, test_port):
    test_host = 'www.foobarz.invalid'
    test_path = '/someurl?barf'

    get_path = None

    def do_GET(self):
        nonlocal get_path
        get_path = self.path
        self.send_response(200)
        self.send_header("Content-Type", 'application/octet-stream')
        self.send_header("Content-Length", '0')
        self.end_headers()

    monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)

    conn = HTTPConnection(test_host,
                          test_port,
                          proxy=(http_server.host, http_server.port))
    try:
        conn.send_request('GET', test_path)
        resp = conn.read_response()
        assert resp.status == 200
        conn.discard()
    finally:
        conn.disconnect()

    if test_port is None:
        exp_path = 'http://%s%s' % (test_host, test_path)
    else:
        exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path)

    assert get_path == exp_path
示例#3
0
文件: gs.py 项目: Binary-Bit/s3ql
    def _get_access_token(self):
        log.info('Requesting new access token')

        headers = CaseInsensitiveDict()
        headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'

        body = urllib.parse.urlencode({
            'client_id': OAUTH_CLIENT_ID,
            'client_secret': OAUTH_CLIENT_SECRET,
            'refresh_token': self.password,
            'grant_type': 'refresh_token' })

        conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy,
                              ssl_context=self.ssl_context)
        try:

            conn.send_request('POST', '/o/oauth2/token', headers=headers,
                              body=body.encode('utf-8'))
            resp = conn.read_response()
            json_resp = self._parse_json_response(resp, conn)

            if resp.status > 299 or resp.status < 200:
                assert 'error' in json_resp
            if 'error' in json_resp:
                raise AuthenticationError(json_resp['error'])
            else:
                self.access_token[self.password] = json_resp['access_token']
        finally:
            conn.disconnect()
def test_invalid_ssl():
    check_http_connection()

    # Don't load certificates
    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
    context.options |= ssl.OP_NO_SSLv2
    context.verify_mode = ssl.CERT_REQUIRED

    conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context)
    with pytest.raises(ssl.SSLError):
        conn.send_request('GET', '/')
    conn.disconnect()
示例#5
0
文件: gs.py 项目: naeloob/s3ql
    def _get_access_token(self):
        log.info('Requesting new access token')

        headers = CaseInsensitiveDict()
        headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'

        body = urlencode({'client_id': OAUTH_CLIENT_ID,
                          'client_secret': OAUTH_CLIENT_SECRET,
                          'refresh_token': self.password,
                          'grant_type': 'refresh_token' })

        conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy,
                              ssl_context=self.ssl_context)
        try:

            conn.send_request('POST', '/o/oauth2/token', headers=headers,
                              body=body.encode('utf-8'))
            resp = conn.read_response()

            if resp.status > 299 or resp.status < 200:
                raise HTTPError(resp.status, resp.reason, resp.headers)

            content_type = resp.headers.get('Content-Type', None)
            if content_type:
                hit = re.match(r'application/json(?:; charset="(.+)")?$',
                               resp.headers['Content-Type'], re.IGNORECASE)
            else:
                hit = None

            if not hit:
                log.error('Unexpected server reply when refreshing access token:\n%s',
                          self._dump_response(resp))
                raise RuntimeError('Unable to parse server response')

            charset = hit.group(1) or 'utf-8'
            body = conn.readall().decode(charset)
            resp_json = json.loads(body)

            if not isinstance(resp_json, dict):
                log.error('Invalid json server response. Expected dict, got:\n%s', body)
                raise RuntimeError('Unable to parse server response')

            if 'error' in resp_json:
                raise AuthenticationError(resp_json['error'])

            if 'access_token' not in resp_json:
                log.error('Unable to find access token in server response:\n%s', body)
                raise RuntimeError('Unable to parse server response')

            self.access_token[self.password] = resp_json['access_token']

        finally:
            conn.disconnect()
def test_invalid_ssl():
    check_http_connection()

    # Don't load certificates
    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
    context.options |= ssl.OP_NO_SSLv2
    context.verify_mode = ssl.CERT_REQUIRED

    conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context)
    with pytest.raises(ssl.SSLError):
        conn.send_request('GET', '/')
    conn.disconnect()
def test_connect_proxy(http_server, monkeypatch, test_port):
    test_host = 'www.foobarz.invalid'
    test_path = '/someurl?barf'

    connect_path = None

    def do_CONNECT(self):
        # Pretend we're the remote server too
        nonlocal connect_path
        connect_path = self.path
        self.send_response(200)
        self.end_headers()
        self.close_connection = 0

    monkeypatch.setattr(MockRequestHandler,
                        'do_CONNECT',
                        do_CONNECT,
                        raising=False)

    get_path = None

    def do_GET(self):
        nonlocal get_path
        get_path = self.path
        self.send_response(200)
        self.send_header("Content-Type", 'application/octet-stream')
        self.send_header("Content-Length", '0')
        self.end_headers()

    monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)

    # We don't *actually* want to establish SSL, that'd be
    # to complex for our mock server
    monkeypatch.setattr('ssl.match_hostname', lambda x, y: True)
    conn = HTTPConnection(test_host,
                          test_port,
                          proxy=(http_server.host, http_server.port),
                          ssl_context=FakeSSLContext())
    try:
        conn.send_request('GET', test_path)
        resp = conn.read_response()
        assert resp.status == 200
        conn.discard()
    finally:
        conn.disconnect()

    if test_port is None:
        test_port = 443
    exp_path = '%s:%d' % (test_host, test_port)
    assert connect_path == exp_path
    assert get_path == test_path
def test_connect_ssl():
    check_http_connection()

    ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
    ssl_context.options |= ssl.OP_NO_SSLv2
    ssl_context.verify_mode = ssl.CERT_REQUIRED
    ssl_context.set_default_verify_paths()

    conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context)
    conn.send_request('GET', '/')
    resp = conn.read_response()
    assert resp.status in (200, 301, 302)
    assert resp.path == '/'
    conn.discard()
    conn.disconnect()
def test_connect_ssl():
    check_http_connection()

    ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
    ssl_context.options |= ssl.OP_NO_SSLv2
    ssl_context.verify_mode = ssl.CERT_REQUIRED
    ssl_context.set_default_verify_paths()

    conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context)
    conn.send_request('GET', '/')
    resp = conn.read_response()
    assert resp.status in (200, 301, 302)
    assert resp.path == '/'
    conn.discard()
    conn.disconnect()
示例#10
0
def test_connect_proxy(http_server, monkeypatch, test_port):
    test_host = 'www.foobarz.invalid'
    test_path = '/someurl?barf'

    connect_path = None
    def do_CONNECT(self):
        # Pretend we're the remote server too
        nonlocal connect_path
        connect_path = self.path
        self.send_response(200)
        self.end_headers()
        self.close_connection = 0
    monkeypatch.setattr(MockRequestHandler, 'do_CONNECT',
                        do_CONNECT, raising=False)

    get_path = None
    def do_GET(self):
        nonlocal get_path
        get_path = self.path
        self.send_response(200)
        self.send_header("Content-Type", 'application/octet-stream')
        self.send_header("Content-Length", '0')
        self.end_headers()
    monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET)

    # We don't *actually* want to establish SSL, that'd be
    # to complex for our mock server
    monkeypatch.setattr('ssl.match_hostname', lambda x,y: True)
    conn = HTTPConnection(test_host, test_port,
                          proxy=(http_server.host, http_server.port),
                          ssl_context=FakeSSLContext())
    try:
        conn.send_request('GET', test_path)
        resp = conn.read_response()
        assert resp.status == 200
        conn.discard()
    finally:
        conn.disconnect()

    if test_port is None:
        test_port = 443
    exp_path = '%s:%d' % (test_host, test_port)
    assert connect_path == exp_path
    assert get_path == test_path
示例#11
0
文件: backblaze.py 项目: ethus3h/s3ql
class Backend(AbstractBackend, metaclass=ABCDocstMeta):
    needs_login = True

    known_options = {'test-string', 'ssl-ca-path'}

    authorize_url = "/b2api/v1/b2_authorize_account"
    authorize_hostname = "api.backblaze.com"
    hdr_prefix = 'X-Bz-Info-'

    # no chunksize limit on backend, so we try to limit the number of meta headers
    hdr_chunksize = 2000

    _add_meta_headers_s3 = s3c.Backend._add_meta_headers
    _extractmeta_s3 = s3c.Backend._extractmeta

    def __init__(self, storage_url, login, password, options):
        super().__init__()
        self.options = options
        self.bucket_name = None
        self.prefix = None
        self.auth_token = None
        self.api_host = None
        self.download_host = None
        self.conn_api = None
        self.conn_download = None
        self.account_id = login
        self.account_key = password
        self.bucket_id = None
        self.bucket_name = None

        self._parse_storage_url(storage_url)

        # Add test header povided to all requests.
        if options.get('test-string'):
            # sanitize entry string
            self.test_string = ''.join([
                x for x in options.get('test-string')
                if x in 'abcdefghijklmnopqrstuvwxyz_'
            ])
        else:
            self.test_string = None

        self.ssl_context = get_ssl_context(options.get('ssl-ca-path', None))

        self._login()

        self._connect_bucket(self.bucket_name)

    def _do_request(self,
                    method,
                    path,
                    conn,
                    headers=None,
                    body=None,
                    auth_token=None,
                    download_body=True,
                    body_size=None):
        """Send request, read and return response object

        This method modifies the *headers* dictionary.
        conn must by a HTTPConnection

        When download_body is True, need to receive data before making new connection

        """
        def _debug_body(b):
            if isinstance(b, str):
                return b
            elif b is None:
                return "None"
            else:
                return 'byte_body'

        def _debug_hostname(c):
            try:
                return c.hostname
            except:
                return "None"

        log.debug('started with %r, %r, %r, %r, %r', method,
                  _debug_hostname(conn), path, headers, _debug_body(body))

        if headers is None:
            headers = CaseInsensitiveDict()

        if auth_token is None:
            headers['Authorization'] = self.auth_token
        else:
            headers['Authorization'] = auth_token

        if self.test_string:
            headers['X-Bz-Test-Mode'] = self.test_string

        try:
            if isinstance(body, io.FileIO):
                if body_size is None:
                    raise ValueError(
                        "Body size is necessary when uploading from file")
                conn.send_request(method,
                                  path,
                                  headers=headers,
                                  body=BodyFollowing(body_size))
                while True:
                    buf = body.read(BUFSIZE)
                    if not buf:
                        break
                    conn.write(buf)
            else:
                conn.send_request(method, path, headers=headers, body=body)
            resp = conn.read_response()
            if download_body or resp.status != 200:
                body = conn.readall()
            else:
                # caller need to download body itself before making new request
                body = None
        except Exception as exc:
            if is_temp_network_error(exc):
                # We probably can't use the connection anymore
                conn.disconnect()
            raise

        if resp.status == 200 or resp.status == 206:
            return resp, body

        try:
            # error code is in body
            j = json.loads(str(body, encoding='UTF-8'))
        except ValueError:
            raise HTTPError(resp.status, resp.reason, resp.headers)

        # Expired auth token
        if resp.status == 401:
            if j['code'] == 'expired_auth_token':
                log.info(
                    'BackBlaze auth token seems to have expired, requesting new one.'
                )
                self.conn_api.disconnect()
                self.conn_download.disconnect()
                # Force constructing a new connection with a new token, otherwise
                # the connection will be reestablished with the same token.
                self.conn_api = None
                self.conn_download = None
                self._login()
                raise AuthenticationExpired(j['message'])
            else:
                raise AuthorizationError(j['message'])

        # File not found
        if resp.status == 404:
            raise NoSuchObject(path)

        # Backend error
        raise B2Error(j['status'], j['code'], j['message'], headers=headers)

    def _login(self):
        """
        Login with backend and make a new connection to API and download server

        """

        id_and_key = self.account_id + ':' + self.account_key
        basic_auth_string = 'Basic ' + str(
            base64.b64encode(bytes(id_and_key, 'UTF-8')), encoding='UTF-8')

        with HTTPConnection(self.authorize_hostname,
                            443,
                            ssl_context=self.ssl_context) as conn:
            resp, body = self._do_request('GET',
                                          self.authorize_url,
                                          conn,
                                          auth_token=basic_auth_string)

            j = json.loads(str(body, encoding='UTF-8'))

            api_url = urllib.parse.urlparse(j['apiUrl'])
            download_url = urllib.parse.urlparse(j['downloadUrl'])

            self.api_host = api_url.hostname
            self.auth_token = j['authorizationToken']
            self.download_host = download_url.hostname

            self.conn_api = HTTPConnection(self.api_host,
                                           443,
                                           ssl_context=self.ssl_context)
            self.conn_download = HTTPConnection(self.download_host,
                                                443,
                                                ssl_context=self.ssl_context)

    def _connect_bucket(self, bucket_name):
        """
        Get id of bucket_name
        """

        log.debug('started with %s' % (bucket_name))

        resp, body = self._do_request(
            'GET', '/b2api/v1/b2_list_buckets?accountId=%s' % self.account_id,
            self.conn_api)
        bucket_id = None
        j = json.loads(str(body, encoding='UTF-8'))

        for b in j['buckets']:
            if b['bucketName'] == bucket_name:
                bucket_id = b['bucketId']

        if bucket_id is None:
            raise DanglingStorageURLError(bucket_name)
        self.bucket_id = bucket_id
        self.bucket_name = bucket_name

    def _add_meta_headers(self, headers, metadata):
        self._add_meta_headers_s3(headers,
                                  metadata,
                                  chunksize=self.hdr_chunksize)
        # URL encode headers
        for i in count():
            # Headers is an email.message object, so indexing it
            # would also give None instead of KeyError
            key = '%smeta-%03d' % (self.hdr_prefix, i)
            part = headers.get(key, None)
            if part is None:
                break
            headers[key] = urllib.parse.quote(part.encode('utf-8'))
            # Check we dont reach the metadata backend lmits
            if i > 10:
                raise RuntimeError("Too metadata for the backend")

    def _extractmeta(self, resp, filename):
        # URL decode headers
        for i in count():
            # Headers is an email.message object, so indexing it
            # would also give None instead of KeyError
            key = '%smeta-%03d' % (self.hdr_prefix, i)
            part = resp.headers.get(key, None)
            if part is None:
                break
            resp.headers.replace_header(key,
                                        urllib.parse.unquote_plus(str(part)))
        return self._extractmeta_s3(resp, filename)

    def _get_upload_url(self):
        """Get a single use URL to upload a file"""

        log.debug('started')

        body = bytes(json.dumps({'bucketId': self.bucket_id}),
                     encoding='UTF-8')

        resp, body = self._do_request('POST',
                                      '/b2api/v1/b2_get_upload_url',
                                      self.conn_api,
                                      body=body)

        j = json.loads(str(body, encoding='UTF-8'))
        return j['authorizationToken'], j['uploadUrl']

    def _parse_storage_url(self, storage_url):
        """Init instance variables from storage url"""

        hit = re.match(r'^b2?://([^/]+)(?:/(.*))?$', storage_url)
        if not hit:
            raise QuietError('Invalid storage URL', exitcode=2)

        bucket_name = hit.group(1)

        if not re.match('^(?!b2-)[a-z0-9A-Z\-]{6,50}$', bucket_name):
            raise QuietError('Invalid bucket name.', exitcode=2)

        prefix = hit.group(2) or ''
        #remove trailing slash if exist
        prefix = prefix[:-1] if prefix[-1] == '/' else prefix

        self.bucket_name = bucket_name
        self.prefix = prefix

    def _delete_file_id(self, filelist):

        for file in filelist:
            body = bytes(json.dumps({
                'fileName': file['fileName'],
                'fileId': file['fileId']
            }),
                         encoding='UTF-8')

            log.debug('started with /file/%s/%s/%s' %
                      (self.bucket_name, self.prefix, file['fileName']))

            try:
                self._do_request('POST',
                                 '/b2api/v1/b2_delete_file_version',
                                 self.conn_api,
                                 body=body)
            except B2Error as err:
                # Server may return file_not_present
                # just let it close connection and retry
                if err.code == 'file_not_present':
                    pass
                else:
                    raise err

    @retry
    def _list_file_version(self, key, max_filecount=1000):
        if max_filecount > 1000:
            raise ValueError('max_filecount maximum is 1000')

        request_dict = dict(bucketId=self.bucket_id,
                            maxFileCount=max_filecount)
        request_dict['startFileName'] = self.prefix + '/' + self._encode_key(
            key)

        # supposing there is less than 1000 old file version
        body = bytes(json.dumps(request_dict), encoding='UTF-8')

        resp, body = self._do_request('POST',
                                      '/b2api/v1/b2_list_file_versions',
                                      self.conn_api,
                                      body=body)

        j = json.loads(str(body, encoding='UTF-8'))
        r = []

        # We suppose there is less than 1000 file version
        for f in j['files']:
            if self._decode_key(f['fileName']) == self.prefix + '/' + key:
                r.append({'fileName': f['fileName'], 'fileId': f['fileId']})
        return r

    @retry
    def _list_file_name(self, start_filename=None, max_filecount=1000):
        if max_filecount > 1000:
            raise ValueError('max_filecount maximum is 1000')

        request_dict = {
            'bucketId': self.bucket_id,
            'maxFileCount': max_filecount
        }
        if start_filename is not None:
            request_dict['startFileName'] = start_filename
        body = bytes(json.dumps(request_dict), encoding='UTF-8')

        resp, body = self._do_request('POST',
                                      '/b2api/v1/b2_list_file_names',
                                      self.conn_api,
                                      body=body)

        j = json.loads(str(body, encoding='UTF-8'))
        filelist = [f['fileName'] for f in j['files']]

        return j['nextFileName'], filelist

    def _encode_key(self, filename):

        # URLencode filename
        filename = urllib.parse.quote(filename.encode('utf-8'), safe='/\\')

        # DIRTY HACK :
        # Backend does not support backslashes, we change them to pass test
        filename = filename.replace("\\", "__")

        return filename

    def _decode_key(self, filename):

        # DIRTY HACK :
        # Backend does not support backslashes, we change them to pass test
        filename = filename.replace("__", "\\")

        # URLencode filename
        filename = urllib.parse.unquote(filename)

        return filename

    def has_native_rename(self):
        """True if the backend has a native, atomic rename operation"""
        return False

    @copy_ancestor_docstring
    def is_temp_failure(self, exc):

        if isinstance(exc, AuthenticationExpired):
            return True

        if isinstance(exc, ssl.SSLError):
            return True

        if is_temp_network_error(exc):
            return True

        if isinstance(exc, HTTPError):
            return True

        if isinstance(exc, B2Error):
            if (exc.status == 400 or \
                exc.status == 408 or \
                (exc.status >= 500 and exc.status <= 599)):
                return True

        return False

    @retry
    @copy_ancestor_docstring
    def lookup(self, key):
        log.debug('started with %s', key)

        key = self._encode_key(key)
        headers = CaseInsensitiveDict()
        headers['Range'] = "bytes=0-1"  # Only get first byte

        resp, data = self._do_request('GET',
                                      '/file/%s/%s/%s' %
                                      (self.bucket_name, self.prefix, key),
                                      self.conn_download,
                                      headers=headers)

        meta = self._extractmeta(resp, key)
        return meta

    @retry
    def get_size(self, key):
        """Return size of object stored under *key*"""
        log.debug('started with %s', key)

        key = self._encode_key(key)
        key_with_prefix = "%s/%s" % (self.prefix, key)
        request_dict = {
            'bucketId': self.bucket_id,
            'startFileName': key_with_prefix,
            'maxFileCount': 1
        }

        body = bytes(json.dumps(request_dict), encoding='UTF-8')
        resp, body = self._do_request('POST',
                                      '/b2api/v1/b2_list_file_names',
                                      self.conn_api,
                                      body=body)

        j = json.loads(str(body, encoding='UTF-8'))
        if j['files'][0]['fileName'] == key_with_prefix:
            return j['files'][0]['contentLength']

        raise NoSuchObject(key_with_prefix)

    @retry
    @copy_ancestor_docstring
    def open_read(self, key):
        log.debug('started with %s', key)

        key = self._encode_key(key)

        resp, data = self._do_request('GET',
                                      '/file/%s/%s/%s' %
                                      (self.bucket_name, self.prefix, key),
                                      self.conn_download,
                                      download_body=False)

        meta = self._extractmeta(resp, key)

        return ObjectR(key, self.conn_download, resp, metadata=meta)

    @copy_ancestor_docstring
    def open_write(self, key, metadata=None, is_compressed=False):
        log.debug('started with %s', key)

        key = self._encode_key(key)

        return ObjectW(key, self, metadata)

    @retry
    @copy_ancestor_docstring
    def clear(self):
        log.debug('started')
        for file in self.list():
            self.delete(file, force=True)

    @retry
    @copy_ancestor_docstring
    def delete(self, key, force=False):
        log.debug('started with %s', key)

        todel_id = self._list_file_version(key)

        if not todel_id and not force:
            raise NoSuchObject(key)

        self._delete_file_id(todel_id)

    @copy_ancestor_docstring
    def list(self, prefix=''):
        log.debug('started with %s', prefix)

        prefix = self._encode_key(prefix)

        next_filename = self.prefix + '/' + prefix
        keys_remaining = True

        while keys_remaining and next_filename is not None:
            next_filename, filelist = self._list_file_name(next_filename)

            while filelist:
                file = filelist.pop(0)

                if file.startswith(self.prefix + '/' + prefix):
                    # remove prefix before return
                    r = file[len(self.prefix + '/'):]
                    yield self._decode_key(r)
                else:
                    keys_remaining = False
                    break

    @prepend_ancestor_docstring
    def copy(self, src, dest, metadata=None):
        """No atomic copy operation on backend. Hope this does not append to often"""
        log.debug('started with %s, %s', src, dest)

        data, src_meta = self.fetch(src)

        # Delete dest file if already exist
        self.delete(dest, force=True)

        if metadata is None:
            dst_meta = src_meta
        else:
            dst_meta = metadata

        self.store(dest, data, dst_meta)

    @copy_ancestor_docstring
    def update_meta(self, key, metadata):
        log.debug('started with %s', key)
        self.copy(key, key, metadata)

    def close(self):
        log.debug('started')
        self.conn_api.disconnect()
        self.conn_download.disconnect()