def test_copy(self): """ L{Headers.copy} creates a new independent copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(u'test\u00E1', [u'foo\u2603']) i = h.copy() # The copy contains the same value as the original self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603']) self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83']) # Add a header to the original h.addRawHeader(u'test\u00E1', u'bar') # Verify that the copy has not changed self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603']) self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83']) # Add a header to the copy i.addRawHeader(u'test\u00E1', b'baz') # Verify that the orignal does not have it self.assertEqual( h.getRawHeaders(u'test\u00E1'), [u'foo\u2603', u'bar']) self.assertEqual( h.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83', b'bar'])
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath): """ Issue a new request, given the endpoint and the path sent as part of the request. """ # Create minimal headers, if necessary: if headers is None: headers = Headers() if not headers.hasHeader('host'): headers = headers.copy( ) # not supported in twisted <= 11.1, and it doesn't affects us headers.addRawHeader( 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, parsedURI.port)) d = self._pool.getConnection(key, endpoint) def cbConnected(proto): return proto.request( Request(method, requestPath, headers, bodyProducer, persistent=self._pool.persistent)) d.addCallback(cbConnected) return d
def test_copy(self): """ L{Headers.copy} creates a new independent copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders("test\u00E1", ["foo\u2603"]) i = h.copy() # The copy contains the same value as the original self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"]) self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"]) # Add a header to the original h.addRawHeader("test\u00E1", "bar") # Verify that the copy has not changed self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"]) self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"]) # Add a header to the copy i.addRawHeader("test\u00E1", b"baz") # Verify that the orignal does not have it self.assertEqual(h.getRawHeaders("test\u00E1"), ["foo\u2603", "bar"]) self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83", b"bar"])
def request(self, method, uri, headers=None, bodyProducer=None): parsedURI = client._parse(uri) host_addr = address.IPv4Address('TCP', parsedURI.host, parsedURI.port) # ripped from _AgentBase._requestWithEndpoint if headers is None: headers = Headers() if not headers.hasHeader('host'): headers = headers.copy() headers.addRawHeader( 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, parsedURI.port)) request = client.Request(method, parsedURI.path, headers, bodyProducer, persistent=False) c = ClientProtocol(request) # ouch self.root.putChild('', self.root) server = Site(self.root).buildProtocol(self.addr) loopbackAsync(server, c, host_addr, self.addr) return c.response.addBoth(self._done, c)
def request(self, method, uri, headers=None, bodyProducer=None): """ Issue a new request to the wrapped L{Agent}. Send a I{Cookie} header if a cookie for C{uri} is stored in L{CookieAgent.cookieJar}. Cookies are automatically extracted and stored from requests. If a C{'cookie'} header appears in C{headers} it will override the automatic cookie header obtained from the cookie jar. @see: L{Agent.request} """ if headers is None: headers = Headers() lastRequest = _FakeUrllib2Request(uri) # Setting a cookie header explicitly will disable automatic request # cookies. if not headers.hasHeader('cookie'): self.cookieJar.add_cookie_header(lastRequest) cookieHeader = lastRequest.get_header('Cookie', None) if cookieHeader is not None: headers = headers.copy() headers.addRawHeader('cookie', cookieHeader) d = self._agent.request(method, uri, headers, bodyProducer) d.addCallback(self._extractCookies, lastRequest) return d
def request(self, method, uri, headers=None, bodyProducer=None): """ Issue a new request to the wrapped L{Agent}. Send a I{Cookie} header if a cookie for C{uri} is stored in L{CookieAgent.cookieJar}. Cookies are automatically extracted and stored from requests. If a C{'cookie'} header appears in C{headers} it will override the automatic cookie header obtained from the cookie jar. @see: L{Agent.request} """ if headers is None: headers = Headers() lastRequest = _FakeUrllib2Request(uri) # Setting a cookie header explicitly will disable automatic request # cookies. if not headers.hasHeader('cookie'): self.cookieJar.add_cookie_header(lastRequest) cookieHeader = lastRequest.get_header('Cookie', None) if cookieHeader is not None: headers = headers.copy() headers.addRawHeader('cookie', cookieHeader) d = self._agent.request(method, uri, headers, bodyProducer) d.addCallback(self._extractCookies, lastRequest) return d
def _connectAndRequest(self, method, uri, headers, bodyProducer, requestPath=None): """ Internal helper to make the request. @param requestPath: If specified, the path to use for the request instead of the path extracted from C{uri}. """ scheme, host, port, path = _parse(uri) if requestPath is None: requestPath = path d = self._connect(scheme, host, port) if headers is None: headers = Headers() if not headers.hasHeader('host'): headers = headers.copy() headers.addRawHeader('host', self._computeHostValue(scheme, host, port)) def cbConnected(proto): return proto.request( Request(method, requestPath, headers, bodyProducer)) d.addCallback(cbConnected) return d
def fetch(self, uri, headers=None): headers = Headers() if headers is None else headers.copy() headers.setRawHeaders(b'referer', ['https://www.pixiv.net/']) headers.setRawHeaders(b'host', [self.host]) if (self.port == 80 and self.scheme == 'http://') \ or (self.port == 443 and self.scheme == 'https://') \ or self.port is None: url = self.scheme + self.host else: url = self.scheme + self.host + ':' + str(self.port) url = url + uri dfd = self.agent.request('GET', url, headers) dfd.addErrback(self.on_failure) def _wait_body(response): body_dfd = readBody(response) body_dfd.addCallback(lambda b: setattr(response, 'body', b)) body_dfd.addCallback(lambda _: response) return body_dfd dfd.addCallback(_wait_body) return dfd
def request(self, method, uri, headers=None, bodyProducer=None): if headers is None: headers = Headers() else: headers = headers.copy() if not headers.hasHeader('user-agent'): headers.addRawHeader('user-agent', self.user_agent) return self.agent.request(method, uri, headers, bodyProducer)
def request(self, method, uri, headers=None, bodyProducer=None): if headers is None: headers = Headers() else: headers = headers.copy() for name, value in self._header_list: headers.addRawHeader(name, value) return self._agent.request(method, uri, headers, bodyProducer)
def request(self, method, url, headers=None, bodyProducer=None): """ Issue a request with some extra headers. :see: ``twisted.web.iweb.IAgent.request`` """ if headers is None: headers = Headers() else: headers = headers.copy() for k, vs in self._to_inject.getAllRawHeaders(): headers.setRawHeaders(k, vs) return self._agent.request(method, url, headers, bodyProducer)
def request(self, method, uri, headers=None, bodyProducer=None): """ Send a client request which declares supporting compressed content. @see: L{Agent.request}. """ if headers is None: headers = Headers() else: headers = headers.copy() headers.addRawHeader('accept-encoding', self._supported) deferred = self._agent.request(method, uri, headers, bodyProducer) return deferred.addCallback(self._handleResponse)
def request(self, method, uri, headers=None, bodyProducer=None): """ Send a client request which declares supporting compressed content. @see: L{Agent.request}. """ if headers is None: headers = Headers() else: headers = headers.copy() headers.addRawHeader('accept-encoding', self._supported) deferred = self._agent.request(method, uri, headers, bodyProducer) return deferred.addCallback(self._handleResponse)
def test_copy(self): """ L{Headers.copy} creates a new independant copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(b"test", [b"foo"]) i = h.copy() self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) h.addRawHeader(b"test", b"bar") self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) i.addRawHeader(b"test", b"baz") self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def test_copy(self): """ L{Headers.copy} creates a new independant copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(b'test', [b'foo']) i = h.copy() self.assertEqual(i.getRawHeaders(b'test'), [b'foo']) h.addRawHeader(b'test', b'bar') self.assertEqual(i.getRawHeaders(b'test'), [b'foo']) i.addRawHeader(b'test', b'baz') self.assertEqual(h.getRawHeaders(b'test'), [b'foo', b'bar'])
def test_copy(self): """ L{Headers.copy} creates a new independent copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(b"test", [b"foo"]) i = h.copy() self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) h.addRawHeader(b"test", b"bar") self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) i.addRawHeader(b"test", b"baz") self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def test_copy(self): """ L{Headers.copy} creates a new independant copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders('test', ['foo']) i = h.copy() self.assertEquals(i.getRawHeaders('test'), ['foo']) h.addRawHeader('test', 'bar') self.assertEquals(i.getRawHeaders('test'), ['foo']) i.addRawHeader('test', 'baz') self.assertEquals(h.getRawHeaders('test'), ['foo', 'bar'])
def CheckupURL(): HTTPHeader0 = Headers({'User-Agent': ['PSO2Proxy']}) #We need to send a User-Agent if eqalert_config.key_exists('api'): eq_URL = eqalert_config.get_key('api') else: eq_URL = None if eq_URL: HTTPHeaderX = HTTPHeader0.copy() if ETag_Header: HTTPHeaderX.addRawHeader('If-None-Match', ETag_Header) if Modified_Header: HTTPHeaderX.addRawHeader('If-Modified-Since', Modified_Header) EQ0 = agent.request('GET', eq_URL, HTTPHeaderX) EQ0.addCallback(EQResponse) EQ0.addErrback(log.err)
def CheckupURL(): HTTPHeader0 = Headers({'User-Agent': ['PSO2Proxy']}) load_eqJP_names() # Reload file for shipNum in config.globalConfig.get_key('enabledShips'): if eqnotice_config.key_exists(str(shipNum)): eq_URL = eqnotice_config.get_key(str(shipNum)) else: eq_URL = None if eq_URL: HTTPHeaderX = HTTPHeader0.copy() if ETag_Headers[shipNum]: HTTPHeaderX.addRawHeader('If-None-Match', ETag_Headers[shipNum]) if Modified_Headers[shipNum]: HTTPHeaderX.addRawHeader('If-Modified-Since', Modified_Headers[shipNum]) EQ0 = agent.request('GET', eq_URL, HTTPHeaderX) EQ0.addCallback(EQResponse, shipNum) EQ0.addErrback(log.err)
def CheckupURL(): if eqalert_config.key_exists('User-Agent'): # We need to send a User-Agent HTTPHeader0 = Headers({'User-Agent': [eqalert_config['User-Agent']]}) else: HTTPHeader0 = Headers({'User-Agent': ['PSO2Proxy']}) if eqalert_config.key_exists('api'): eq_URL = eqalert_config['api'] else: eq_URL = None if eq_URL: HTTPHeaderX = HTTPHeader0.copy() if ETag_Header: HTTPHeaderX.addRawHeader('If-None-Match', ETag_Header) if Modified_Header: HTTPHeaderX.addRawHeader('If-Modified-Since', Modified_Header) EQ0 = agent.request('GET', eq_URL, HTTPHeaderX) EQ0.addCallback(EQResponse) EQ0.addErrback(log.err)
def CheckupURL(): try: HTTPHeader0 = Headers({'User-Agent': ['PSO2Proxy']}) #We need to send a User-Agent if eqalert_config.key_exists('api'): eq_URL = eqalert_config.get_key('api') else: eq_URL = None if eq_URL: HTTPHeaderX = HTTPHeader0.copy() if ETag_Header: HTTPHeaderX.addRawHeader('If-None-Match', ETag_Header) if Modified_Header: HTTPHeaderX.addRawHeader('If-Modified-Since', Modified_Header) #EQ0 = agent.request('GET', eq_URL, HTTPHeaderX) EQ0 = agent.request('GET', eq_URL) EQ0.addCallback(EQResponse) EQ0.addErrback(log.err) except Exception as ex1: print("[EQ Alert] Opps Sorry, %s" % ex1)
def request(self, method, uri, headers, bodyProducer): if headers is None: headers = Headers() else: headers = headers.copy() contentType = headers.getRawHeaders('content-type', [""])[0] date = headers.getRawHeaders('date', [""])[0] or self._generateRequestDate(uri) headers.setRawHeaders('date', [date]) uri_origin_form = URI.fromBytes(uri).originForm contentMD5 = headers.getRawHeaders('content-md5', [""])[0] if not contentMD5 and bodyProducer is not None: r = getattr(self.agent, '_reactor') or reactor bodyConsumer = StringConsumer(callLater=r.callLater) yield bodyProducer.startProducing(bodyConsumer) body = bodyConsumer.value() bodyProducer = StringBodyProducer(body) if body: contentMD5 = binascii.b2a_base64( hashlib.md5(body).digest()).strip() headers.addRawHeader('content-md5', contentMD5) sts = "\n".join([ method, contentType or "", contentMD5, date or "", uri_origin_form ]) mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest() encodedMAC = binascii.b2a_base64(mac).strip() auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC) headers.addRawHeader('authorization', auth_header) d = yield self.agent.request(method, uri, headers, bodyProducer) self._handleResponseDate(uri, d) defer.returnValue(d)
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath): if headers is None: headers = Headers() if not headers.hasHeader('host'): headers = headers.copy() headers.addRawHeader( 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, parsedURI.port)) d = self._pool.getConnection(key, endpoint) def cbConnected(proto): return proto.request( SigningRequest._construct(self._signData.makeSigner(), method, requestPath, headers, bodyProducer, persistent=self._pool.persistent, parsedURI=parsedURI)) d.addCallback(cbConnected) return d
def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath): """ Issue a new request, given the endpoint and the path sent as part of the request. """ # Create minimal headers, if necessary: if headers is None: headers = Headers() if not headers.hasHeader('host'): headers = headers.copy() # not supported in twisted <= 11.1, and it doesn't affects us headers.addRawHeader( 'host', self._computeHostValue(parsedURI.scheme, parsedURI.host, parsedURI.port)) d = self._pool.getConnection(key, endpoint) def cbConnected(proto): return proto.request( Request(method, requestPath, headers, bodyProducer, persistent=self._pool.persistent)) d.addCallback(cbConnected) return d
def _connectAndRequest(self, method, uri, headers, bodyProducer, requestPath=None): """ Internal helper to make the request. @param requestPath: If specified, the path to use for the request instead of the path extracted from C{uri}. """ scheme, host, port, path = _parse(uri) if requestPath is None: requestPath = path d = self._connect(scheme, host, port) if headers is None: headers = Headers() if not headers.hasHeader('host'): headers = headers.copy() headers.addRawHeader( 'host', self._computeHostValue(scheme, host, port)) def cbConnected(proto): return proto.request( Request(method, requestPath, headers, bodyProducer)) d.addCallback(cbConnected) return d
def request(self, method, uri, headers, bodyProducer): if headers is None: headers = Headers() else: headers = headers.copy() contentType = headers.getRawHeaders('content-type', [""])[0] date = headers.getRawHeaders('date', [""])[0] or self._generateRequestDate(uri) headers.setRawHeaders('date', [date]) uri_origin_form = URI.fromBytes(uri).originForm contentMD5 = headers.getRawHeaders('content-md5', [""])[0] if not contentMD5 and bodyProducer is not None: r = getattr(self.agent, '_reactor') or reactor bodyConsumer = StringConsumer(callLater=r.callLater) yield bodyProducer.startProducing(bodyConsumer) body = bodyConsumer.value() bodyProducer = StringBodyProducer(body) if body: contentMD5 = binascii.b2a_base64(hashlib.md5(body).digest()).strip() headers.addRawHeader('content-md5', contentMD5) sts = "\n".join([method, contentType or "", contentMD5, date or "", uri_origin_form]) mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest() encodedMAC = binascii.b2a_base64(mac).strip() auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC) headers.addRawHeader('authorization', auth_header) d = yield self.agent.request(method, uri, headers, bodyProducer) self._handleResponseDate(uri, d) defer.returnValue(d)
def request(self, method, uri, headers=None, bodyProducer=None): """ Args: method (bytes): HTTP method: GET/POST/etc uri (bytes): Absolute URI to be retrieved headers (twisted.web.http_headers.Headers|None): HTTP headers to send with the request, or None to send no extra headers. bodyProducer (twisted.web.iweb.IBodyProducer|None): An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or None if the request is to have no body. Returns: Deferred[twisted.web.iweb.IResponse]: fires when the header of the response has been received (regardless of the response status code). Fails if there is any problem which prevents that response from being received (including problems that prevent the request from being sent). """ parsed_uri = URI.fromBytes(uri, defaultPort=-1) res = yield self._route_matrix_uri(parsed_uri) # set up the TLS connection params # # XXX disabling TLS is really only supported here for the benefit of the # unit tests. We should make the UTs cope with TLS rather than having to make # the code support the unit tests. if self._tls_client_options_factory is None: tls_options = None else: tls_options = self._tls_client_options_factory.get_options( res.tls_server_name.decode("ascii")) # make sure that the Host header is set correctly if headers is None: headers = Headers() else: headers = headers.copy() if not headers.hasHeader(b"host"): headers.addRawHeader(b"host", res.host_header) class EndpointFactory(object): @staticmethod def endpointForURI(_uri): ep = LoggingHostnameEndpoint(self._reactor, res.target_host, res.target_port) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool) res = yield make_deferred_yieldable( agent.request(method, uri, headers, bodyProducer)) return res
def request(self, method, uri, headers=None, bodyProducer=None): """ Args: method (bytes): HTTP method: GET/POST/etc uri (bytes): Absolute URI to be retrieved headers (twisted.web.http_headers.Headers|None): HTTP headers to send with the request, or None to send no extra headers. bodyProducer (twisted.web.iweb.IBodyProducer|None): An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or None if the request is to have no body. Returns: Deferred[twisted.web.iweb.IResponse]: fires when the header of the response has been received (regardless of the response status code). Fails if there is any problem which prevents that response from being received (including problems that prevent the request from being sent). """ # We use urlparse as that will set `port` to None if there is no # explicit port. parsed_uri = urllib.parse.urlparse(uri) # If this is a matrix:// URI check if the server has delegated matrix # traffic using well-known delegation. # # We have to do this here and not in the endpoint as we need to rewrite # the host header with the delegated server name. delegated_server = None if (parsed_uri.scheme == b"matrix" and not _is_ip_literal(parsed_uri.hostname) and not parsed_uri.port): well_known_result = yield self._well_known_resolver.get_well_known( parsed_uri.hostname) delegated_server = well_known_result.delegated_server if delegated_server: # Ok, the server has delegated matrix traffic to somewhere else, so # lets rewrite the URL to replace the server with the delegated # server name. uri = urllib.parse.urlunparse(( parsed_uri.scheme, delegated_server, parsed_uri.path, parsed_uri.params, parsed_uri.query, parsed_uri.fragment, )) parsed_uri = urllib.parse.urlparse(uri) # We need to make sure the host header is set to the netloc of the # server and that a user-agent is provided. if headers is None: headers = Headers() else: headers = headers.copy() if not headers.hasHeader(b"host"): headers.addRawHeader(b"host", parsed_uri.netloc) if not headers.hasHeader(b"user-agent"): headers.addRawHeader(b"user-agent", self.user_agent) res = yield make_deferred_yieldable( self._agent.request(method, uri, headers, bodyProducer)) return res
def request(self, method, uri, headers=None, bodyProducer=None): headers = Headers() if headers is None else headers.copy() headers.addRawHeader(b'authorization', self._auth_header) return self._agent.request(method, uri, headers, bodyProducer)
def request(self, method, uri, headers=None, bodyProducer=None): """ :param method: HTTP method (GET/POST/etc). :type method: bytes :param uri: Absolute URI to be retrieved. :type uri: bytes :param headers: HTTP headers to send with the request, or None to send no extra headers. :type headers: twisted.web.http_headers.Headers, None :param bodyProducer: An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or None if the request is to have no body. :type bodyProducer: twisted.web.iweb.IBodyProducer, None :returns a deferred that fires when the header of the response has been received (regardless of the response status code). Fails if there is any problem which prevents that response from being received (including problems that prevent the request from being sent). :rtype: Deferred[twisted.web.iweb.IResponse] """ parsed_uri = URI.fromBytes(uri, defaultPort=-1) res = yield self._route_matrix_uri(parsed_uri) # set up the TLS connection params # # XXX disabling TLS is really only supported here for the benefit of the # unit tests. We should make the UTs cope with TLS rather than having to make # the code support the unit tests. if self._tls_client_options_factory is None: tls_options = None else: tls_options = self._tls_client_options_factory.get_options( res.tls_server_name.decode("ascii") ) # make sure that the Host header is set correctly if headers is None: headers = Headers() else: headers = headers.copy() if not headers.hasHeader(b'host'): headers.addRawHeader(b'host', res.host_header) class EndpointFactory(object): @staticmethod def endpointForURI(_uri): ep = LoggingHostnameEndpoint( self._reactor, res.target_host, res.target_port, ) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool) res = yield agent.request(method, uri, headers, bodyProducer) defer.returnValue(res)
def request(self, method, uri, headers=None, bodyProducer=None): headers = Headers() if headers is None else headers.copy() headers.addRawHeader(b'authorization', self._auth_header) return self._agent.request(method, uri, headers, bodyProducer)