def process(self):
host = self.getHeader('Host')
if not host:
log.err("No host header given")
self.setResponseCode(400)
self.finish()
return
port = 80
if self.isSecure() == True:
port = 443
if ':' in host:
host, port = host.split(':')
port = int(port)
log.msg("self: %s" % (self))
log.msg("host:port: %s:%s" % (host, port))
self.setHost(host, port)
self.content.seek(0, 0)
postData = self.content.read()
endpoint = TCP4ClientEndpoint(self.reactor, os.environ.get("PROXY_HOST"), int(os.environ.get("PROXY_PORT")))
agent = ProxyAgent(endpoint)
scheme = b"http"
if self.isSecure() == True:
scheme = b"https"
userpw = None
url = scheme + b"://" + str.encode(host) + b":" + str.encode(str(port)) + self.uri
if self.getUser() != None and self.getPassword() != None:
userpw = self.getUser() + b":" + self.getPassword()
url = scheme + b"://" + userpw + b"@" + str.encode(host) + b":" + str.encode(str(port)) + self.uri
log.msg("URL: %s" % (url))
d = Deferred()
log.msg("Method: %s" % (self.method))
headers = {};
for key, values in self.requestHeaders.getAllRawHeaders():
if key.lower() == "connection":
headers[key] = ['close']
elif key.lower() == 'keep-alive':
next
else:
headers[key] = values
if self.method == b"POST" or self.method == b"PUT":
log.msg("POST Data: %s" % (postData))
body = BytesProducer(postData)
d = agent.request(self.method, url, Headers(headers), body)
else:
d = agent.request(self.method, url, Headers(headers))
d.addCallback(self.forwardToClient)
def _request(self, request, method):
"""
Helper method that sends the given HTTP request.
"""
# Copy the headers from the request.
headers = Headers()
for (key, value) in request.headers.iteritems():
headers.addRawHeader(key, value)
# If a username and password are given, then add basic authentication.
if (self.options.username is not None and
self.options.password is not None):
auth = "%s:%s" % (self.options.username, self.options.password)
auth = auth.encode("base64").strip()
headers.addRawHeader('Authorization', 'Basic ' + auth)
# Determine if the user has configured a proxy server.
url_parts = urlparse.urlparse(request.url)
proxy = self.options.proxy.get(url_parts.scheme, None)
# Construct an agent to send the request.
if proxy is not None:
(hostname, port) = proxy.split(":")
endpoint = TCP4ClientEndpoint(reactor, hostname, int(port),
timeout = self.options.timeout)
agent = ProxyAgent(endpoint)
else:
agent = Agent(reactor, self.httpsPolicy,
connectTimeout = self.options.timeout)
url = request.url.encode("utf-8")
producer = StringProducer(request.message or "")
response = yield agent.request(method, url, headers, producer)
# If the initial request returned a redirection response, attempt to follow it.
http_redirect_codes = [http.MOVED_PERMANENTLY, # 301
http.FOUND, # 302
http.SEE_OTHER, # 303
http.TEMPORARY_REDIRECT] # 307
if response.code in http_redirect_codes and response.headers.hasHeader("Location"):
new_url = response.headers.getRawHeaders("Location")[0]
producer = StringProducer(request.message or "")
response = yield agent.request(method, new_url, headers, producer)
# Construct a simple response consumer and give it the response body.
consumer = StringResponseConsumer()
response.deliverBody(consumer)
yield consumer.getDeferred()
consumer.response = response
defer.returnValue(consumer)
def main():
endpoint = lib.twisted.TCP4ClientEndpoint()
agent = ProxyAgent(endpoint)
d = agent.request("GET", "http://echo.ioloop.io/a/b?format=json")
d.addCallbacks(display, err)
d.addCallback(lambda ignored: reactor.stop())
reactor.run()
def get_device_from_smcloud(self, user_agent):
"""
Queries ScientiaMobile's API and returns a dictionary of the device.
"""
# Create basic auth string
b64 = base64.encodestring(self.smcloud_api_key).strip()
if self.http_proxy_host:
headers = {
"X-Cloud-Client": [self.SMCLOUD_CONFIG["client_version"]],
"Authorization": ["Basic %s" % b64],
"User-Agent": [str(user_agent)],
}
if self.http_proxy_username and self.http_proxy_password:
auth = base64.encodestring("%s:%s" % (self.http_proxy_username, self.http_proxy_password)).strip()
# Cater for many proxy servers
headers["Proxy-Authorization"] = ["Basic %s" % auth]
headers["Proxy-Authenticate"] = ["Basic %s" % auth]
headers["Proxy-Authentication"] = ["Basic %s" % auth]
endpoint = TCP4ClientEndpoint(reactor, self.http_proxy_host, self.http_proxy_port or 80, timeout=5)
agent = ProxyAgent(endpoint)
response = yield agent.request("GET", self.SMCLOUD_CONFIG["url"], headers=Headers(headers))
if response.code != 200:
raise ProxyConnectError()
d = Deferred()
response.deliverBody(SimpleReceiver(d))
body = yield d
else:
headers = {"X-Cloud-Client": self.SMCLOUD_CONFIG["client_version"], "Authorization": "Basic %s" % b64}
try:
body = yield getPage(self.SMCLOUD_CONFIG["url"], headers=headers, agent=user_agent, timeout=5)
except ConnectError, exc:
raise ScientiaMobileCloudHandlerConnectError(exc)
def _asynchronous_check(self, item):
logger.debug('checking...')
from twisted.internet import reactor # must here import
endpoint = HostnameEndpoint(reactor, item[0], int(item[1]))
agent = ProxyAgent(endpoint) #
headers = {'User-Agent': ['Mozilla/5.0']}
headers = Headers(headers) # headers wrapper
cd = agent.request(b'GET', b'https://www.baidu.com/',
headers=headers) # ?
cd._connectTimeout = 3
def check_code(response, **kwargs):
if response.code < 300:
logger.info('valid ip!')
return kwargs.pop('item', None)
else:
raise Exception('invalid')
def err(f):
logger.debug(f)
return f
cd.addCallbacks(check_code, err, callbackKeywords={'item': item})
# cd.addErrback(err)
return cd
def proxy_crawl(self, url, ip, reactor):
endpoint = TCP4ClientEndpoint(reactor, ip.split(':')[0], int(ip.split(':')[1]),
timeout=self.settings.get('DOWNLOAD_TIMEOUT', 10))
agent = ProxyAgent(endpoint)
response = agent.request(b"GET", url.encode('ascii'),
Headers(self.headers))
return response
def doRequest(noreason):
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 8787)
agent = ProxyAgent(proxyEndpoint, reactor)
log.msg("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" % self.url)
request = agent.request("GET", self.url)
request.addCallback(readBody)
request.addCallback(addResultToReport)
request.addCallback(self.processDirector.close)
return request
def request_echo(self, content):
"""
Send a get request to request the test server to echo the content
"""
ouinet_client_endpoint = TCP4ClientEndpoint(
reactor, "127.0.0.1", TestFixtures.FIRST_CLIENT["port"])
agent = ProxyAgent(ouinet_client_endpoint)
return agent.request(
"GET", "http://127.0.0.1:" +
str(TestFixtures.TEST_HTTP_SERVER_PORT) + "/?content=" + content)
def doRequest(noreason):
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 8787)
agent = ProxyAgent(proxyEndpoint, reactor)
log.msg("Doing HTTP request via Lantern (127.0.0.1:8787) for %s" %
self.url)
request = agent.request("GET", self.url)
request.addCallback(readBody)
request.addCallback(addResultToReport)
request.addCallback(self.processDirector.close)
return request
def get_page(self, url, *args, **kwds):
"""
Define our own get_page method so that we can easily override the
factory when we need to. This was copied from the following:
* twisted.web.client.getPage
* twisted.web.client._makeGetterFactory
"""
contextFactory = None
scheme, host, port, path = parse(url)
data = kwds.get('postdata', None)
self._method = method = kwds.get('method', 'GET')
self.request_headers = self._headers(kwds.get('headers', {}))
if (self.body_producer is None) and (data is not None):
self.body_producer = FileBodyProducer(StringIO(data))
if scheme == "https":
proxy_endpoint = os.environ.get("https_proxy")
if proxy_endpoint:
proxy_url = urlparse.urlparse(proxy_endpoint)
endpoint = TCP4ClientEndpoint(self.reactor, proxy_url.hostname, proxy_url.port)
agent = ProxyAgent(endpoint)
else:
if self.endpoint.ssl_hostname_verification:
contextFactory = WebVerifyingContextFactory(host)
else:
contextFactory = WebClientContextFactory()
agent = Agent(self.reactor, contextFactory)
self.client.url = url
d = agent.request(method, url, self.request_headers,
self.body_producer)
else:
proxy_endpoint = os.environ.get("http_proxy")
if proxy_endpoint:
proxy_url = urlparse.urlparse(proxy_endpoint)
endpoint = TCP4ClientEndpoint(self.reactor, proxy_url.hostname, proxy_url.port)
agent = ProxyAgent(endpoint)
else:
agent = Agent(self.reactor)
d = agent.request(method, url, self.request_headers,
self.body_producer)
d.addCallback(self._handle_response)
return d
def request(method, url, headers, producer=None, use_tor=False):
"""Make a HTTP request and returns a deferred
"""
if use_tor:
host = urlparse(url).netloc
port = config.Application().tor_socks_port
proxy = TCP4ClientEndpoint(reactor, "localhost", int(port))
agent = ProxyAgent(SOCKS5ClientEndpoint(host, 80, proxy))
else:
agent = Agent(reactor)
if headers is None:
headers = Headers({'User-Agent': ['Twisted Web Client']})
log.msg('Using TOR network' if use_tor else 'Using standard network')
log.msg('Request URL: {}'.format(url))
return agent.request(method, url, headers, producer)
def main():
"""Main loop
"""
reactor.callLater(0, consumeEvents)
logging.basicConfig(level=logging.DEBUG)
log.debug('Starting...')
o = urlparse.urlparse(DOCKER_HOST)
if o.scheme == 'unix':
endpoint = UNIXClientEndpoint(reactor, o.path)
elif o.scheme in ('tcp', 'http'):
port = o.port or 80
endpoint = TCP4ClientEndpoint(reactor, o.hostname, port)
else:
assert 0
agent = ProxyAgent(endpoint)
d = agent.request('GET', '/events')
d.addCallback(callback)
d.addErrback(errback)
return d
def _request(self, request, method):
"""
Helper method that sends the given HTTP request.
"""
# Copy the headers from the request.
headers = Headers()
for (key, value) in request.headers.iteritems():
headers.addRawHeader(key, value)
# If a username and password are given, then add basic authentication.
if (self.options.username is not None and
self.options.password is not None):
auth = "%s:%s" % (self.options.username, self.options.password)
auth = auth.encode("base64").strip()
headers.addRawHeader('Authorization', 'Basic ' + auth)
# Determine if the user has configured a proxy server.
url_parts = urlparse.urlparse(request.url)
proxy = self.options.proxy.get(url_parts.scheme, None)
# Construct an agent to send the request.
if proxy is not None:
(hostname, port) = proxy.split(":")
endpoint = TCP4ClientEndpoint(reactor, hostname, int(port),
timeout = self.options.timeout)
agent = ProxyAgent(endpoint)
else:
agent = Agent(reactor, self.contextFactory,
connectTimeout = self.options.timeout)
url = request.url.encode("utf-8")
producer = StringProducer(request.message or "")
response = yield agent.request(method, url, headers, producer)
# Construct a simple response consumer and give it the response body.
consumer = StringResponseConsumer()
response.deliverBody(consumer)
yield consumer.getDeferred()
consumer.response = response
defer.returnValue(consumer)
def request_url(self, port, url):
ouinet_client_endpoint = TCP4ClientEndpoint(reactor, "127.0.0.1", port)
agent = ProxyAgent(ouinet_client_endpoint)
return agent.request("GET", url)
def json_call(self, method, **kwargs):
https = kwargs.pop('https', False)
blowfish = kwargs.pop('blowfish', True)
url_args = {'method': method}
if self.partner_id:
url_args['partner_id'] = self.partner_id
if self.user_id:
url_args['user_id'] = self.user_id
if self.user_auth_token or self.partner_auth_token:
url_args['auth_token'] = (self.user_auth_token or
self.partner_auth_token)
protocol = 'https' if https else 'http'
url = protocol + self.rpc_url + urllib.urlencode(url_args)
if self.time_offset:
kwargs['syncTime'] = int(time.time() + self.time_offset)
if self.user_auth_token:
kwargs['userAuthToken'] = self.user_auth_token
elif self.partner_auth_token:
kwargs['partnerAuthToken'] = self.partner_auth_token
data = json.dumps(kwargs)
if blowfish:
data = self.pandora_encrypt(data)
if self.proxy_host:
endpoint = TCP4ClientEndpoint(reactor, self.proxy_host,
self.proxy_port)
agent = ProxyAgent(endpoint, WebClientContextFactory())
else:
agent = Agent(reactor, WebClientContextFactory())
headers = Headers({'User-Agent': ['pithos'],
'Content-type': ['text/plain']})
body = FileBodyProducer(StringIO(data))
response = yield agent.request('POST', url, headers, body)
body = yield readBody(response)
tree = json.loads(body)
if tree['stat'] == 'fail':
code = tree['code']
msg = tree['message']
log.msg('fault code: {} message: {}'.format(code, msg))
if code == API_ERROR.INVALID_AUTH_TOKEN:
raise PandoraAuthTokenInvalid(msg)
elif code == API_ERROR.COUNTRY_NOT_SUPPORTED:
raise PandoraError(
"Pandora not available", code,
"Pandora is not available outside the US.")
elif code == API_ERROR.API_VERSION_NOT_SUPPORTED:
raise PandoraAPIVersionError(msg)
elif code == API_ERROR.INSUFFICIENT_CONNECTIVITY:
raise PandoraError(
"Out of sync", code, "Correct your system's clock.")
elif code == API_ERROR.READ_ONLY_MODE:
raise PandoraError(
"Pandora maintenance", code,
"Pandora is in read-only mode as it is performing "
"maintenance. Try again later.")
elif code == API_ERROR.INVALID_LOGIN:
raise PandoraError(
"Login Error", code, "Invalid username or password.")
elif code == API_ERROR.LISTENER_NOT_AUTHORIZED:
raise PandoraError(
"Pandora One Error", code,
"A Pandora One account is required to access this "
"feature.")
elif code == API_ERROR.PARTNER_NOT_AUTHORIZED:
raise PandoraError(
"Login Error", code, "Invalid Pandora partner keys.")
else:
raise PandoraError(msg, code)
if 'result' in tree:
returnValue(tree['result'])
def _test_proxy_alive(self,
host,
port,
protocol,
proxy_type,
url=b'http://www.baidu.com',
timeout=10):
endpoint = TCP4ClientEndpoint(reactor, host, int(port))
agent = ProxyAgent(endpoint)
d = agent.request(b'GET', url)
self.currents += 1
proxy = '{}:{}'.format(host, port)
key = 'proxy_info:' + proxy
if proxy_type == 'rookies_checking':
def _callback(ignored):
pipe = self.conn.pipeline(False)
pipe.zrem('rookies_checking', proxy)
pipe.hset(key, 'failed_times', 0)
# Move proxy from rookies to availables
pipe.smove('rookie_proxies', 'available_proxies',
'{}://{}'.format(protocol, proxy))
pipe.zadd('availables_checking', proxy, time.time() + 30)
pipe.execute()
def _errback(err):
if self.conn.hincrby(key, 'failed_times', 1) < 3:
# If not reach the maximum of failed_times
# Since it is not important so re-check it after 10 seconds
self.conn.zadd('rookies_checking', proxy, time.time() + 10)
else:
pipe = self.conn.pipeline(False)
pipe.zrem('rookies_checking', proxy)
pipe.smove('rookie_proxies', 'dead_proxies',
'{}://{}'.format(protocol, proxy))
pipe.execute()
else:
def _callback(ignored):
pipe = self.conn.pipeline(False)
pipe.hset(key, 'failed_times', 0)
pipe.zadd('availables_checking', proxy, time.time() + 30)
pipe.smove('lost_proxies', 'available_proxies',
'{}://{}'.format(protocol, proxy))
pipe.execute()
def _errback(err):
pipe = self.conn.pipeline(False)
if self.conn.hincrby(key, 'failed_times', 1) < 3:
pipe.zadd('availables_checking', proxy, time.time() + 10)
pipe.smove('available_proxies', 'lost_proxies',
'{}://{}'.format(protocol, proxy))
else:
pipe.zrem('availables_checking', proxy)
pipe.smove('lost_proxies', 'dead_proxies',
'{}://{}'.format(protocol, proxy))
pipe.delete(key)
pipe.execute()
d.addCallbacks(_callback, _errback)
reactor.callLater(timeout, d.cancel)
def _clean(ignored):
self.currents -= 1
d.addBoth(_clean)
print(pformat(list(response.headers.getAllRawHeaders())))
print('Response code:', response.code)
print('Response phrase:', response.phrase)
# url = response.headers.getRawHeaders('Location')[0].encode('utf-8')
#
# d = agent.request(b'GET',url, Headers({'Proxy-Authorization': [encode_user]}))
# d.addCallback(cbRequest)
# d.addCallback(get_onePage)
# d = readBody(response)
# d.addCallback(get_onePage)
return d
host= "211.147.239.101"
port = 44370
user_name = base64.b64encode('spider:123456'.encode('utf-8')).strip()
encode_user = b'Basic '+user_name
print(encode_user)
endpoint = TCP4ClientEndpoint(reactor, host, port)
agent = ProxyAgent(endpoint)
d = agent.request(b"GET", b"https://www.baidu.com")
d.addCallback(redirect)
d.addErrback(display)
d.addCallback(lambda ignored: reactor.stop())
reactor.run()
print(pformat(list(response.headers.getAllRawHeaders())))
print('Response code:', response.code)
print('Response phrase:', response.phrase)
# url = response.headers.getRawHeaders('Location')[0].encode('utf-8')
#
# d = agent.request(b'GET',url, Headers({'Proxy-Authorization': [encode_user]}))
# d.addCallback(cbRequest)
# d.addCallback(get_onePage)
d = readBody(response)
d.addCallback(get_onePage)
return d
host_01 = "149.28.192.96"
host = '47.105.165.81'
port = 5527
# header = Headers()
user_name = base64.b64encode('spider:123456'.encode('utf-8')).strip()
encode_user = b'Basic ' + user_name
endpoint = TCP4ClientEndpoint(reactor, host, port)
agent = ProxyAgent(endpoint)
d = agent.request(b"GET", b"http://www.zimuzu.tv/",
Headers({'Proxy-Authorization': [encode_user]}))
d.addCallback(redirect)
d.addErrback(display)
d.addCallback(lambda ignored: reactor.stop())
reactor.run()
