def __init__(self, url):
HTTPClientFactory.__init__(self, url)
self.deferred = defer.Deferred()
self.waiting = 1
self.on_page_part = Event()
self.on_page_end = Event()
def __init__(self, export_info, log, media_ids, title, description,
options):
self.export_info = export_info
self.log = log
self.media_ids = media_ids
self.title = title
self.description = description
self.options = options
self.status = 0
self.headers = {}
self.response = ""
self.body = self._build_post()
header_dict = {
'Content-Type':
"application/atom+xml",
'Authorization':
"AuthSub token=\"%s\"" %
self.export_info['password'].encode("utf-8")
}
HTTPClientFactory.__init__(
self,
"http://www.blogger.com/feeds/%s/posts/default" %
self.export_info['service_extra'].encode("utf-8"),
"POST",
self.body,
agent="Zoto/3.0.1",
headers=header_dict)
def __init__(self, temp_token, log):
self.temp_token = temp_token
self.log = log
self.url = "https://www.google.com/accounts/AuthSubSessionToken"
"""
self.keyfile = aztk_config.services.get('servers.httpserver', "google_ssl_priv_key")
self.log.debug("keyfile: [%s]" % self.keyfile)
self.passphrase = aztk_config.services.get('servers.httpserver', "google_ssl_priv_passphrase")
self.time_stamp = int(time.time())
self.rand_num = str(getrandbits(64))
self.data = "GET %s %s %s" % (self.url, self.time_stamp, self.rand_num)
"""
#self.make_sig()
header_dict = {
'Accept': "*/*",
#'Authorization': str("AuthSub token=\"%s\" data=\"%s\" sig=\"%s\" sigalg=\"rsa-sha1\"" % (self.temp_token, self.data, self.sig)),
'Authorization': "AuthSub token=\"%s\"" % (self.temp_token),
'Content-Type': "application/x-www-form-urlencoded"
}
self.log.debug("header_dict:\n%s" % pformat(header_dict))
HTTPClientFactory.__init__(self, self.url, headers=header_dict, agent="Zoto/3.0.1")
self.deferred.addCallback(self.handle_response)
self.deferred.addErrback(self.handle_error)
self.perm_token = ""
self.expire_time = ""
def getPage(url, bindAddress = None, *arg, **kw):
# reimplemented here to insert bindAddress
# _parse() deprecated in twisted 13.1.0 in favor of the _URI class
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
# _URI class renamed to URI in 15.0.0
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPClientFactory(url, *arg, **kw)
factory.noisy = False
if scheme == 'https':
from twisted.internet import ssl
context = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, context,
bindAddress = bindAddress)
else:
reactor.connectTCP(host, port, factory, bindAddress = bindAddress)
return factory.deferred
def getPage(url, bindAddress=None, *arg, **kw):
# reimplemented here to insert bindAddress
# _parse() deprecated in twisted 13.1.0 in favor of the _URI class
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
# _URI class renamed to URI in 15.0.0
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPClientFactory(url, *arg, **kw)
factory.noisy = False
if scheme == 'https':
from twisted.internet import ssl
context = ssl.ClientContextFactory()
reactor.connectSSL(host,
port,
factory,
context,
bindAddress=bindAddress)
else:
reactor.connectTCP(host, port, factory, bindAddress=bindAddress)
return factory.deferred
def __init__(self,
cacheDir,
url,
method='GET',
postdata=None,
headers={},
agent="Gazouilleur with Twisted ConditionalPageGetter",
timeout=0,
cookies=None,
followRedirect=True,
redirectLimit=5):
self.cachefile = path.join(cacheDir, get_hash(url))
self.last_modified = None
if path.exists(self.cachefile):
with open(self.cachefile) as cache:
self.last_modified = cache.readline().strip()
headers['If-Modified-Since'] = self.last_modified
HTTPClientFactory.__init__(self,
url,
method=method,
postdata=postdata,
headers=headers,
agent=agent,
timeout=timeout,
cookies=cookies,
followRedirect=followRedirect,
redirectLimit=redirectLimit)
def __init__(self,
url,
method='GET',
postdata=None,
headers={},
agent="Tor2Web (https://github.com/globaleaks/tor2web-3.0)",
timeout=0,
cookies=None,
followRedirect=1):
if url in self.cache:
if 'etag' in self.cache[url]:
headers['etag'] = self.cache[url]['etag']
elif 'last-modified' in self.cache[url]:
headers['if-modified-since'] = self.cache[url]['last-modified']
elif 'date' in self.cache[url]:
headers['if-modified-since'] = self.cache[url]['date']
HTTPClientFactory.__init__(self,
url=url,
method=method,
postdata=postdata,
headers=headers,
agent=agent,
timeout=timeout,
cookies=cookies,
followRedirect=followRedirect)
self.deferred = Deferred()
def __init__(self, url, contextFactory=None, retries=0): url = stripNoPrint(url) if retries > 0: print "Retrying: ", url else: print "Get: ", url self.retries = retries self.url = url self.charset = None scheme, host, port, path = _parse(url) HTTPClientFactory.__init__(self, url, method='GET', postdata=None, headers=None, agent='Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US;' + ' rv:1.9.2.10) Gecko/20100914 Firefox/3.6.10') if scheme == 'https': from twisted.internet import ssl if contextFactory is None: contextFactory = ssl.ClientContextFactory() reactor.connectSSL(host, port, self, contextFactory) else: reactor.connectTCP(host, port, self) self.deferred.addCallbacks(self.getCharset, self.Err) self.deferred.addCallbacks(self.getTitle, self.Err)
def __init__(self, temp_token, log):
self.temp_token = temp_token
self.log = log
self.url = "https://www.google.com/accounts/AuthSubSessionToken"
"""
self.keyfile = aztk_config.services.get('servers.httpserver', "google_ssl_priv_key")
self.log.debug("keyfile: [%s]" % self.keyfile)
self.passphrase = aztk_config.services.get('servers.httpserver', "google_ssl_priv_passphrase")
self.time_stamp = int(time.time())
self.rand_num = str(getrandbits(64))
self.data = "GET %s %s %s" % (self.url, self.time_stamp, self.rand_num)
"""
#self.make_sig()
header_dict = {
'Accept': "*/*",
#'Authorization': str("AuthSub token=\"%s\" data=\"%s\" sig=\"%s\" sigalg=\"rsa-sha1\"" % (self.temp_token, self.data, self.sig)),
'Authorization': "AuthSub token=\"%s\"" % (self.temp_token),
'Content-Type': "application/x-www-form-urlencoded"
}
self.log.debug("header_dict:\n%s" % pformat(header_dict))
HTTPClientFactory.__init__(self,
self.url,
headers=header_dict,
agent="Zoto/3.0.1")
self.deferred.addCallback(self.handle_response)
self.deferred.addErrback(self.handle_error)
self.perm_token = ""
self.expire_time = ""
def __init__(self, cacheDir, url, method='GET', postdata=None, headers={}, agent="Gazouilleur with Twisted ConditionalPageGetter", timeout=0, cookies=None, followRedirect=True, redirectLimit=5):
self.cachefile = path.join(cacheDir, get_hash(url))
self.last_modified = None
if path.exists(self.cachefile):
with open(self.cachefile) as cache:
self.last_modified = cache.readline().strip()
headers['If-Modified-Since'] = self.last_modified
HTTPClientFactory.__init__(self, url, method=method, postdata=postdata, headers=headers, agent=agent, timeout=timeout, cookies=cookies, followRedirect=followRedirect, redirectLimit=redirectLimit)
def __init__(self, user, password, callback, stream_type='spritzer', agent='TwitterHooks', query_params=None):
url = 'http://stream.twitter.com/%s.json' % stream_type
if query_params:
url = '%s?%s' % (url, urllib.urlencode(query_params))
self.auth = base64.b64encode('%s:%s' % (user, password))
self.callback = callback
self.last_retry = 0
HTTPClientFactory.__init__(self, url=url, agent=agent)
def __init__(self, url, proxy_host, proxy_port, *args, **kwargs):
"""
Strips out proxy_host and proxy_port, then passes the rest of the
arguments to base class (HTTPClientFactory)
"""
self.proxy_host = proxy_host
self.proxy_port = proxy_port
HTTPClientFactory.__init__(self, url, *args, **kwargs)
def __init__(self, **kwargs):
self.reqs = kwargs.pop('requests')
self.report = kwargs.pop('report')
self.ip = kwargs.pop('ip')
self.host_port = kwargs.pop('port')
if 'auth' in kwargs:
self.protocol.auth = kwargs.pop('auth')
self.activecons = 0
self.total_reqs = len(self.reqs)
HTTPClientFactory.__init__(self, timeout=15, **kwargs)
def __init__(self, frob, log):
self.frob = frob
self.log = log
self.sig = md5.md5("%sapi_key%sfrob%smethodflickr.auth.getToken" % (SECRET, API_KEY, self.frob)).hexdigest()
url = "http://api.flickr.com/services/rest/?method=flickr.auth.getToken&api_key=%s&frob=%s&api_sig=%s" % (API_KEY, self.frob, self.sig)
self.log.debug("url: %s" % url)
HTTPClientFactory.__init__(self, url, agent="Zoto/3.0.1")
self.deferred.addCallback(self.handle_response)
self.deferred.addErrback(self.handle_error)
self.token = None
def __init__(self, *args, **kwds):
HTTPClientFactory.__init__(self, *args, **kwds)
if self.url in self.cache:
if 'etag' in self.cache[self.url]:
self.headers['etag'] = self.cache[self.url]['etag']
elif 'last-modified' in self.cache[self.url]:
self.headers['if-modified-since'] = self.cache[self.url]['last-modified']
elif 'date' in self.cache[self.url]:
self.headers['if-modified-since'] = self.cache[self.url]['date']
def __init__(self, **kwargs):
self.reqs = kwargs.pop('requests')
self.report = kwargs.pop('report')
self.ip = kwargs.pop('ip')
self.host_port = kwargs.pop('port')
if 'auth' in kwargs:
self.protocol.auth = kwargs.pop('auth')
self.activecons = 0
self.total_reqs = len(self.reqs)
HTTPClientFactory.__init__(self, timeout=15, **kwargs)
def __call__(self):
HTTPClientFactory.__init__(self, self.page_url,
method=self.method, agent=self.agent,
timeout=self.defaults.interval)
BaseMonitor.__call__(self)
# this deferred is created above when
# HTTPClientFactory.__init__() is called.
d = self.deferred
d.addCallback(self.logStatus)
d.addErrback(self.errorHandlerPartialPage)
d.addErrback(self.errorHandlerTimeout)
def __init__(self, frob, log):
self.frob = frob
self.log = log
self.sig = md5.md5("%sapi_key%sfrob%smethodflickr.auth.getToken" %
(SECRET, API_KEY, self.frob)).hexdigest()
url = "http://api.flickr.com/services/rest/?method=flickr.auth.getToken&api_key=%s&frob=%s&api_sig=%s" % (
API_KEY, self.frob, self.sig)
self.log.debug("url: %s" % url)
HTTPClientFactory.__init__(self, url, agent="Zoto/3.0.1")
self.deferred.addCallback(self.handle_response)
self.deferred.addErrback(self.handle_error)
self.token = None
def __call__(self):
HTTPClientFactory.__init__(self,
self.page_url,
method=self.method,
agent=self.agent,
timeout=self.defaults.interval)
BaseMonitor.__call__(self)
# this deferred is created above when
# HTTPClientFactory.__init__() is called.
d = self.deferred
d.addCallback(self.logStatus)
d.addErrback(self.errorHandlerPartialPage)
d.addErrback(self.errorHandlerTimeout)
def send_json(url, **kw):
qurl =url+'?'+urlencode(kw.get('postdata', {}))
# debug(0,'URLSTATS: %s',qurl)
factory = HTTPClientFactory(str(qurl))
factory.noisy = False
if url.startswith('http://'):
reactor.connectTCP(factory.host, factory.port, factory)
elif url.startswith('https://'):
reactor.connectSSL(factory.host, factory.port, factory,
ssl.ClientContextFactory())
else:
raise Exception('Url error: %s' %url)
return factory
def flush(self):
scheme, host, port, path = _parse(self.url)
factory = HTTPClientFactory(self.url, method='POST', postdata=self._buffer.getvalue())
factory.noisy = False
if scheme == 'https':
from twisted.internet import ssl
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
factory.deferred.addCallbacks(self._handle, self._error)
self._buffer = StringIO()
def _gotConn(self, conn, url):
factory = HTTPClientFactory(url, **kwargs)
print "!!", transport
adr = None
p = factory.buildProtocol(adr)
p.makeConnection(transport)
def nest(x):
print "!!"
self._returnConn(transport)
de.callback(x)
factory.deferred.callback(nest)
def getPage(url, contextFactory=None, *args, **kwargs):
url = str(url)
scheme, host, port, path = _parse(url)
factory = HTTPClientFactory(url, *args, **kwargs)
if False: # use a proxy
host, port = 'proxy', 6060
factory.path = url
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def getPage(url, contextFactory=None, *args, **kwargs): url = str(url) scheme, host, port, path = _parse(url) factory = HTTPClientFactory(url, *args, **kwargs) if False: # use a proxy host, port = 'proxy', 6060 factory.path = url if scheme == 'https': from twisted.internet import ssl if contextFactory is None: contextFactory = ssl.ClientContextFactory() reactor.connectSSL(host, port, factory, contextFactory) else: reactor.connectTCP(host, port, factory) return factory.deferred
def __init__(self, url, method='GET', postdata=None, headers=None,
agent="Tor2Web (https://github.com/globaleaks/tor2web-3.0)", timeout=0, cookies=None,
followRedirect=1):
headers = {}
if url in self.cache:
if 'last-modified' in self.cache[url]:
headers['if-modified-since'] = self.cache[url]['last-modified']
elif 'date' in self.cache[url]:
headers['if-modified-since'] = self.cache[url]['date']
HTTPClientFactory.__init__(self, url=url, method=method,
postdata=postdata, headers=headers, agent=agent,
timeout=timeout, cookies=cookies, followRedirect=followRedirect)
self.deferred = Deferred()
def make_http_request(url, method='GET'):
"""
This is a thin wrapper around twisted HTTPClientFactory and
reactor.connect(TCP/SSL).
Code which makes HTTP client requests should accept a parameter
which provides this interface for all HTTP connections, defaulting
to this function. This allows unittests to inject a fake http
request mechanism.
@param url: The url to request.
@param method: The HTTP request method.
@return: A L{Deferred} which fires with the response body.
"""
client = HTTPClientFactory(url, method='POST')
pr = urlparse(url)
if pr.scheme == 'https':
port = pr.port or 443
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(pr.hostname, port, client, contextFactory)
else:
port = pr.port or 80
reactor.connectTCP(pr.hostname, port, client)
return client.deferred
def buildProtocol(self, addr):
if self.timeout:
elapsed = time.time() - self.__start_time
self.timeout = self.timeout - elapsed
if self.timeout <= 0:
self.timeout = 0.1
return HTTPClientFactory.buildProtocol(self, addr)
def __init__(self, ctx, *p, **k):
"""
@param url
@param method
@param postdata
@param headers
@param timeout
@param cookies
@param followRedirect
@param redirectLimit
"""
HTTPClientFactory.__init__(self, *p, agent=self.agent, **k)
self.finished=False
self._ctx=ctx
(self._amethod, self._cdic, _) = ctx
self.eback=self._cdic["e"]
def __init__(self, token, log):
self.re_ID = re.compile(
"^tag:blogger\.com\,[0-9]+:user-([0-9]+)\.blog-([0-9]+)$")
self.token = token
self.log = log
url = "http://www.blogger.com/feeds/default/blogs"
header_dict = {
'Accept': "*/*",
'Authorization': "AuthSub token=\"%s\"" % self.token,
'Content-Type': "application/atom+xml"
}
HTTPClientFactory.__init__(self,
url,
headers=header_dict,
agent="Zoto/3.0.1")
self.deferred.addCallback(self.handle_response)
self.deferred.addErrback(self.handle_error)
def _getPage(url, host, port):
"""
Fetch the body of the given url via HTTP, connecting to the given host
and port.
@param url: The URL to GET
@type url: C{str}
@param host: The hostname to connect to
@type host: C{str}
@param port: The port number to connect to
@type port: C{int}
@return: A deferred; upon 200 success the body of the response is returned,
otherwise a twisted.web.error.Error is the result.
"""
factory = HTTPClientFactory(url)
factory.protocol = HTTPPageGetter
connect(GAIEndpoint(reactor, host, port), factory)
return factory.deferred
def __init__(self,
url,
method='GET',
postdata=None,
headers=None,
agent="Twisted Remotetimer",
timeout=0,
cookies=None,
followRedirect=1,
lastModified=None,
etag=None):
HTTPClientFactory.__init__(self,
url,
postdata=postdata,
agent=agent,
followRedirect=followRedirect,
timeout=timeout,
cookies=cookies)
def __init__(self, url, method='GET', postdata=None, headers=None,
agent=None, timeout=0, cookies=None, followRedirect=1):
self.url = url
if url in feed_storage:
lastModified = time.ctime(feed_storage[url][0])
if headers is not None:
headers['last-modified'] = lastModified
else:
headers = {'last-modified': lastModified}
HTTPClientFactory.__init__(self, url, method=method, postdata=postdata,
headers=headers, agent=agent, timeout=timeout, cookies=cookies,
followRedirect=followRedirect)
self.waiting = True
self.deferred = defer.Deferred()
def _start(self):
self.saved['Request URL'] = self.request_url
# Generate a request id if possible
if uuid:
request_id = str(uuid.uuid1())
self.saved['Request ID'] = request_id
self.headers['X-Request-Id'] = request_id
factory = HTTPClientFactory(url=self.conf['path'],
method=self.conf['method'], postdata=self.conf['data'],
headers=self.headers, agent=self.agent,
timeout=self.conf['timeout'], followRedirect=0)
factory.host = self.headers_host
factory.noisy = False
factory.deferred.addErrback(self._failure_tcp)
factory.deferred.addErrback(self._failure_http)
self._connect(factory)
return factory.deferred
def __init__(self, request):
HTTPClientFactory.__init__(self, request)
self.method = request.method
self.body = request.body or None
self.headers = request.headers
self.cookies = request.cookies
self.start_time = time()
self.status = None
#self.deferred.addCallback(
# lambda data: (data, self.status, self.response_headers))
self._set_connection_attributes(request)
self.deferred = defer.Deferred().addCallback(self._build_response, request)
if self.body is not None:
self.headers['Content-length'] = [len(self.body)]
# just in case a broken http/1.1 decides to keep connection alive
self.headers.setdefault("Connection", "close")
def getPageU(url, contextFactory=None, *args, **kwargs):
"""Download a web page as a unicode object.
Download a page. Return a deferred, which will callback with a
page (as a string) or errback with a description of the error.
See HTTPClientFactory to see what extra args can be passed.
"""
scheme, host, port, path = _parse(url)
factory = HTTPClientFactory(url, *args, **kwargs)
factory.protocol = UnicodeHTTPPageGetter
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def __init__(self,
url,
method='GET',
postdata=None,
headers=None,
agent="Internet-Radio",
timeout=0,
cookies=None,
followRedirect=1,
lastModified=None,
etag=None):
HTTPClientFactory.__init__(self,
url,
method=method,
postdata=postdata,
headers=headers,
agent=agent,
timeout=timeout,
cookies=cookies,
followRedirect=followRedirect)
def __init__(self, export_info, log, media_info, file_data, title,
description):
self.export_info = export_info
self.log = log
self.log.debug("flickr_poster constructor")
self.media_info = media_info
self.file_data = file_data
self.title = title
self.description = description
self.boundary = mimetools.choose_boundary()
self.body = self.build_post()
header_dict = {
'Content-Type': "multipart/form-data; boundary=%s" % self.boundary
}
HTTPClientFactory.__init__(self,
"http://api.flickr.com/services/upload/",
"POST",
self.body,
agent="Zoto/3.0.1",
headers=header_dict)
self.log.debug("flickr_poster ctor exiting")
def __init__(self, *args, **kwargs):
self.authMechanism = None
self.contextFactory = kwargs['factory_contextFactory']
del kwargs['factory_contextFactory']
self.digestMgr = typed(kwargs['digestMgr'], AbstractHTTPDigestMgr)
logger.debug('USING DIGEST MANAGER %r', self.digestMgr)
del kwargs['digestMgr']
self.goodSSLCertHandler = kwargs['goodSSLCertHandler']
del kwargs['goodSSLCertHandler']
self.realm = kwargs['realm']
del kwargs['realm']
self.session = kwargs['session']
del kwargs['session']
# In fact, C{HTTPClientFactory.__init__} sets or erases
# C{self.postdata}, so we rewrite it after calling the base class.
HTTPClientFactory.__init__(self, *args, **kwargs)
def getPage(url, bindAddress=None, *arg, **kw):
# reimplemented here to insert bindAddress
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPClientFactory(url, *arg, **kw)
factory.noisy = False
if scheme == 'https':
from twisted.internet import ssl
context = ssl.ClientContextFactory()
reactor.connectSSL(host,
port,
factory,
context,
bindAddress=bindAddress)
else:
reactor.connectTCP(host, port, factory, bindAddress=bindAddress)
return factory.deferred
def get_json(url, *args, **kwargs):
"""
:param json: JSON data
:param url:
:param args:
:param kwargs:
:return:
"""
j = kwargs.pop('json', None)
if j:
kwargs['postdata'] = as_json(j)
kwargs.setdefault('agent', 'Twisted JSON Adapter')
uri = URI.fromBytes(url)
factory = HTTPClientFactory(url, *args, **kwargs)
factory.noisy = 0
if uri.scheme == b'https':
from twisted.internet import ssl
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(
nativeString(uri.host), uri.port, factory, contextFactory)
else:
reactor.connectTCP(nativeString(uri.host), uri.port, factory)
return factory.deferred.addCallback(_json_loads)
def get_json(url, *args, **kwargs):
"""
:param json: JSON data
:param url:
:param args:
:param kwargs:
:return:
"""
j = kwargs.pop('json', None)
if j:
kwargs['postdata'] = as_json(j)
kwargs.setdefault('agent', 'Twisted JSON Adapter')
uri = URI.fromBytes(url)
factory = HTTPClientFactory(url, *args, **kwargs)
factory.noisy = 0
if uri.scheme == b'https':
from twisted.internet import ssl
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(nativeString(uri.host), uri.port, factory,
contextFactory)
else:
reactor.connectTCP(nativeString(uri.host), uri.port, factory)
return factory.deferred.addCallback(_json_loads)
def _getPage(self, uri, **kwargs):
"""
C{getPage}-like.
"""
url = self.url_template % (uri, )
if not 'headers' in kwargs:
kwargs['headers'] = {}
kwargs["headers"]["Accept"] = "application/json"
if 'timeout' not in kwargs:
kwargs['timeout'] = 10
factory = HTTPClientFactory(url, **kwargs)
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def getPage(url, bindAddress=None, *arg, **kw):
# reimplemented here to insert bindAddress
scheme, host, port, path = _parse(url)
factory = HTTPClientFactory(url, *arg, **kw)
if scheme == 'https':
from twisted.internet import ssl
context = ssl.ClientContextFactory()
reactor.connectSSL(host,
port,
factory,
context,
bindAddress=bindAddress)
else:
reactor.connectTCP(host, port, factory, bindAddress=bindAddress)
return factory.deferred
def __init__(
self,
url,
method="GET",
postdata=None,
headers=None,
agent="Twisted Remotetimer",
timeout=0,
cookies=None,
followRedirect=1,
lastModified=None,
etag=None,
):
HTTPClientFactory.__init__(
self,
url,
method=method,
postdata=postdata,
headers=headers,
agent=agent,
timeout=timeout,
cookies=cookies,
followRedirect=followRedirect,
)
def _getPage(url, descriptor):
"""
Fetch the body of the given url via HTTP, connecting to the given host
and port.
@param url: The URL to GET
@type url: C{str}
@param descriptor: The endpoint descriptor to use
@type descriptor: C{str}
@return: A deferred; upon 200 success the body of the response is returned,
otherwise a twisted.web.error.Error is the result.
"""
point = endpoints.clientFromString(reactor, descriptor)
factory = HTTPClientFactory(url, timeout=10)
point.connect(factory)
return factory.deferred
def got_results(results):
for r in results:
if name == '-'.join(r.name_short):
factory = HTTPClientFactory(r.url)
if factory.scheme == 'https':
if ssl:
reactor.connectSSL(factory.host, factory.port, factory, ssl.ClientContextFactory())
else:
d_result.errback(Exception("{0} is not available because this installation does not have SSL support!".format(name)))
else:
reactor.connectTCP(factory.host, factory.port, factory)
factory.deferred.addCallback(lambda d: got_data(factory, d))
factory.deferred.addErrback(d_result.errback)
return
d_result.errback(Exception("{0} is not available!".format(name)))
def getPage(url, contextFactory=None, *args, **kwargs):
scheme, host, port, path, username, password = _parse(url)
if username and password:
url = scheme + '://' + host + ':' + str(port) + path
basicAuth = encodestring("%s:%s" % (username, password))
authHeader = "Basic " + basicAuth.strip()
AuthHeaders = {"Authorization": authHeader}
if kwargs.has_key("headers"):
kwargs["headers"].update(AuthHeaders)
else:
kwargs["headers"] = AuthHeaders
factory = HTTPClientFactory(url, *args, **kwargs)
reactor.connectTCP(host, port, factory)
return factory.deferred
def _make_request(self, method, url, headers, body):
# This is a modified version of twisted.web.client.getPage() that
# saves the factory object itself instead of the deferred so that
# we can get access to the server status, etc.
from twisted.internet import reactor
self._request = HTTPClientFactory(url,
method=method,
headers=headers,
postdata=body,
cookies=self.cookies,
agent="Python AsyncNOXWSClient",
followRedirect=0)
if self.https:
from twisted.internet import ssl
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(self.host, self.port, self._request,
contextFactory)
else:
reactor.connectTCP(self.host, self.port, self._request)
def getPage(url, contextFactory=None, *args, **kwargs):
"""Download a web page as a string.
Download a page. Return a deferred, which will callback with a
page (as a string) or errback with a description of the error.
See HTTPClientFactory to see what extra args can be passed.
"""
scheme, host, port, path = _parsed(url)
print(host,port)
factory = HTTPClientFactory(url.encode('utf-8'), *args, **kwargs)
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def _RestApiClient__request(cls,
url,
method,
postdata=None,
cookies={},
timeout=None,
credentials=None):
extraHeaders = {}
if postdata is not None:
postdata = json.dumps(postdata)
if credentials is not None:
cred = "%s:%s" % (credentials[0], credentials[1])
extraHeaders["Authorization"] = "Basic " + base64.encodestring(
cred).replace('\012', '')
scheme, host, port, path = _parse(str(url))
# replace multiple slashes in the url to a single slash
# in the name of genericism this might be a bad idea but
# whatever
while "//" in path:
path = path.replace("//", "/")
if path[0] == "/":
path = path[1:]
url = str("%s://%s:%s/%s" % (scheme, host, port, path))
log.debug("REST API Client request: %s %s" % (method, url))
factory = HTTPClientFactory(url,
method=method,
postdata=postdata,
cookies=cookies,
timeout=timeout,
headers=extraHeaders)
reactor.connectTCP(host, port, factory)
return factory
class ConditionalHTTPClientFactory(HTTPClientFactory):
protocol = ConditionalHTTPPageGetter
def __init__(self,
url,
method='GET',
postdata=None,
headers=None,
agent=None,
timeout=0,
cookies=None,
followRedirect=1):
self.url = url
try:
if url in feed_storage:
lastModified = time.ctime(feed_storage[url][0])
if headers is not None:
headers['last-modified'] = lastModified
else:
headers = {'last-modified': lastModified}
except Exception, e:
log.error("Unable to iterate over feed_storage")
HTTPClientFactory.__init__(self,
url,
method=method,
postdata=postdata,
headers=headers,
agent=agent,
timeout=timeout,
cookies=cookies,
followRedirect=followRedirect)
self.waiting = True
self.deferred = defer.Deferred()
def setURL(self, url):
HTTPClientFactory.setURL(self, url)
self.path = url
def __init__(self, url, method='GET', postdata=None, headers=None, agent="Internet-Radio", timeout=0, cookies=None, followRedirect=1, lastModified=None, etag=None): HTTPClientFactory.__init__(self, url, method=method, postdata=postdata, headers=headers, agent=agent, timeout=timeout, cookies=cookies,followRedirect=followRedirect)
def setURL(self, url):
HTTPClientFactory.setURL(self, url)
self.path = url
def buildProtocol(self, addr):
self.p = HTTPClientFactory.buildProtocol(self, addr)
return self.p
def __init__(self, url, keepalive_timeout=0, *args, **kwargs):
self.keepalive_timeout = keepalive_timeout
HTTPClientFactory.__init__(self, url, *args, **kwargs)
def connectionMade(self):
HTTPClientFactory.connectionMade(self)
self.keepalive_received()
self.resetDelay()
