import timeit
from gevent import monkey; monkey.patch_all()
import gevent
from restkit import *
from restkit.globals import set_manager, get_manager
from restkit.manager.mgevent import GeventManager
set_logging("debug")
print "Manager was: %s" % type(get_manager())
set_manager(GeventManager())
print"Manager is set to: %s" %type(get_manager())
urls = [
"http://yahoo.fr",
"http://google.com",
"http://friendpaste.com",
"http://benoitc.io",
"http://couchdb.apache.org"]
allurls = []
for i in range(10):
allurls.extend(urls)
def fetch(u):
r = request(u, follow_redirect=True)
print "RESULT: %s: %s (%s)" % (u, r.status, len(r.body_string()))
def extract():
import timeit
import eventlet
eventlet.monkey_patch()
from restkit import *
from restkit.globals import set_manager, get_manager
from restkit.manager.meventlet import EventletManager
#set_logging("debug")
print "Manager was: %s" % type(get_manager())
set_manager(EventletManager())
print "Manager is set to: %s" % type(get_manager())
pool = eventlet.GreenPool()
urls = [
"http://yahoo.fr", "http://google.com", "http://friendpaste.com",
"http://benoitc.io", "http://couchdb.apache.org"
]
allurls = []
for i in range(10):
allurls.extend(urls)
def fetch(u):
r = request(u, follow_redirect=True)
print "RESULT: %s: %s (%s)" % (u, r.status, len(r.body_string()))
def __init__(self,
follow_redirect=False,
force_follow_redirect=False,
max_follow_redirect=MAX_FOLLOW_REDIRECTS,
filters=None,
decompress=True,
max_status_line_garbage=None,
max_header_count=0,
manager=None,
response_class=None,
timeout=None,
use_proxy=False,
max_tries=5,
wait_tries=1.0,
**ssl_args):
"""
Client parameters
~~~~~~~~~~~~~~~~~
:param follow_redirect: follow redirection, by default False
:param max_ollow_redirect: number of redirections available
:filters: http filters to pass
:param decompress: allows the client to decompress the response
body
:param max_status_line_garbage: defines the maximum number of ignorable
lines before we expect a HTTP response's status line. With
HTTP/1.1 persistent connections, the problem arises that broken
scripts could return a wrong Content-Length (there are more
bytes sent than specified). Unfortunately, in some cases, this
cannot be detected after the bad response, but only before the
next one. So the client is abble to skip bad lines using this
limit. 0 disable garbage collection, None means unlimited number
of tries.
:param max_header_count: determines the maximum HTTP header count
allowed. by default no limit.
:param manager: the manager to use. By default we use the global
one.
:parama response_class: the response class to use
:param timeout: the default timeout of the connection
(SO_TIMEOUT)
:param max_tries: the number of tries before we give up a
connection
:param wait_tries: number of time we wait between each tries.
:param ssl_args: named argument, see ssl module for more
informations
"""
self.follow_redirect = follow_redirect
self.force_follow_redirect = force_follow_redirect
self.max_follow_redirect = max_follow_redirect
self.decompress = decompress
self.filters = filters or []
self.max_status_line_garbage = max_status_line_garbage
self.max_header_count = max_header_count
self.use_proxy = use_proxy
self.request_filters = []
self.response_filters = []
self.load_filters()
# set manager
if manager is None:
manager = get_manager()
self._manager = manager
# change default response class
if response_class is not None:
self.response_class = response_class
self.max_tries = max_tries
self.wait_tries = wait_tries
self.timeout = timeout
self._nb_redirections = self.max_follow_redirect
self._url = None
self._initial_url = None
self._write_cb = None
self._headers = None
self._sock_key = None
self._sock = None
self._original = None
self.method = 'GET'
self.body = None
self.ssl_args = ssl_args or {}
def __init__(self,
follow_redirect=False,
force_follow_redirect=False,
max_follow_redirect=MAX_FOLLOW_REDIRECTS,
filters=None,
decompress=True,
max_status_line_garbage=None,
max_header_count=0,
manager=None,
response_class=None,
timeout=None,
use_proxy=False,
max_tries=5,
wait_tries=1.0,
**ssl_args):
"""
Client parameters
~~~~~~~~~~~~~~~~~
:param follow_redirect: follow redirection, by default False
:param max_ollow_redirect: number of redirections available
:filters: http filters to pass
:param decompress: allows the client to decompress the response
body
:param max_status_line_garbage: defines the maximum number of ignorable
lines before we expect a HTTP response's status line. With
HTTP/1.1 persistent connections, the problem arises that broken
scripts could return a wrong Content-Length (there are more
bytes sent than specified). Unfortunately, in some cases, this
cannot be detected after the bad response, but only before the
next one. So the client is abble to skip bad lines using this
limit. 0 disable garbage collection, None means unlimited number
of tries.
:param max_header_count: determines the maximum HTTP header count
allowed. by default no limit.
:param manager: the manager to use. By default we use the global
one.
:parama response_class: the response class to use
:param timeout: the default timeout of the connection
(SO_TIMEOUT)
:param max_tries: the number of tries before we give up a
connection
:param wait_tries: number of time we wait between each tries.
:param ssl_args: named argument, see ssl module for more
informations
"""
self.follow_redirect = follow_redirect
self.force_follow_redirect = force_follow_redirect
self.max_follow_redirect = max_follow_redirect
self.decompress = decompress
self.filters = filters or []
self.max_status_line_garbage = max_status_line_garbage
self.max_header_count = max_header_count
self.use_proxy = use_proxy
self.request_filters = []
self.response_filters = []
self.load_filters()
# set manager
if manager is None:
manager = get_manager()
self._manager = manager
# change default response class
if response_class is not None:
self.response_class = response_class
self.max_tries = max_tries
self.wait_tries = wait_tries
self.timeout = timeout
self._nb_redirections = self.max_follow_redirect
self._url = None
self._initial_url = None
self._write_cb = None
self._headers = None
self._sock_key = None
self._sock = None
self._original = None
self.method = 'GET'
self.body = None
self.ssl_args = ssl_args or {}
