import timeit from gevent import monkey; monkey.patch_all() import gevent from restkit import * from restkit.globals import set_manager, get_manager from restkit.manager.mgevent import GeventManager set_logging("debug") print "Manager was: %s" % type(get_manager()) set_manager(GeventManager()) print"Manager is set to: %s" %type(get_manager()) urls = [ "http://yahoo.fr", "http://google.com", "http://friendpaste.com", "http://benoitc.io", "http://couchdb.apache.org"] allurls = [] for i in range(10): allurls.extend(urls) def fetch(u): r = request(u, follow_redirect=True) print "RESULT: %s: %s (%s)" % (u, r.status, len(r.body_string())) def extract():
import timeit import eventlet eventlet.monkey_patch() from restkit import * from restkit.globals import set_manager, get_manager from restkit.manager.meventlet import EventletManager #set_logging("debug") print "Manager was: %s" % type(get_manager()) set_manager(EventletManager()) print "Manager is set to: %s" % type(get_manager()) pool = eventlet.GreenPool() urls = [ "http://yahoo.fr", "http://google.com", "http://friendpaste.com", "http://benoitc.io", "http://couchdb.apache.org" ] allurls = [] for i in range(10): allurls.extend(urls) def fetch(u): r = request(u, follow_redirect=True) print "RESULT: %s: %s (%s)" % (u, r.status, len(r.body_string()))
def __init__(self, follow_redirect=False, force_follow_redirect=False, max_follow_redirect=MAX_FOLLOW_REDIRECTS, filters=None, decompress=True, max_status_line_garbage=None, max_header_count=0, manager=None, response_class=None, timeout=None, use_proxy=False, max_tries=5, wait_tries=1.0, **ssl_args): """ Client parameters ~~~~~~~~~~~~~~~~~ :param follow_redirect: follow redirection, by default False :param max_ollow_redirect: number of redirections available :filters: http filters to pass :param decompress: allows the client to decompress the response body :param max_status_line_garbage: defines the maximum number of ignorable lines before we expect a HTTP response's status line. With HTTP/1.1 persistent connections, the problem arises that broken scripts could return a wrong Content-Length (there are more bytes sent than specified). Unfortunately, in some cases, this cannot be detected after the bad response, but only before the next one. So the client is abble to skip bad lines using this limit. 0 disable garbage collection, None means unlimited number of tries. :param max_header_count: determines the maximum HTTP header count allowed. by default no limit. :param manager: the manager to use. By default we use the global one. :parama response_class: the response class to use :param timeout: the default timeout of the connection (SO_TIMEOUT) :param max_tries: the number of tries before we give up a connection :param wait_tries: number of time we wait between each tries. :param ssl_args: named argument, see ssl module for more informations """ self.follow_redirect = follow_redirect self.force_follow_redirect = force_follow_redirect self.max_follow_redirect = max_follow_redirect self.decompress = decompress self.filters = filters or [] self.max_status_line_garbage = max_status_line_garbage self.max_header_count = max_header_count self.use_proxy = use_proxy self.request_filters = [] self.response_filters = [] self.load_filters() # set manager if manager is None: manager = get_manager() self._manager = manager # change default response class if response_class is not None: self.response_class = response_class self.max_tries = max_tries self.wait_tries = wait_tries self.timeout = timeout self._nb_redirections = self.max_follow_redirect self._url = None self._initial_url = None self._write_cb = None self._headers = None self._sock_key = None self._sock = None self._original = None self.method = 'GET' self.body = None self.ssl_args = ssl_args or {}