Example #1
0
    def _request(self, request, callback):
        global pnconn_pool

        ## Build URL
        '''
        url = self.origin + '/' + "/".join([
            "".join([ ' ~`!@#$%^&*()+=[]\\{}|;\':",./<>?'.find(ch) > -1 and
                hex(ord(ch)).replace( '0x', '%' ).upper() or
                ch for ch in list(bit)
            ]) for bit in request])
        '''
        url = self.getUrl(request)
        agent = ContentDecoderAgent(
            RedirectAgent(
                Agent(reactor,
                      contextFactory=WebClientContextFactory(),
                      pool=self.ssl and None or pnconn_pool)),
            [('gzip', GzipDecoder)])
        request = agent.request('GET', url, Headers(self.headers), None)

        def received(response):
            finished = Deferred()
            response.deliverBody(PubNubResponse(finished))
            return finished

        def complete(data):
            callback(eval(data))

        request.addCallback(received)
        request.addBoth(complete)
Example #2
0
def request_gzipped_url(url, callback, errback=None, timeout=None, **kwargs):
    ''' Get URL with gzip-decoder support. '''
    agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])
    d = agent.request('GET', url, Headers({'User-Agent': ['gzip']}))

    def handleResponse(response, **kwargs):
        receiverDeferred = Deferred()
        receiverDeferred.addCallback(callback, **kwargs)
        receiver = StringReceiver(receiverDeferred)
        response.deliverBody(receiver)

    if timeout:
        timeoutCall = reactor.callLater(timeout, d.cancel)

        def completed(passthrough):
            if timeoutCall.active():
                timeoutCall.cancel()
            return passthrough

        d.addBoth(completed)

    d.addCallback(handleResponse, **kwargs)
    if errback:
        d.addErrback(errback, **kwargs)
    return d
Example #3
0
    def _request( self, request, callback ) :
        global pnconn_pool

        ## Build URL
        '''
        url = self.origin + '/' + "/".join([
            "".join([ ' ~`!@#$%^&*()+=[]\\{}|;\':",./<>?'.find(ch) > -1 and
                hex(ord(ch)).replace( '0x', '%' ).upper() or
                ch for ch in list(bit)
            ]) for bit in request])
        '''
        url = self.getUrl(request)
        agent       = ContentDecoderAgent(RedirectAgent(Agent(
            reactor,
            contextFactory = WebClientContextFactory(),
            pool = self.ssl and None or pnconn_pool
        )), [('gzip', GzipDecoder)])
        request     = agent.request( 'GET', url, Headers(self.headers), None )

        def received(response):
            finished = Deferred()
            response.deliverBody(PubNubResponse(finished))
            return finished

        def complete(data):
            callback(eval(data))

        request.addCallback(received)
        request.addBoth(complete)
Example #4
0
def request_gzipped_url(url, callback, errback=None, timeout=None, **kwargs):
    ''' Get URL with gzip-decoder support. '''
    agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])
    d = agent.request(
        'GET',
        url,
        Headers({'User-Agent': ['gzip']}))

    def handleResponse(response, **kwargs):
        receiverDeferred = Deferred()
        receiverDeferred.addCallback(callback, **kwargs)
        receiver = StringReceiver(receiverDeferred)
        response.deliverBody(receiver)

    if timeout:
        timeoutCall = reactor.callLater(timeout, d.cancel)

        def completed(passthrough):
            if timeoutCall.active():
                timeoutCall.cancel()
            return passthrough
        d.addBoth(completed)

    d.addCallback(handleResponse, **kwargs)
    if errback:
        d.addErrback(errback, **kwargs)
    return d
Example #5
0
    def __init__(self, reactor, email, password):
        self.reactor = reactor
        self.email = email
        self.password = password

        # Set up an agent for sending HTTP requests.  Uses cookies
        # (part of the authentication), persistent HTTP connection
        # pool, automatic content decoding (gzip)

        # container to keep track of cookies
        self.cookiejar = cookielib.CookieJar()

        # HTTP persistent connection pool
        self.pool = HTTPConnectionPool(self.reactor, persistent=True)
        # for some reason, using >1 connection per host fails
        self.pool.maxPersistentPerHost = 1

        self.agent = ContentDecoderAgent(
            CookieAgent(Agent(self.reactor, pool=self.pool), self.cookiejar),
            [('gzip', GzipDecoder)])

        # this is the token that is used to authenticate API requests
        self.xsrf_token = None
        self.auth_token = None

        # who we are
        self.player_nickname = None
        self.player_guid = None
        self.team = None
        self.ap = None
        self.level = None
        self.start_date = None
        self.new_version = False
        self.inventory_done = False
        self.profile_done = False

        # for keeping track of item inventory
        self.inventory = b07.inventory.Inventory()

        # for keeping track of API requests that are delayed until
        # authentication has completed
        self._deferred_api_requests = []

        # for keeping track of periodic inventory refreshes
        self._periodic_inventory_refresh_delayedcall = None

        # list of functions to call every time inventory is refreshed
        self._on_inventory_refreshed = []

        # do an immediate inventory refresh
        self._first_inventory_ready = self._defer_until_authenticated(
            self._inventory0, (), {})

        # do an immediate profile refresh
        self._first_profile_ready = self._defer_until_authenticated(
            self._profile0, (), {})

        # start the authentication process
        self.reactor.callLater(0, self._authenticate0)
Example #6
0
def main():
    agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])

    d = agent.request('GET', 'http://www.yahoo.com/')
    d.addCallback(printBody)
    d.addErrback(log.err)
    d.addCallback(lambda ignored: reactor.stop())
    reactor.run()
def main():
    agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])

    d = agent.request('GET', 'http://www.yahoo.com/')
    d.addCallback(printBody)
    d.addErrback(log.err)
    d.addCallback(lambda ignored: reactor.stop())
    reactor.run()
Example #8
0
def main():
    agent = ContentDecoderAgent(Agent(reactor), [(b'gzip', GzipDecoder)])

    d = agent.request(b'GET', b'http://httpbin.org/gzip')
    d.addCallback(printBody)
    d.addErrback(log.err)
    d.addCallback(lambda ignored: reactor.stop())
    reactor.run()
Example #9
0
 def __init__(self, *argz, **kwz):
     super(txOneDriveAPI, self).__init__(*argz, **kwz)
     pool = self.request_pool = QuietHTTPConnectionPool(
         reactor,
         debug_requests=self.debug_requests,
         **self.request_pool_options)
     self.request_agent = ContentDecoderAgent(
         RedirectAgent(
             Agent(reactor,
                   TLSContextFactory(self.ca_certs_files),
                   pool=pool)), [('gzip', GzipDecoder)])
Example #10
0
 def assert_title(self, cassette_name, expected):
     cassette_agent = CassetteAgent(Agent(reactor),
                                    cassette_path(cassette_name),
                                    preserve_exact_body_bytes=True)
     agent = ContentDecoderAgent(RedirectAgent(cassette_agent),
                                 [('gzip', GzipDecoder)])
     finished = agent.request(
         'GET', 'http://127.0.0.1:5000/{}'.format(cassette_name))
     finished.addCallback(self.extractor.extract)
     finished.addCallback(self.assertEqual, expected)
     finished.addBoth(cassette_agent.save)
     return finished
Example #11
0
class HttpClient(object):
    def __init__(self):
        self.cookies = CookieJar()
        self.agent = Agent(reactor)
        self.agent = CookieAgent(self.agent, self.cookies)
        self.agent = ContentDecoderAgent(self.agent, [('gzip', GzipDecoder)])

    def post(self, url, body_producer, headers=None):
        return self.agent.request('POST', url, headers, body_producer)

    def get(self, url, headers=None):
        return self.agent.request('GET', url, headers)
Example #12
0
	def __init__(self, scraper, pool=None):
		self.scraper = scraper
		self._pool = pool

		self._agents = {} #map proxy->an agent


		redirectLimit = scraper.config.get('max_redirects')
		if redirectLimit is None:
			redirectLimit = 3

		#create an agent for direct requests
		self._direct_agent = Agent(reactor, pool=self._pool, connectTimeout=scraper.config.get('timeout') or 30)
		if redirectLimit>0:
			self._direct_agent = BrowserLikeRedirectAgent(self._direct_agent, redirectLimit=redirectLimit)
		
		self._direct_agent = ContentDecoderAgent(self._direct_agent, [('gzip', GzipDecoder)])
		self.cj = self.scraper.client.opener.cj
		

		if self.cj is not None:
			
			self._direct_agent = CookieAgent(self._direct_agent, self.cj)

		#create an agent for http-proxy requests
		#no endpoint yet, use __ instead of _ to backup the instance
		self.__http_proxy_agent = ProxyAgent(None, pool=self._pool) 

		if redirectLimit>0:
			self._http_proxy_agent = BrowserLikeRedirectAgent(self.__http_proxy_agent, redirectLimit=redirectLimit)

			self._http_proxy_agent = ContentDecoderAgent(self._http_proxy_agent, [('gzip', GzipDecoder)])
		else:

			self._http_proxy_agent = ContentDecoderAgent(self.__http_proxy_agent, [('gzip', GzipDecoder)])
			

		if self.cj is not None:
			self._http_proxy_agent = CookieAgent(self._http_proxy_agent, self.cj)

		#create an agent for https-proxy requests
		#no endpoint yet, use __ instead of _ to backup the instance
		self.__https_proxy_agent = TunnelingAgent(reactor=reactor, proxy=None, contextFactory=ScrapexClientContextFactory(), connectTimeout=30, pool=self._pool) #no proxy yet
		if redirectLimit>0:
			self._https_proxy_agent = BrowserLikeRedirectAgent(self.__https_proxy_agent, redirectLimit=redirectLimit)

			self._https_proxy_agent = ContentDecoderAgent(self._https_proxy_agent, [('gzip', GzipDecoder)])
		else:
			self._https_proxy_agent = ContentDecoderAgent(self.__https_proxy_agent, [('gzip', GzipDecoder)])

			
		if self.cj is not None:
			self._https_proxy_agent = CookieAgent(self._https_proxy_agent, self.cj)
Example #13
0
    def _setUp(self):
        super(HTTPTest, self)._setUp()

        try:
            import OpenSSL
        except:
            log.err("Warning! pyOpenSSL is not installed. https websites will "
                    "not work")

        self.control_agent = TrueHeadersSOCKS5Agent(
            reactor,
            proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1',
                                             config.tor.socks_port))

        self.report['socksproxy'] = None
        if self.localOptions['socksproxy']:
            try:
                sockshost, socksport = self.localOptions['socksproxy'].split(
                    ':')
                self.report['socksproxy'] = self.localOptions['socksproxy']
            except ValueError:
                raise InvalidSocksProxyOption
            socksport = int(socksport)
            self.agent = TrueHeadersSOCKS5Agent(
                reactor,
                proxyEndpoint=TCP4ClientEndpoint(reactor, sockshost,
                                                 socksport))
        else:
            self.agent = TrueHeadersAgent(reactor)

        self.report['agent'] = 'agent'

        if self.followRedirects:
            try:
                self.control_agent = FixedRedirectAgent(self.control_agent)
                self.agent = FixedRedirectAgent(
                    self.agent,
                    ignorePrivateRedirects=self.ignorePrivateRedirects)
                self.report['agent'] = 'redirect'
            except:
                log.err("Warning! You are running an old version of twisted "
                        "(<= 10.1). I will not be able to follow redirects."
                        "This may make the testing less precise.")

        if len(self.contentDecoders) > 0:
            self.control_agent = ContentDecoderAgent(self.control_agent,
                                                     self.contentDecoders)
            self.agent = ContentDecoderAgent(self.agent, self.contentDecoders)
        self.processInputs()
        log.debug("Finished test setup")
Example #14
0
    def call(self, action, params=None, callback=None):
        if params is None:
            params = StalkerRequest.getDefaults(action)
        headers = StalkerRequest.getHeaders(self._identity,
                                            action,
                                            referer=self.baseurl)
        headers["X-User-Agent"] = [
            "Model: MAG250; Link: WiFi",
        ]
        url = "%s%s?%s" % (self.baseurl, DEFAULT_ENDPOINT, urlencode(params))
        Log.w(url)
        agent = ContentDecoderAgent(Agent(reactor),
                                    [('gzip', GzipDecoder)])  #Agent(reactor)

        def bodyCB(body):
            if isinstance(body, Failure):
                if isinstance(body.value, PartialDownloadError):
                    body = body.value.response
                else:
                    Log.w(body)
                    callback(None)
                    return
            try:
                result = json.loads(unicode(body))
                Log.d(result)
                callback(result)
            except Exception as e:
                Log.w(body)
                callback(None)

        def bodyErrorCB(error=None):
            Log.w(error)

        def responseCB(response):
            d = readBody(response)
            d.addBoth(bodyCB)

        def errorCB(error=None):
            if (isinstance(error, PartialDownloadError)):
                responseCB(error.response)
                return
            Log.w(error)

        d = agent.request(
            'GET',
            url,
            Headers(headers),
        )
        d.addCallback(responseCB)
        d.addErrback(errorCB)
Example #15
0
def build_agent(req):
    uri = URI.fromBytes(req.url)
    proxy = req.get('proxy')
    if req.get('use_proxy') is False:
        proxy = None

    if proxy:
        if uri.scheme == 'https':

            agent = TunnelingAgent(
                reactor=reactor,
                proxy=proxy,
                contextFactory=ScrapexClientContextFactory(),
                connectTimeout=req.get('timeout'))
        else:
            endpoint = TCP4ClientEndpoint(reactor,
                                          host=proxy.host,
                                          port=proxy.port,
                                          timeout=req.get('timeout'))
            agent = ProxyAgent(endpoint)
            if proxy.auth_header:
                req.get('headers')['Proxy-Authorization'] = proxy.auth_header
    else:
        agent = Agent(reactor)

    agent = RedirectAgent(agent, redirectLimit=3)
    agent = ContentDecoderAgent(agent, [('gzip', GzipDecoder)])
    return agent
Example #16
0
    def run(self) -> Type[Tuple]:
        agent = Agent(reactor, connectTimeout=self._timeout)
        # Add the gzip decoder
        if self._compressed:
            agent = ContentDecoderAgent(agent, [(b"gzip", GzipDecoder)])

        binaryPayloadRequestProducer = _BinaryPayloadRequestProducer(
            self._payload,
            self._meta,
            self._isPayloadGzipped,
        )
        # Make the web request
        response = yield agent.request(
            self._httpMethod,
            self._url,
            Headers(self._headers),
            binaryPayloadRequestProducer,
        )

        self._meta = binaryPayloadRequestProducer.meta
        self._meta.code = response.code
        self._meta.version = response.version
        self._meta.headers = {k.decode(): v[0].decode() for k, v in response.headers.getAllRawHeaders()}

        # Get the responseTuple data
        responseProducer = self._cbResponse(response, self._meta)

        self._meta = responseProducer.meta
        return self._meta
Example #17
0
 def __init__(self, hs):
     SimpleHttpClient.__init__(self, hs)
     # clobber the base class's agent and UA:
     self.agent = ContentDecoderAgent(
         BrowserLikeRedirectAgent(
             Agent.usingEndpointFactory(reactor,
                                        SpiderEndpointFactory(hs))),
         [(b'gzip', GzipDecoder)])
Example #18
0
def get(url, data=None, on_response=None, on_error=None):
    errback = on_error or make_errback(frames_back=2)
    try:

        def handle_response(response):
            if response.code == 200:
                response.deliverBody(JsonReceiver.create(on_response, errback))
            else:
                errback("returned %s" % response.code)

        agent = ContentDecoderAgent(Agent(reactor, pool=pool), [("gzip", GzipDecoder)])
        headers = Headers(get_auth_headers())
        headers.addRawHeader("User-Agent", "gzip")
        d = agent.request("GET", url, headers=headers, bodyProducer=JsonProducer(data) if data else None)
        d.addCallbacks(handle_response, errback)
    except Exception as ex:
        errback("error %s" % ex)
Example #19
0
    def __init__(self, reactor, email, password):
        self.reactor = reactor
        self.email = email
        self.password = password
        
        # Set up an agent for sending HTTP requests.  Uses cookies
        # (part of the authentication), persistent HTTP connection
        # pool, automatic content decoding (gzip)

        # container to keep track of cookies
        self.cookiejar = cookielib.CookieJar()

        # HTTP persistent connection pool
        self.pool = HTTPConnectionPool(self.reactor, persistent = True)
        # for some reason, using >1 connection per host fails
        self.pool.maxPersistentPerHost = 1

        self.agent = ContentDecoderAgent(CookieAgent(Agent(self.reactor,
                                                           pool = self.pool),
                                                     self.cookiejar),
                                         [('gzip', GzipDecoder)])

        # this is the token that is used to authenticate API requests
        self.xsrf_token = None
        self.auth_token = None
        
        # who we are
        self.player_nickname = None
        self.player_guid = None
        self.team = None
        self.ap = None
        self.level = None
        self.start_date = None
        self.new_version = False
        self.inventory_done = False
        self.profile_done = True

        # for keeping track of item inventory
        self.inventory = b07.inventory.Inventory()

        # for keeping track of API requests that are delayed until
        # authentication has completed
        self._deferred_api_requests = []

        # for keeping track of periodic inventory refreshes
        self._periodic_inventory_refresh_delayedcall = None

        # list of functions to call every time inventory is refreshed
        self._on_inventory_refreshed = []

        # do an immediate inventory refresh
        self._first_inventory_ready = self._defer_until_authenticated(self._inventory0, (), {})

        # do an immediate profile refresh
        # self._first_profile_ready = self._defer_until_authenticated(self._profile0, (), {})

        # start the authentication process
        self.reactor.callLater(0, self._authenticate0)
Example #20
0
 def __init__(self):
     #: The Twisted Web `Agent` used to make HTTP requests.
     self.agent = ContentDecoderAgent(
         RedirectAgent(BlacklistingAgent(Agent(reactor))),
         [('gzip', GzipDecoder)])
     #: A dictionary of extractors enabled for this fetcher.
     self.extractors = default_extractors
     #: The maximum number of "soft" redirects to follow per request.
     self.max_soft_redirects = 2
Example #21
0
	def _create_agent(self, req):

		""" create right agent for specific request """

		agent = None

		uri = URI.fromBytes(req.url)
		proxy = req.get('proxy')
		if req.get('use_proxy') is False:
			proxy = None
		
		if proxy:	
			if uri.scheme == 'https':
				
				agent_key = 'httpsproxy-%s-%s' % (proxy.host, proxy.port)
				agent = self._agents.get(agent_key)

				if not agent:
					
					agent = TunnelingAgent(reactor=reactor, proxy=proxy, contextFactory=ScrapexClientContextFactory(), connectTimeout=30, pool=self._pool)

					self._agents[agent_key] = agent

			else:
				#http
				agent_key = 'httpproxy-%s-%s' % (proxy.host, proxy.port)
				agent = self._agents.get(agent_key)

				if not agent:
					endpoint = TCP4ClientEndpoint(reactor, host=proxy.host, port=proxy.port , timeout=req.get('timeout'))
					agent = ProxyAgent(endpoint, pool=self._pool)
					self._agents[agent_key] = agent


				if proxy.auth_header:
					req.get('headers')['Proxy-Authorization'] = proxy.auth_header

		else:
			
			agent = self._direct_agent #use single agent when no proxies used


		redirectLimit = self.scraper.config.get('max_redirects')
		if redirectLimit is None:
			redirectLimit = 3
	
		if redirectLimit>0:
			agent = BrowserLikeRedirectAgent(agent, redirectLimit=redirectLimit)

		
		agent = ContentDecoderAgent(agent, [('gzip', GzipDecoder)])

		if self.cj is not None:
			agent = CookieAgent(agent, self.cj)
		
		return agent	
Example #22
0
    def __init__(self, **config):
        for k, v in config.viewitems():
            try:
                x = getattr(self, k)
            except AttributeError:
                raise AttributeError(
                    'Unrecognized configuration key: {}'.format(k))
            if isinstance(x, Mapping) and isinstance(v, Mapping):
                v = AttrDict(v)
                v.rebase(AttrDict(x))
            setattr(self, k, v)

        pool = QuietHTTPConnectionPool(reactor, persistent=True)
        for k, v in self.request_pool_options.viewitems():
            getattr(pool, k)  # to somewhat protect against typos
            setattr(pool, k, v)
        self.request_agent = ContentDecoderAgent(
            RedirectAgent(
                Agent(reactor,
                      TLSContextFactory(self.ca_certs_files),
                      pool=pool)), [('gzip', GzipDecoder)])
Example #23
0
def main():
    """Main command line entry point."""
    parser = argparse.ArgumentParser(
        description='Make requests to one or more HTTP or HTTPS URIs, '
                    'and record the interactions in a cassette.',
        epilog='If no URIs are passed on the command line, they are '
               'read from standard input, one per line.')
    parser.add_argument(
        'uris', metavar='URI', nargs='*', help='URI to fetch')
    parser.add_argument(
        'cassette_path', metavar='CASSETTE',
        help='path to output cassette')
    args = parser.parse_args()
    uris = args.uris or imap(lambda x: x.strip(), sys.stdin)
    cassette_agent = CassetteAgent(Agent(reactor), args.cassette_path)
    agent = ContentDecoderAgent(
        RedirectAgent(cassette_agent), [('gzip', GzipDecoder)])
    finished = DeferredList([agent.request('GET', uri) for uri in uris])
    finished.addCallback(save_and_exit, cassette_agent)
    finished.addErrback(fail_and_exit)
    reactor.run()
Example #24
0
def get(url, data=None, on_response=None, on_error=None):
    errback = on_error or make_errback(frames_back=2)
    try:

        def handle_response(response):
            if response.code == 200:
                response.deliverBody(JsonReceiver.create(on_response, errback))
            else:
                errback('returned %s' % response.code)

        agent = ContentDecoderAgent(Agent(reactor, pool=pool),
                                    [('gzip', GzipDecoder)])
        headers = Headers(get_auth_headers())
        headers.addRawHeader('User-Agent', 'gzip')
        d = agent.request('GET',
                          url,
                          headers=headers,
                          bodyProducer=JsonProducer(data) if data else None)
        d.addCallbacks(handle_response, errback)
    except Exception as ex:
        errback('error %s' % ex)
def fetch(method, url, headers={}, data=''):
	""" Fetches a page and returns a deferred with the response """
	headers = Headers(headers)
	d = Deferred()
	response_data = {'code':0, 'content':None, 'headers':Headers({})}
	class StringPrinter(Protocol):
		# used to record the result from a url fetch
		def __init__(self):
			self.buffer = ''
		def dataReceived(self, bytes):
			self.buffer = self.buffer + bytes
		def connectionLost(self, reason):
			response_data['content'] = self.buffer
			d.callback(response_data)
	class StringProducer(object):
		implements(IBodyProducer)
		def __init__(self, data):
			self.data = data
			self.length = len(data)
		def startProducing(self, consumer):
			consumer.write(self.data)
			return succeed(None)
		def pauseProducing(self):
			pass
		def stopProducing(self):
			pass

	def onResponse(response):
		# proxied response started coming back, headers are load
		response_data['code'] = response.code
		response_data['headers'] = response.headers
		response.deliverBody(StringPrinter())

	# start the connection
	agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])
	body = StringProducer(data)
	fetcher = agent.request(method, url, headers, body)
	fetcher.addCallback(onResponse)
	fetcher.addErrback(d.errback)
	return d
Example #26
0
	def __init__(self, **kwz):
		for k, v in kwz.viewitems():
			getattr(self, k) # to somewhat protect against typos
			if v is not None: setattr(self, k, v)

		pool = QuietHTTPConnectionPool(reactor, persistent=True)
		for k, v in self.request_pool_options.viewitems():
			getattr(pool, k) # to somewhat protect against typos
			setattr(pool, k, v)
		self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
			reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])

		self.fetch_cache = dict() # {url: {header_name: processed_value, ...}, ...}
def proxy_response(request, url):
	""" Send a new request to the given url, return a deferred with content """
	d = Deferred()
	response_data = {'code':0, 'content':None, 'headers':Headers({})}
	class StringPrinter(Protocol):
		# used to record the result from a url fetch
		def __init__(self):
			self.buffer = ''
		def dataReceived(self, bytes):
			self.buffer = self.buffer + bytes
		def connectionLost(self, reason):
			response_data['content'] = self.buffer
			d.callback(response_data)
	def onResponse(response):
		# proxied response started coming back, headers are load ed
		response_data['code'] = response.code
		response_data['headers'] = response.headers
		response.deliverBody(StringPrinter())

	class RequestWritingPrinter(Protocol):
		# used to send from the proxied response to the original request
		def __init__(self, request):
			self.request = request
		def dataReceived(self, bytes):
			self.request.write(bytes)
		def connectionLost(self, reason):
			self.request.finish()

	# start the connection
	agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])
	body = FileBodyProducer(request.content)
	headers = request.requestHeaders
	headers.setRawHeaders('Host', [urlparse(url)[1]])
	headers.removeHeader('Content-Length')
	fetcher = agent.request(request.method, url, headers, body)
	fetcher.addCallback(onResponse)
	fetcher.addErrback(d.errback)
	return d
Example #28
0
    def run(self):
        # convert tuple to json
        body = self._postTuple.tupleToRestfulJsonDict()
        body = json.dumps(body).encode("utf-8")

        # add http headers
        headers = Headers({
            "User-Agent": ["synerty/1.0"],
            "Content-Type": ["application/json"]
        })

        # Add the gzip decoder
        agent = ContentDecoderAgent(Agent(reactor), [(b"gzip", GzipDecoder)])

        # Make the web request
        response = yield agent.request(self._httpMethod, self._url, headers,
                                       _BytesProducer(body))

        # Get the response data
        responseData = yield self._cbResponse(response)

        # Convert the bytes into a tuple and return
        return self._parseTuple(responseData)
Example #29
0
class HTTPClient:
    user_agent = b'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:28.0) Gecko/20100101 Firefox/28.0'

    def __init__(self):
        self.agent = ContentDecoderAgent(RedirectAgent(Agent(reactor)),
                                         [(b'gzip', GzipDecoder)])

    @defer.inlineCallbacks
    def fetch(self, url, receiver=None):
        resp = yield self.agent.request(
            b'GET', url.encode('utf-8'),
            Headers({b'User-Agent': [self.user_agent]}))
        resp.error = None
        resp.headers = dict(resp.headers.getAllRawHeaders())

        for k, v in resp.headers.copy().items():
            resp.headers[k.decode('utf-8')] = parse_header(
                v[0].decode('utf-8'))
            del resp.headers[k]

        if 'Content-Type' not in resp.headers.keys():
            resp.headers['Content-Type'] = [
                'application/octet-stream', {
                    'charset': 'utf-8'
                }
            ]
        elif 'charset' not in resp.headers['Content-Type'][1].keys():
            resp.headers['Content-Type'][1]['charset'] = 'utf-8'

        if receiver is None:
            mime_type = resp.headers['Content-Type'][0]
            if mime_type.startswith('image'):
                receiver = ImageReceiver
            elif mime_type == 'application/json':
                receiver = JSONReceiver
            elif mime_type == 'text/html':
                receiver = HTMLReceiver
            else:
                receiver = Receiver

        d = defer.Deferred()
        resp.receiver = receiver
        resp.deliverBody(resp.receiver(resp, d))
        resp.body = yield d

        if resp.length is UNKNOWN_LENGTH:
            resp.length = None  # A null value serves as a good unknown

        defer.returnValue(resp)
Example #30
0
 def __init__(self):
     self.pool = HTTPConnectionPool(reactor, persistent=True)
     self.pool.maxPersistentPerHost = 5  # 默认一个IP最大保持两个链接
     self.pool.cachedConnectionTimeout = 50  # 默认240秒
     contextFactory = WebClientContextFactory()
     raw_agent = Agent(reactor, contextFactory, pool=self.pool)
     agent = RedirectAgent(
         ContentDecoderAgent(raw_agent, [('gzip', GzipDecoder)]))
     self.cookieJar = CookieJar()
     self.agent = CookieAgent(agent, self.cookieJar)
     self.headers = {'User-agent': ['Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1'],
                     'Accept-Language': ['zh-Hans-CN,zh-Hans;q=0.5'],
                     'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'],
                     'Accept-Encoding': ['gb2313,utf-8;q=0.7,*;q=0.7'],
                     'Cache-Control': ['max-age=0']}
Example #31
0
	def __init__(self, **config):
		for k, v in config.viewitems():
			try: x = getattr(self, k)
			except AttributeError:
				raise AttributeError('Unrecognized configuration key: {}'.format(k))
			if isinstance(x, Mapping) and isinstance(v, Mapping):
				v = AttrDict(v)
				v.rebase(AttrDict(x))
			setattr(self, k, v)

		pool = QuietHTTPConnectionPool(reactor, persistent=True)
		for k, v in self.request_pool_options.viewitems():
			getattr(pool, k) # to somewhat protect against typos
			setattr(pool, k, v)
		self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
			reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])
Example #32
0
    def __init__(self, url, callID=0, maxPersistentPerHost=2, useCompression=False, connectTimeout=None):

        self.url = url
        self.connectTimeout = connectTimeout
        self.encoder = self.get_encoder()

        assert IEncoder.providedBy(self.encoder), 'no encoder available or encoder does not provide IEncoder'
        assert isinstance(callID, (int, long)), "callID must be <type 'int'> or <type 'long'>"

        self.__callID = callID
        self.__callsCounter = 0
        if maxPersistentPerHost > 0:
            self.pool = HTTPConnectionPool(reactor, persistent=True)
            self.pool.maxPersistentPerHost = maxPersistentPerHost
        else:
            self.pool = None

        agent = Agent(reactor, connectTimeout=self.connectTimeout, pool=self.pool)
        if useCompression:
            self.agent = ContentDecoderAgent(agent, [('gzip', GzipDecoder)])
        else:
            self.agent = agent
Example #33
0
class TitleFetcher(object):
    """Does exactly what it says on the tin."""

    def __init__(self):
        #: The Twisted Web `Agent` used to make HTTP requests.
        self.agent = ContentDecoderAgent(
            RedirectAgent(BlacklistingAgent(Agent(reactor))),
            [('gzip', GzipDecoder)])
        #: A dictionary of extractors enabled for this fetcher.
        self.extractors = default_extractors
        #: The maximum number of "soft" redirects to follow per request.
        self.max_soft_redirects = 2

    @inlineCallbacks
    def fetch_title(self, url, hostname_tag=False, friendly_errors=False):
        """Fetch the document at *url* and return a `Deferred` yielding
        the document title or summary as a Unicode string.  *url* may be
        a Unicode string IRI, a byte string URI, or a Twisted `URL`.

        If *hostname_tag* is true, prefix the extracted title with the
        hostname of the initially requested URI or IRI in the form that
        was originally provided, as well as the hostname of the final
        ASCII-only URI if it differs due to redirects or normalization.

        If *friendly_errors* is true, catch common connection errors and
        return a description of the error as the extracted title instead
        of reraising.  Otherwise, all errors bubble to the caller.
        """
        title = None
        if isinstance(url, unicode):
            url = URL.fromText(url)
        elif isinstance(url, str):
            url = URL.fromText(url.decode('ascii'))
        current = url
        response = None
        for _ in xrange(self.max_soft_redirects):
            last_response = response
            # This encoding should be safe, since asURI() only returns
            # URIs with ASCII code points.
            request = self.agent.request(
                'GET', current.asURI().asText().encode('ascii'))
            if friendly_errors:
                request.addErrback(describe_error)
            response = yield request
            if isinstance(response, basestring):
                # We got an error message from describe_error.  Bail.
                title = response
                break
            response.setPreviousResponse(last_response)
            content_type = cgi.parse_header(
                response.headers.getRawHeaders('Content-Type', [''])[0])[0]
            if content_type in self.extractors:
                extractor = self.extractors[content_type]
                extracted = yield extractor.extract(response)
                if isinstance(extracted, Redirect):
                    current = URL.fromText(
                        response.request.absoluteURI.decode('ascii')).click(
                        extracted.location)
                    continue
                title = extracted
            # The only case where we'd want to loop again is when the
            # response returned is a soft redirect.
            break
        else:
            if friendly_errors:
                title = u'Encountered too many redirects.'
            else:
                raise ResponseFailed([Failure(InfiniteRedirection(
                    599, 'Too many soft redirects',
                    location=current.asURI().asText().encode('ascii')))])
        if title is None:
            title = u'{} document'.format(content_type or u'Unknown')
            if response.length is not UNKNOWN_LENGTH:
                title += u' ({})'.format(filesize(response.length))
        if hostname_tag:
            tag = url.host
            if isinstance(response, Response):
                initial = url.host
                final = URL.fromText(
                    response.request.absoluteURI.decode('ascii')).host
                if initial != final:
                    tag = u'{} \u2192 {}'.format(initial, final)
            title = u'[{}] {}'.format(tag, title)
        returnValue(title)
Example #34
0
 def _setContentDecoder(self, proxy):
     proxy.agent = ContentDecoderAgent(proxy.agent, [('gzip', GzipDecoder)])
Example #35
0
 def __init__(self):
     self.cookies = CookieJar()
     self.agent = Agent(reactor)
     self.agent = CookieAgent(self.agent, self.cookies)
     self.agent = ContentDecoderAgent(self.agent, [('gzip', GzipDecoder)])
Example #36
0
 def __init__(self, hosts, clacks, pool, reactor, extraHeaders, anonymous):
     self._clacks = clacks
     self._hosts = hosts
     self._agent = ContentDecoderAgent(Agent(reactor, pool=pool), [('gzip', GzipDecoder)])
     self._extraHeaders = extraHeaders
     self._anonymous = anonymous
Example #37
0
class Command(BaseCommand):
    help = 'Mass geocode using Google Geocode API'

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        self.agent = ContentDecoderAgent(Agent(reactor),
                                         [(b'gzip', GzipDecoder)])
        self.deferred_results = []

    def add_arguments(self, parser):
        parser.add_argument(
            'infile',
            nargs='?',
            type=argparse.FileType('r'),
            help=
            'Input JSON file, should be a list of objects with "address" property',
            default=sys.stdin)
        parser.add_argument(
            'outfile',
            nargs='?',
            type=argparse.FileType('w'),
            help=
            ('Output JSON file, the properties from input file will be preserved. '
             'Two fields will be added to each object: "lat" and "lng"'),
            default=sys.stdout)
        parser.add_argument(
            '--coordinators-poolsize',
            type=int,
            default=20,
            help=
            'Coordinators pool size, defaults to 20, larger mean faster download'
        )

    def geocode_api_url(self, address):
        return (
            f'https://maps.googleapis.com/maps/api/geocode/json?address={urllib.quote_plus(address)}'
            f'&components=country:US&key={settings.GOOGLE_GEOCODE_APIKEY}'
        ).encode('ascii', 'ignore')

    def handle_readbody_err(self, failure):
        failure.trap(PartialDownloadError)
        return failure.value.response

    def response_received(self, response):
        return readBody(response)

    def updateLatLng(self, result, index, pbar):
        obj = self.store[index]
        try:
            obj['geocode_result'] = json.loads(
                result)['results'][0]['geometry']['location']
        except (IndexError, KeyError):
            obj['geocode_result'] = 'error'
        pbar.update(1)

    def outputResult(self, result, outfile, pbar):
        pbar.close()
        outfile.write(json.dumps(self.store))

    def printError(self, error):
        self.stderr.write(error)

    def generate_requests(self, pbar):
        for ind, obj in enumerate(self.store):
            result = self.agent.request('GET', self.geocode_api_url(obj['address']))\
                .addCallback(self.response_received)\
                .addErrback(self.handle_readbody_err)\
                .addCallback(self.updateLatLng, ind, pbar)\
                .addErrback(self.printError)
            self.deferred_results.append(result)
            yield result

    def kickstart(self, infile, outfile, coordinators_poolsize):
        self.store = json.loads(infile.read())
        pbar = tqdm(total=len(self.store))
        generator = self.generate_requests(pbar)
        DeferredList([
                cooperate(generator).whenDone()
                for _ in range(coordinators_poolsize)
            ])\
            .addCallback(lambda _: gatherResults(self.deferred_results))\
            .addCallback(self.outputResult, outfile, pbar)\
            .addErrback(self.printError)\
            .addBoth(lambda ign: reactor.callWhenRunning(reactor.stop))

    def handle(self, infile, outfile, coordinators_poolsize, *args, **options):
        reactor.callLater(0.1, self.kickstart, infile, outfile,
                          coordinators_poolsize)
        reactor.run()
Example #38
0
class HTTPClient(object):

	use_cache_headers = True
	request_pool_options = dict(
		maxPersistentPerHost=10,
		cachedConnectionTimeout=600,
		retryAutomatically=True )
	ca_certs_files = b'/etc/ssl/certs/ca-certificates.crt'
	user_agent = b'bordercamp-irc-bot/{} twisted/{}'\
		.format(bordercamp.__version__, twisted.__version__)
	hide_connection_errors = False
	sync_fallback_timeout = 180 # timeout for synchronous fallback requests in a thread

	def __init__(self, **kwz):
		for k, v in kwz.viewitems():
			getattr(self, k) # to somewhat protect against typos
			if v is not None: setattr(self, k, v)

		pool = QuietHTTPConnectionPool(reactor, persistent=True)
		for k, v in self.request_pool_options.viewitems():
			getattr(pool, k) # to somewhat protect against typos
			setattr(pool, k, v)
		self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
			reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])

		self.fetch_cache = dict() # {url: {header_name: processed_value, ...}, ...}

	@defer.inlineCallbacks
	def sync_wrap(self, func, *argz, **kwz):
		timeout = defer.Deferred()
		reactor.callLater( self.sync_fallback_timeout,
			lambda: not timeout.called and timeout.callback(SyncTimeout) )
		def thread_wrapper():
			# Do not let exception to be raise in a thread - twisted logs these as unhandled
			try: return func(*argz, **kwz)
			except Exception as err: return SyncError(err)
		res = yield first_result(timeout, threads.deferToThread(thread_wrapper))
		if res is SyncTimeout: raise res()
		if isinstance(res, SyncError): raise res.args[0]
		defer.returnValue(res)

	@defer.inlineCallbacks
	def request(self, url, method='get', decode=None, encode=None, data=None):
		method, url = force_bytes(method).upper(), force_bytes(url)
		headers = {'User-Agent': self.user_agent}

		if method == 'GET' and self.use_cache_headers:
			# Avoid doing extra work
			cache = self.fetch_cache.get(url, dict())
			if 'cache-control' in cache and cache['cache-control'] >= time.time():
				defer.returnValue(None) # no need to re-process same thing
			if 'last-modified' in cache:
				headers['If-Modified-Since'] = rfc822date(cache['last-modified'])
			if 'etag' in cache: headers['If-None-Match'] = '"{}"'.format(cache['etag'])

		log.noise( 'HTTP request: {} {} (h: {}, enc: {}, dec: {}, data: {!r})'\
			.format(method, url[:100], headers, encode, decode, type(data)) )

		if data is not None:
			if encode is None:
				if isinstance(data, types.StringTypes): data = io.BytesIO(data)
			elif encode == 'form':
				headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
				data = io.BytesIO(urlencode(data))
			elif encode == 'json':
				headers.setdefault('Content-Type', 'application/json')
				data = io.BytesIO(json.dumps(data))
			else: raise ValueError('Unknown request encoding: {}'.format(encode))
			data_raw, data = data, FileBodyProducer(data)
		else: data_raw = None
		if decode not in ['json', None]:
			raise ValueError('Unknown response decoding method: {}'.format(decode))

		requests = None # indicates fallback to requests module (for e.g. ipv6-only site)
		err = None
		try:
			res = yield self.request_agent.request( method, url,
				Headers(dict((k,[v]) for k,v in (headers or dict()).viewitems())), data )
		except error.DNSLookupError:
			import requests, socket
			try:
				res = yield self.sync_wrap(
					getattr(requests, method.lower()), url, headers=headers, data=data_raw )
			except ( socket.error, SyncTimeout,
				requests.exceptions.RequestException ) as err: pass
		except ( RequestTransmissionFailed,
			RequestNotSent, ResponseFailed ) as err: pass

		if err:
			if not self.hide_connection_errors:
				raise HTTPClientError(None, 'Lookup/connection error: {}'.format(err))
			else:
				log.debug('Lookup/connection error (supressed): {}'.format(err))
				defer.returnValue(None) # should also supress fast refetching

		code, phrase, version = (res.code, res.phrase, res.version)\
			if not requests else ( res.status_code,
				http.RESPONSES[res.status_code], ('HTTP', 1, 1) )
		log.noise( 'HTTP request done ({} {}): {} {} {}'\
			.format(method, url[:100], code, phrase, version) )
		if code in [http.NO_CONTENT, http.NOT_MODIFIED]: defer.returnValue(None)
		if code not in [http.OK, http.CREATED]: raise HTTPClientError(code, phrase)

		if not requests:
			data = defer.Deferred()
			res.deliverBody(DataReceiver(data))
			data = yield data
			headers = dict((k, v[-1]) for k,v in res.headers.getAllRawHeaders())
		else:
			try:
				data = yield self.sync_wrap(getattr, res, 'text')
				headers = yield self.sync_wrap(getattr, res, 'headers')
			except (requests.exceptions.RequestException, SyncTimeout) as err:
				raise HTTPClientError(None, 'Sync connection error: {}'.format(err))

		if method == 'GET' and self.use_cache_headers:
			cache = dict((k.lower(), v) for k,v in headers.items())
			cache = dict( (k, cache[k]) for k in
				['last-modified', 'cache-control', 'etag'] if k in cache )
			# Update headers' cache
			if 'last-modified' in cache:
				ts = rfc822.parsedate_tz(cache['last-modified'])
				cache['last-modified'] = time.mktime(ts[:9]) + (ts[9] or 0)
			if 'cache-control' in cache:
				match = re.search(r'\bmax-age=(\d+)\b', cache.pop('cache-control'))
				if match: cache['cache-control'] = time.time() + int(match.group(1))
			if cache: self.fetch_cache[url] = cache

		defer.returnValue((json.loads(data) if decode is not None else data, headers))
Example #39
0
class HTTPClient(object):

	#: Options to twisted.web.client.HTTPConnectionPool
	request_pool_options = dict(
		maxPersistentPerHost=10,
		cachedConnectionTimeout=600,
		retryAutomatically=True )

	#: Settings for a simple retry mechanism on http error codes
	retry = None

	#: Path string or list of strings
	ca_certs_files = b'/etc/ssl/certs/ca-certificates.crt'

	#: Dump HTTP request data in debug log (insecure!)
	debug_requests = False


	def __init__(self, **config):
		for k, v in config.viewitems():
			try: x = getattr(self, k)
			except AttributeError:
				raise AttributeError('Unrecognized configuration key: {}'.format(k))
			if isinstance(x, Mapping) and isinstance(v, Mapping):
				v = AttrDict(v)
				v.rebase(AttrDict(x))
			setattr(self, k, v)

		pool = QuietHTTPConnectionPool(reactor, persistent=True)
		for k, v in self.request_pool_options.viewitems():
			getattr(pool, k) # to somewhat protect against typos
			setattr(pool, k, v)
		self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
			reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])


	@defer.inlineCallbacks
	def request_with_retries(self, url, method, **request_kwz):
		'''Gets repeated if any of "raise_for" errors or HTTPClientError is raised.
			When number of attempts runs out, raises last error.'''
		assert 'url' not in request_kwz and 'method' not in request_kwz
		request_kwz['url'], request_kwz['method'] = url, method

		log = request_kwz.get('log')
		if not log:
			log = logging.getLogger(__name__)
			request_kwz['log'] = log

		# Shortcut for simple cases
		if not self.retry or self.retry.attempts <= 1:
			defer.returnValue((yield self.request(**request_kwz)))

		retry_on = [HTTPClientError]
		for cls in request_kwz.get('raise_for', dict()).viewvalues():
			assert issubclass(cls, Exception), cls
			retry_on.append(cls)

		if isinstance(self.retry.delay, (tuple, list)):
			delays = list(self.retry.delay)
			for i in xrange(self.retry.attempts - len(delays), 1):
				delays.append(delays[-1])
		else: delays = [float(self.retry.delay)] * (self.retry.attempts - 1)
		delays = list(reversed(delays))

		while True:
			try: defer.returnValue((yield self.request(**request_kwz)))
			except tuple(retry_on) as err:
				if not delays: raise # no attempts left
				delay, d = delays.pop(), defer.Deferred()
				reactor.callLater(delay, d.callback, None)
				if self.debug_requests:
					log.debug( 'Introducing delay after failed'
						' ({}: {}) request: {}s'.format(type(err), err, delay) )
				yield d


	@defer.inlineCallbacks
	def request( self, url, method='get', decode=None,
			encode=None, data=None, chunks=True, headers=dict(),
			raise_for=dict(), queue_lines=None, log=None ):
		'''Make HTTP(S) request.
			decode (response body) = None | json
			encode (data) = None | json | form | files'''
		if not log: log = logging.getLogger(__name__)

		if self.debug_requests:
			log.debug( 'HTTP request: {} {} (h: {}, enc: {}, dec: {}, data: {!r})'\
				.format(method, url[:100], headers, encode, decode, data) )
		headers = dict() if not headers else headers.copy()
		headers.setdefault('User-Agent', 'lafs-backup-tool')

		if data is not None:
			if encode == 'files':
				boundary = os.urandom(16).encode('hex')
				headers.setdefault('Content-Type', 'multipart/form-data; boundary={}'.format(boundary))
				data = MultipartDataSender(data, boundary)
				yield data.calculate_length()
			else:
				if encode is None:
					if isinstance(data, types.StringTypes): data = io.BytesIO(data)
				elif encode == 'form':
					headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
					data = io.BytesIO(urlencode(data))
				elif encode == 'json':
					headers.setdefault('Content-Type', 'application/json')
					data = io.BytesIO(json.dumps(data))
				else: raise ValueError('Unknown request encoding: {}'.format(encode))
				data = (ChunkingFileBodyProducer if chunks else FileBodyProducer)(data)

		if isinstance(url, unicode): url = url.encode('utf-8')
		if isinstance(method, unicode): method = method.lower().encode('ascii')
		if decode not in ['json', None]:
			raise ValueError('Unknown response decoding method: {}'.format(decode))

		code = None
		try:
			res = yield self.request_agent.request(
				method.upper(), url,
				Headers(dict((k,[v]) for k,v in (headers or dict()).viewitems())), data )
			code = res.code
			if self.debug_requests:
				log.debug( 'HTTP request done ({} {}): {} {} {}'\
					.format(method, url[:100], code, res.phrase, res.version) )
			if code in raise_for: raise HTTPClientError(code, res.phrase)
			if code == http.NO_CONTENT: defer.returnValue(None)
			if code not in [http.OK, http.CREATED]: raise HTTPClientError(code, res.phrase)

			data = defer.Deferred()
			if queue_lines is None: res.deliverBody(DataReceiver(data))
			else: res.deliverBody(LineQueue(data, queue_lines))

			data = yield data
			defer.returnValue(json.loads(data) if decode is not None else data)

		except HTTPClientError as err:
			raise raise_for.get(code, HTTPClientError)(code, err.message)
Example #40
0
class API(object):

    class URLS:
        CLIENT_LOGIN = '******'
        SERVICE_LOGIN = '******'
        APPENGINE =  'https://appengine.google.com'
        GAME_API = 'https://betaspike.appspot.com'
        INGRESS =  'http://www.ingress.com'

    class PATHS:
        LOGIN = '******'
        CONFLOGIN = '******'
        class API:
            HANDSHAKE =  '/handshake'
            DROP_ITEM = '/rpc/gameplay/dropItem'
            RECYCLE = '/rpc/gameplay/recycleItem'
            SAY = '/rpc/player/say'
            INVENTORY =  '/rpc/playerUndecorated/getInventory'
            PLEXTS = '/rpc/playerUndecorated/getPaginatedPlexts'
        class INTEL:
            BASE = '/intel'
            PLEXTS = '/rpc/dashboard.getPaginatedPlextsV2'

    HANDSHAKE_PARAMS = {'nemesisSoftwareVersion' : '2013-07-12T15:48:09Z d6f04b1fab4f opt',
                        'deviceSoftwareVersion' : '4.1.1'}

    def __init__(self, reactor, email, password):
        self.reactor = reactor
        self.email = email
        self.password = password
        
        # Set up an agent for sending HTTP requests.  Uses cookies
        # (part of the authentication), persistent HTTP connection
        # pool, automatic content decoding (gzip)

        # container to keep track of cookies
        self.cookiejar = cookielib.CookieJar()

        # HTTP persistent connection pool
        self.pool = HTTPConnectionPool(self.reactor, persistent = True)
        # for some reason, using >1 connection per host fails
        self.pool.maxPersistentPerHost = 1

        self.agent = ContentDecoderAgent(CookieAgent(Agent(self.reactor,
                                                           pool = self.pool),
                                                     self.cookiejar),
                                         [('gzip', GzipDecoder)])

        # this is the token that is used to authenticate API requests
        self.xsrf_token = None
        self.auth_token = None
        
        # who we are
        self.player_nickname = None
        self.player_guid = None
        self.team = None
        self.ap = None
        self.level = None

        # for keeping track of item inventory
        self.inventory = b07.inventory.Inventory()

        # for keeping track of API requests that are delayed until
        # authentication has completed
        self._deferred_api_requests = []

        # for keeping track of periodic inventory refreshes
        self._periodic_inventory_refresh_delayedcall = None

        # list of functions to call every time inventory is refreshed
        self._on_inventory_refreshed = []

        # do an immediate inventory refresh
        self._first_inventory_ready = self._defer_until_authenticated(self._inventory0, (), {})

        # refresh inventory periodically
        self._first_inventory_ready.addCallback(self._setup_periodic_inventory_refresh)

        # start the authentication process
        self.reactor.callLater(0, self._authenticate0)

    def refreshInventory(self):
        """Manually start an inventory refresh."""

        return self._defer_until_authenticated(self._inventory0, (), {})

    def onInventoryRefreshed(self, callback, *args, **kw):
        self._on_inventory_refreshed.append((callback, args, kw))

    def err(self, failure):
        log_failure(failure)
        
    def getInventory(self):
        return self.inventory

    def _defer_until_authenticated(self, func, args, kw):
        kw['finished'] = defer.Deferred()

        if self.xsrf_token is None:
            self._deferred_api_requests.append((func, args, kw))

        else:
            self.reactor.callLater(0, func, *args, **kw)
        
        return kw['finished']

    def _process_deferred_api_requests(self):
        while self._deferred_api_requests:
            func, args, kw = self._deferred_api_requests.pop(0)
            self.reactor.callLater(0, func, *args, **kw)

    def _authenticate0(self):
        auth_params = {'Email': self.email,
                       'Passwd': self.password,
                       'service': 'ah',
                       'source': 'IngressBot',
                       'accountType': 'HOSTED_OR_GOOGLE'}

        body = b07.utils.StringProducer(urllib.urlencode(auth_params))

        d = self.agent.request('POST',
                               self.URLS.CLIENT_LOGIN,
                               Headers({'User-Agent' : ['Nemesis (gzip)'],
                                        'Content-Type': ['application/x-www-form-urlencoded'],
                                        'Accept-Charset': ['utf-8']}),
                               body)
        d.addCallback(self._authenticate1)
        d.addErrback(self.err)

    def _authenticate1(self, response):
        finished = defer.Deferred()
        finished.addCallback(self._authenticate2, response.code)
        lp = b07.utils.LoginProtocol(finished)
        response.deliverBody(lp)

    def _authenticate2(self, result, code):
        if code == 200:
            try:
                self.auth_token = result['Auth']

            except KeyError:
                critical('Authentication failed: Bad Response')

        elif code == 403:
            error = result['Error']
        
            if error == 'BadAuthentication':
                critical('Authentication failed: Username or password wrong')

            elif error == 'NotVerified':
                critical('Authentication failed: Account email address has not been verified')

            elif error == 'TermsNotAgreed':
                critical('Authentication failed: User has not agreed to Googles terms of service')

            elif error == 'CaptchaRequired':
                critical('Authentication failed: CAPTCHA required')

            elif error == 'AccountDeleted':
                critical('Authentication failed: User account has been deleted')

            elif error == 'AccountDisabled':
                critical('Authentication failed: User account has been disabled')

            elif error == 'ServiceDisabled':
                critical('Authentication failed: Service disabled')

            elif error == 'ServiceUnavailable':
                critical('Authentication failed: Service unavailable')

            else:
                critical('Authentication failed: Unknown reason')

        else:
            critical('Authentication failed: Bad response')
    
        d = self.agent.request('GET',
                               self.URLS.GAME_API + self.PATHS.LOGIN + '?' + urllib.urlencode({'auth' : self.auth_token}),
                               Headers({'User-Agent' : ['Nemesis (gzip)'],
                                        'Accept-Charset': ['utf-8']}),
                               None)
        d.addCallback(self._authenticate3)
        d.addErrback(self.err)

    def _authenticate3(self, response):
        trace('{}'.format(response.code))
        for cookie in self.cookiejar:
            trace('{}'.format(cookie))

        urlParams = {'json' : json.dumps(self.HANDSHAKE_PARAMS)}
        d = self.agent.request('GET',
                               self.URLS.GAME_API + self.PATHS.API.HANDSHAKE + '?' + urllib.urlencode({'json': json.dumps(self.HANDSHAKE_PARAMS)}),
                               Headers({'User-Agent' : ['Nemesis (gzip)'],
                                        'Accept-Charset': ['utf-8'],
                                        'Cache-Control': ['max-age=0']}),
                               None)
        d.addCallback(self._authenticate4)
        d.addErrback(self.err)

    def _authenticate4(self, response):
        trace('{}'.format(response.code))

        if response.code == 200:
            finished = defer.Deferred()
            finished.addCallback(self._authenticate5)
            finished.addErrback(self.err)
            jp = b07.utils.JsonProtocol(finished)
            response.deliverBody(jp)

        else:
            critical('Got response code {} after attempting handshake!'.format(response.code))

    def _authenticate5(self, result):
        result = result['result']

        if result['versionMatch'] != 'CURRENT':
            critical('Software version not up-to-date')

        if 'xsrfToken' not in result:
            critical('Authentication with Ingress severs failed for unknown reason')

        self.xsrf_token = str(result['xsrfToken'])
        self.player_nickname = result['nickname']
        self.player_guid = result['playerEntity'][0]
        self.team = result['playerEntity'][2]['controllingTeam']['team']
        self.ap = result['playerEntity'][2]['playerPersonal']['ap']
        self.level = result['playerEntity'][2]['playerPersonal']['clientLevel']

        debug('XSRF Token:      {}'.format(self.xsrf_token))
        debug('Player GUID:     {}'.format(self.player_guid))
        info('Player nickname: {}'.format(self.player_nickname))
        info('Faction: {}'.format(self.team))
        info('AP: {}'.format(self.ap))
        info('Level: {}'.format(self.level))
        debug('Player info: {}'.format(result))

        self._process_deferred_api_requests()

    def _setup_periodic_inventory_refresh(self, result):
        self._periodic_inventory_refresh_delayedcall = self.reactor.callLater(300, self._periodic_inventory_refresh0)

    def _periodic_inventory_refresh0(self):
        finished = self._defer_until_authenticated(self._inventory0, (), {})
        finished.addCallback(self._periodic_inventory_refresh1)

    def _periodic_inventory_refresh1(self, result):
        self._periodic_inventory_refresh_delayedcall = self.reactor.callLater(300, self._periodic_inventory_refresh0)
    
    def _inventory0(self, finished):
        debug('Requesting inventory from server...')
        body = b07.utils.StringProducer(json.dumps({'params' : {'lastQueryTimestamp': self.inventory.last_query_timestamp}}))
        d = self.agent.request('POST',
                               self.URLS.GAME_API + self.PATHS.API.INVENTORY,
                               Headers({'User-Agent' : ['Nemesis (gzip)'],
                                        'Content-Type': ['application/json;charset=UTF-8'],
                                        'X-XsrfToken': [self.xsrf_token]}),
                               body)
                                    
        d.addCallback(self._inventory1, finished)
        d.addErrback(self.err)

    def _inventory1(self, response, finished):
        if response.code == 500:
            error('Got a 500 SERVER ERROR trying to get the inventory!')

        elif response.code == 200:
            debug('Got 200 OK response to inventory request')
            d = defer.Deferred()
            d.addCallback(self._inventory2, finished)
            jp = b07.utils.JsonProtocol(d)
            response.deliverBody(jp)

        else:
            error('Don\'t know what to do with {} code in response to inventory request!'.format(response.code))

    def _inventory2(self, result, finished):
        self.inventory.process_result(result)
        finished.callback(self.inventory)
        for callback, args, kw in self._on_inventory_refreshed:
            self.reactor.callLater(0, callback, self.inventory, *args, **kw)
Example #41
0
    def http_request(self, url, include_http_responses=False):
        cached_value = yield self.lookup('http_request', url)
        if cached_value is not None:
            if include_http_responses is not True:
                cached_value.pop('responses', None)
            defer.returnValue(cached_value)

        page_info = {
            'body_length': -1,
            'status_code': -1,
            'headers': {},
            'failure': None
        }

        agent = ContentDecoderAgent(
            FixedRedirectAgent(TrueHeadersAgent(reactor)),
                               [('gzip', GzipDecoder)]
        )
        try:
            retries = 0
            while True:
                try:
                    response = yield agent.request('GET', url,
                                                   TrueHeaders(REQUEST_HEADERS))
                    headers = {}
                    for name, value in response.headers.getAllRawHeaders():
                        headers[name] = unicode(value[0], errors='ignore')
                    body_length = -1
                    body = None
                    try:
                        body = yield readBody(response)
                        body_length = len(body)
                    except PartialDownloadError as pde:
                        if pde.response:
                            body_length = len(pde.response)
                            body = pde.response
                    page_info['body_length'] = body_length
                    page_info['status_code'] = response.code
                    page_info['headers'] = headers
                    page_info['title'] = extractTitle(body)
                    response.body = body
                    page_info['responses'] = encodeResponses(response)
                    break
                except:
                    if retries > self.http_retries:
                        raise
                    retries += 1
        except DNSLookupError:
            page_info['failure'] = 'dns_lookup_error'
        except TimeoutError:
            page_info['failure'] = 'generic_timeout_error'
        except ConnectionRefusedError:
            page_info['failure'] = 'connection_refused_error'
        except ConnectError:
            page_info['failure'] = 'connect_error'
        except:
            # XXX map more failures
            page_info['failure'] = 'unknown_error'

        yield self.cache_value('http_request', url, page_info)
        if include_http_responses is not True:
            page_info.pop('responses', None)
        defer.returnValue(page_info)
Example #42
0
    def request(self, method, url, **kwargs):
        """
        See :func:`treq.request()`.
        """
        method = method.encode('ascii').upper()

        if isinstance(url, unicode):
            parsed_url = URL.from_text(url)
        else:
            parsed_url = URL.from_text(url.decode('ascii'))

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            parsed_url = parsed_url.replace(
                query=parsed_url.query + tuple(_coerced_query_params(params)))

        url = parsed_url.to_uri().to_text().encode('ascii')

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.items():
                    if isinstance(v, (bytes, unicode)):
                        h.addRawHeader(k, v)
                    elif isinstance(v, list):
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        # since json=None needs to be serialized as 'null', we need to
        # explicitly check kwargs for this key
        has_json = 'json' in kwargs

        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = str(uuid.uuid4()).encode('ascii')
            headers.setRawHeaders(
                b'content-type',
                [b'multipart/form-data; boundary=' + boundary])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(data + files,
                                                       boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(b'content-type',
                                      [b'application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = self._data_to_body_producer(data)
        elif has_json:
            # If data is sent as json, set Content-Type as 'application/json'
            headers.setRawHeaders(b'content-type',
                                  [b'application/json; charset=UTF-8'])
            content = kwargs['json']
            json = json_dumps(content, separators=(u',', u':')).encode('utf-8')
            bodyProducer = self._data_to_body_producer(json)

        cookies = kwargs.get('cookies', {})

        if not isinstance(cookies, CookieJar):
            cookies = cookiejar_from_dict(cookies)

        cookies = merge_cookies(self._cookiejar, cookies)
        wrapped_agent = CookieAgent(self._agent, cookies)

        if kwargs.get('allow_redirects', True):
            if kwargs.get('browser_like_redirects', False):
                wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
            else:
                wrapped_agent = RedirectAgent(wrapped_agent)

        wrapped_agent = ContentDecoderAgent(wrapped_agent,
                                            [(b'gzip', GzipDecoder)])

        auth = kwargs.get('auth')
        if auth:
            wrapped_agent = add_auth(wrapped_agent, auth)

        d = wrapped_agent.request(method,
                                  url,
                                  headers=headers,
                                  bodyProducer=bodyProducer)

        timeout = kwargs.get('timeout')
        if timeout:
            delayedCall = default_reactor(kwargs.get('reactor')).callLater(
                timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        if not kwargs.get('unbuffered', False):
            d.addCallback(_BufferedResponse)

        return d.addCallback(_Response, cookies)
Example #43
0
    def request(self, method, url, **kwargs):
        method = method.upper()

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.get('params')
        if params:
            url = _combine_query_params(url, params)

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.get('headers')
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.iteritems():
                    if isinstance(v, str):
                        h.addRawHeader(k, v)
                    else:
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        # Here we choose a right producer
        # based on the parameters passed in.
        bodyProducer = None
        data = kwargs.get('data')
        files = kwargs.get('files')
        if files:
            # If the files keyword is present we will issue a
            # multipart/form-data request as it suits better for cases
            # with files and/or large objects.
            files = list(_convert_files(files))
            boundary = uuid.uuid4()
            headers.setRawHeaders(
                'content-type', [
                    'multipart/form-data; boundary=%s' % (boundary,)])
            if data:
                data = _convert_params(data)
            else:
                data = []

            bodyProducer = multipart.MultiPartProducer(
                data + files, boundary=boundary)
        elif data:
            # Otherwise stick to x-www-form-urlencoded format
            # as it's generally faster for smaller requests.
            if isinstance(data, (dict, list, tuple)):
                headers.setRawHeaders(
                    'content-type', ['application/x-www-form-urlencoded'])
                data = urlencode(data, doseq=True)
            bodyProducer = IBodyProducer(data)

        cookies = kwargs.get('cookies', {})

        if not isinstance(cookies, CookieJar):
            cookies = cookiejar_from_dict(cookies)

        cookies = merge_cookies(self._cookiejar, cookies)

        wrapped_agent = CookieAgent(self._agent, cookies)

        if kwargs.get('allow_redirects', True):
            wrapped_agent = RedirectAgent(wrapped_agent)

        wrapped_agent = ContentDecoderAgent(wrapped_agent,
                                            [('gzip', GzipDecoder)])

        auth = kwargs.get('auth')
        if auth:
            wrapped_agent = add_auth(wrapped_agent, auth)

        d = wrapped_agent.request(
            method, url, headers=headers,
            bodyProducer=bodyProducer)

        timeout = kwargs.get('timeout')
        if timeout:
            delayedCall = default_reactor(kwargs.get('reactor')).callLater(
                timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        if not kwargs.get('unbuffered', False):
            d.addCallback(_BufferedResponse)

        return d.addCallback(_Response, cookies)
Example #44
0
    def test_EncodingJSONRPCServer(self):

        DATA = {'foo': 'bar'}
        REQUEST = '{"jsonrpc": "2.0", "method": "test", "params": [], "id": 1}'
        RESPONSE = '{"jsonrpc": "2.0", "id": 1, "result": ' + json.dumps(DATA) + '}'


        class RPCServer(JSONRPCServer):
            def jsonrpc_test(self):
                return defer.succeed(DATA)


        class ReceiverProtocol(Protocol):
            def __init__(self, finished):
                self.finished = finished
                self.body = []

            def dataReceived(self, bytes):
                self.body.append(bytes)

            def connectionLost(self, reason):
                self.finished.callback(''.join(self.body))


        class StringProducer(object):
            implements(IBodyProducer)

            def __init__(self, body):
                self.body = body
                self.length = len(body)

            def startProducing(self, consumer):
                consumer.write(self.body)
                return defer.succeed(None)

            def pauseProducing(self):
                pass

            def stopProducing(self):
                pass


        server = RPCServer()
        resource = EncodingJSONRPCServer(server)
        site = Site(resource)

        port = reactor.listenTCP(8888, site, interface='127.0.0.1')


        agent = ContentDecoderAgent(Agent(reactor), [('gzip', GzipDecoder)])

        response = yield agent.request('POST', 'http://127.0.0.1:8888',
                                       Headers({'Accept-Encoding': ['gzip']}),
                                       StringProducer(REQUEST))

        self.assertTrue(isinstance(response, GzipDecoder))

        finished = defer.Deferred()

        response.deliverBody(ReceiverProtocol(finished))

        data = yield finished

        self.assertEqual(data, RESPONSE)

        port.stopListening()
Example #45
0
 def __init__(self, *args, **kwargs):
     super(Command, self).__init__(*args, **kwargs)
     self.agent = ContentDecoderAgent(Agent(reactor),
                                      [(b'gzip', GzipDecoder)])
     self.deferred_results = []
Example #46
0
    def request(self, method, url, **kwargs):
        """
        See :func:`treq.request()`.
        """
        method = method.encode('ascii').upper()
        stacklevel = kwargs.pop('_stacklevel', 2)

        if isinstance(url, DecodedURL):
            parsed_url = url
        elif isinstance(url, EncodedURL):
            parsed_url = DecodedURL(url)
        elif isinstance(url, six.text_type):
            parsed_url = DecodedURL.from_text(url)
        else:
            parsed_url = DecodedURL.from_text(url.decode('ascii'))

        # Join parameters provided in the URL
        # and the ones passed as argument.
        params = kwargs.pop('params', None)
        if params:
            parsed_url = parsed_url.replace(
                query=parsed_url.query + tuple(_coerced_query_params(params)))

        url = parsed_url.to_uri().to_text().encode('ascii')

        # Convert headers dictionary to
        # twisted raw headers format.
        headers = kwargs.pop('headers', None)
        if headers:
            if isinstance(headers, dict):
                h = Headers({})
                for k, v in headers.items():
                    if isinstance(v, (bytes, six.text_type)):
                        h.addRawHeader(k, v)
                    elif isinstance(v, list):
                        h.setRawHeaders(k, v)

                headers = h
        else:
            headers = Headers({})

        bodyProducer, contentType = self._request_body(
            data=kwargs.pop('data', None),
            files=kwargs.pop('files', None),
            json=kwargs.pop('json', _NOTHING),
            stacklevel=stacklevel,
        )
        if contentType is not None:
            headers.setRawHeaders(b'Content-Type', [contentType])

        cookies = kwargs.pop('cookies', {})

        if not isinstance(cookies, CookieJar):
            cookies = cookiejar_from_dict(cookies)

        cookies = merge_cookies(self._cookiejar, cookies)
        wrapped_agent = CookieAgent(self._agent, cookies)

        browser_like_redirects = kwargs.pop('browser_like_redirects', False)
        if kwargs.pop('allow_redirects', True):
            if browser_like_redirects:
                wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
            else:
                wrapped_agent = RedirectAgent(wrapped_agent)

        wrapped_agent = ContentDecoderAgent(wrapped_agent,
                                            [(b'gzip', GzipDecoder)])

        auth = kwargs.pop('auth', None)
        if auth:
            wrapped_agent = add_auth(wrapped_agent, auth)

        d = wrapped_agent.request(method,
                                  url,
                                  headers=headers,
                                  bodyProducer=bodyProducer)

        reactor = kwargs.pop('reactor', None)
        if reactor is None:
            from twisted.internet import reactor
        timeout = kwargs.pop('timeout', None)
        if timeout:
            delayedCall = reactor.callLater(timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        if not kwargs.pop('unbuffered', False):
            d.addCallback(_BufferedResponse)

        if kwargs:
            warnings.warn(
                ("Got unexpected keyword argument: {}."
                 " treq will ignore this argument,"
                 " but will raise TypeError in the next treq release.").format(
                     ", ".join(repr(k) for k in kwargs)),
                DeprecationWarning,
                stacklevel=stacklevel,
            )

        return d.addCallback(_Response, cookies)
Example #47
0
class RProxyResource(Resource):

    isLeaf = True

    def __init__(self, hosts, clacks, pool, reactor, extraHeaders, anonymous):
        self._clacks = clacks
        self._hosts = hosts
        self._agent = ContentDecoderAgent(Agent(reactor, pool=pool), [('gzip', GzipDecoder)])
        self._extraHeaders = extraHeaders
        self._anonymous = anonymous

    def render(self, request):

        host = self._hosts.get(request.getRequestHostname().lower())

        if not host and request.getRequestHostname().lower().startswith("www."):
            host = self._hosts.get(request.getRequestHostname().lower()[4:])

            # The non-www host doesn't want to match to www.
            if not host["wwwtoo"]:
                host = None

        if not host:
            request.code = 404
            request.responseHeaders.setRawHeaders("Server",
                                                  [__version__.package + " " + __version__.base()])
            return b"I can't seem to find a domain by that name. Look behind the couch?"

        url = "{}://{}:{}/{}".format(
            "https" if host["proxysecure"] else "http",
            host["host"], host["port"], request.path[1:])

        urlFragments = urlparse(request.uri)

        if urlFragments.query:
            url += "?" + urlFragments.query

        for x in [b'content-length', b'connection', b'keep-alive', b'te',
            b'trailers', b'transfer-encoding', b'upgrade',
            b'proxy-connection']:
            request.requestHeaders.removeHeader(x)

        prod = StringProducer(request.content)

        d = self._agent.request(request.method, url,
                                request.requestHeaders, prod)

        def write(res):

            request.code = res.code
            old_headers = request.responseHeaders
            request.responseHeaders = res.headers
            request.responseHeaders.setRawHeaders(
                'content-encoding',
                old_headers.getRawHeaders('content-encoding', []))
            if not self._anonymous:
                request.responseHeaders.addRawHeader("X-Proxied-By",
                                                     __version__.package + " " + __version__.base())

            if request.isSecure() and host["sendhsts"]:
                request.responseHeaders.setRawHeaders("Strict-Transport-Security",
                                                      ["max-age=31536000"])

            if self._clacks:
                request.responseHeaders.addRawHeader("X-Clacks-Overhead",
                                                     "GNU Terry Pratchett")

            for name, values in self._extraHeaders:
                request.responseHeaders.setRawHeaders(name, values)

            f = Deferred()
            res.deliverBody(Downloader(f, request.write))
            f.addCallback(lambda _: request.finish())
            return f

        def failed(res):
            request.setResponseCode(http.INTERNAL_SERVER_ERROR)
            for name, values in self._extraHeaders:
                request.responseHeaders.setRawHeaders(name, values)
            request.write(str(res))
            request.finish()
            return res

        d.addCallback(write)
        d.addErrback(failed)

        return server.NOT_DONE_YET
Example #48
0
	def __init__(self, *argz, **kwz):
		super(txBoxAPI, self).__init__(*argz, **kwz)
		pool = self.request_pool = QuietHTTPConnectionPool( reactor,
				debug_requests=self.debug_requests, **self.request_pool_options )
		self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
			reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])
Example #49
0
class HTTPClient(object):

    #: Options to twisted.web.client.HTTPConnectionPool
    request_pool_options = dict(maxPersistentPerHost=10,
                                cachedConnectionTimeout=600,
                                retryAutomatically=True)

    #: Settings for a simple retry mechanism on http error codes
    retry = None

    #: Path string or list of strings
    ca_certs_files = b'/etc/ssl/certs/ca-certificates.crt'

    #: Dump HTTP request data in debug log (insecure!)
    debug_requests = False

    def __init__(self, **config):
        for k, v in config.viewitems():
            try:
                x = getattr(self, k)
            except AttributeError:
                raise AttributeError(
                    'Unrecognized configuration key: {}'.format(k))
            if isinstance(x, Mapping) and isinstance(v, Mapping):
                v = AttrDict(v)
                v.rebase(AttrDict(x))
            setattr(self, k, v)

        pool = QuietHTTPConnectionPool(reactor, persistent=True)
        for k, v in self.request_pool_options.viewitems():
            getattr(pool, k)  # to somewhat protect against typos
            setattr(pool, k, v)
        self.request_agent = ContentDecoderAgent(
            RedirectAgent(
                Agent(reactor,
                      TLSContextFactory(self.ca_certs_files),
                      pool=pool)), [('gzip', GzipDecoder)])

    @defer.inlineCallbacks
    def request_with_retries(self, url, method, **request_kwz):
        '''Gets repeated if any of "raise_for" errors or HTTPClientError is raised.
			When number of attempts runs out, raises last error.'''
        assert 'url' not in request_kwz and 'method' not in request_kwz
        request_kwz['url'], request_kwz['method'] = url, method

        log = request_kwz.get('log')
        if not log:
            log = logging.getLogger(__name__)
            request_kwz['log'] = log

        # Shortcut for simple cases
        if not self.retry or self.retry.attempts <= 1:
            defer.returnValue((yield self.request(**request_kwz)))

        retry_on = [HTTPClientError]
        for cls in request_kwz.get('raise_for', dict()).viewvalues():
            assert issubclass(cls, Exception), cls
            retry_on.append(cls)

        if isinstance(self.retry.delay, (tuple, list)):
            delays = list(self.retry.delay)
            for i in xrange(self.retry.attempts - len(delays), 1):
                delays.append(delays[-1])
        else:
            delays = [float(self.retry.delay)] * (self.retry.attempts - 1)
        delays = list(reversed(delays))

        while True:
            try:
                defer.returnValue((yield self.request(**request_kwz)))
            except tuple(retry_on) as err:
                if not delays: raise  # no attempts left
                delay, d = delays.pop(), defer.Deferred()
                reactor.callLater(delay, d.callback, None)
                if self.debug_requests:
                    log.debug('Introducing delay after failed'
                              ' ({}: {}) request: {}s'.format(
                                  type(err), err, delay))
                yield d

    @defer.inlineCallbacks
    def request(self,
                url,
                method='get',
                decode=None,
                encode=None,
                data=None,
                chunks=True,
                headers=dict(),
                raise_for=dict(),
                queue_lines=None,
                log=None):
        '''Make HTTP(S) request.
			decode (response body) = None | json
			encode (data) = None | json | form | files'''
        if not log: log = logging.getLogger(__name__)

        if self.debug_requests:
            log.debug( 'HTTP request: {} {} (h: {}, enc: {}, dec: {}, data: {!r})'\
             .format(method, url[:100], headers, encode, decode, data) )
        headers = dict() if not headers else headers.copy()
        headers.setdefault('User-Agent', 'lafs-backup-tool')

        if data is not None:
            if encode == 'files':
                boundary = os.urandom(16).encode('hex')
                headers.setdefault(
                    'Content-Type',
                    'multipart/form-data; boundary={}'.format(boundary))
                data = MultipartDataSender(data, boundary)
                yield data.calculate_length()
            else:
                if encode is None:
                    if isinstance(data, types.StringTypes):
                        data = io.BytesIO(data)
                elif encode == 'form':
                    headers.setdefault('Content-Type',
                                       'application/x-www-form-urlencoded')
                    data = io.BytesIO(urlencode(data))
                elif encode == 'json':
                    headers.setdefault('Content-Type', 'application/json')
                    data = io.BytesIO(json.dumps(data))
                else:
                    raise ValueError(
                        'Unknown request encoding: {}'.format(encode))
                data = (ChunkingFileBodyProducer
                        if chunks else FileBodyProducer)(data)

        if isinstance(url, unicode): url = url.encode('utf-8')
        if isinstance(method, unicode): method = method.lower().encode('ascii')
        if decode not in ['json', None]:
            raise ValueError(
                'Unknown response decoding method: {}'.format(decode))

        code = None
        try:
            res = yield self.request_agent.request(
                method.upper(), url,
                Headers(
                    dict(
                        (k, [v]) for k, v in (headers or dict()).viewitems())),
                data)
            code = res.code
            if self.debug_requests:
                log.debug( 'HTTP request done ({} {}): {} {} {}'\
                 .format(method, url[:100], code, res.phrase, res.version) )
            if code in raise_for: raise HTTPClientError(code, res.phrase)
            if code == http.NO_CONTENT: defer.returnValue(None)
            if code not in [http.OK, http.CREATED]:
                raise HTTPClientError(code, res.phrase)

            data = defer.Deferred()
            if queue_lines is None: res.deliverBody(DataReceiver(data))
            else: res.deliverBody(LineQueue(data, queue_lines))

            data = yield data
            defer.returnValue(json.loads(data) if decode is not None else data)

        except HTTPClientError as err:
            raise raise_for.get(code, HTTPClientError)(code, err.message)
Example #50
0
    def http_request(self, url, http_request_headers, include_http_responses=False):
        key = url + json.dumps(http_request_headers)
        cached_value = yield self.lookup('http_request', key)
        if cached_value is not None:
            if include_http_responses is not True:
                cached_value.pop('responses', None)
            defer.returnValue(cached_value)

        page_info = {
            'body_length': -1,
            'status_code': -1,
            'headers': {},
            'failure': None
        }

        agent = ContentDecoderAgent(
            FixedRedirectAgent(TrueHeadersAgent(reactor),
                               ignorePrivateRedirects=True),
            [('gzip', GzipDecoder)]
        )
        try:
            retries = 0
            while True:
                try:
                    response = yield agent.request('GET', url,
                                                   TrueHeaders(http_request_headers))
                    headers = {}
                    for name, value in response.headers.getAllRawHeaders():
                        headers[name] = unicode(value[0], errors='ignore')
                    body_length = -1
                    body = None
                    try:
                        body = yield readBody(response)
                        body_length = len(body)
                    except PartialDownloadError as pde:
                        if pde.response:
                            body_length = len(pde.response)
                            body = pde.response
                    page_info['body_length'] = body_length
                    page_info['status_code'] = response.code
                    page_info['headers'] = headers
                    page_info['title'] = extractTitle(body)
                    response.body = body
                    page_info['responses'] = encodeResponses(response)
                    break
                except:
                    if retries > self.http_retries:
                        raise
                    retries += 1
        except DNSLookupError:
            page_info['failure'] = 'dns_lookup_error'
        except TimeoutError:
            page_info['failure'] = 'generic_timeout_error'
        except ConnectionRefusedError:
            page_info['failure'] = 'connection_refused_error'
        except ConnectError:
            page_info['failure'] = 'connect_error'
        except Exception as exc:
            # XXX map more failures
            page_info['failure'] = 'unknown_error'
            log.err("Unknown error occurred")
            log.exception(exc)

        yield self.cache_value('http_request', key, page_info)
        if include_http_responses is not True:
            page_info.pop('responses', None)
        defer.returnValue(page_info)
Example #51
0
class API(object):
    class URLS:
        CLIENT_LOGIN = '******'
        SERVICE_LOGIN = '******'
        APPENGINE = 'https://appengine.google.com'
        GAME_API = 'https://betaspike.appspot.com'
        INGRESS = 'http://www.ingress.com'

    class PATHS:
        LOGIN = '******'
        CONFLOGIN = '******'

        class API:
            HANDSHAKE = '/handshake'
            DROP_ITEM = '/rpc/gameplay/dropItem'
            RECYCLE = '/rpc/gameplay/recycleItem'
            SAY = '/rpc/player/say'
            INVENTORY = '/rpc/playerUndecorated/getInventory'
            PROFILE = '/rpc/playerUndecorated/getPlayerProfile'
            PLEXTS = '/rpc/playerUndecorated/getPaginatedPlexts'

        class INTEL:
            BASE = '/intel'
            PLEXTS = '/rpc/dashboard.getPaginatedPlextsV2'

    HANDSHAKE_PARAMS = {
        'nemesisSoftwareVersion': '2015-02-0T23:44:16Z ac501401a6f3 opt',
        'deviceSoftwareVersion': '4.1.1'
    }

    def __init__(self, reactor, email, password):
        self.reactor = reactor
        self.email = email
        self.password = password

        # Set up an agent for sending HTTP requests.  Uses cookies
        # (part of the authentication), persistent HTTP connection
        # pool, automatic content decoding (gzip)

        # container to keep track of cookies
        self.cookiejar = cookielib.CookieJar()

        # HTTP persistent connection pool
        self.pool = HTTPConnectionPool(self.reactor, persistent=True)
        # for some reason, using >1 connection per host fails
        self.pool.maxPersistentPerHost = 1

        self.agent = ContentDecoderAgent(
            CookieAgent(Agent(self.reactor, pool=self.pool), self.cookiejar),
            [('gzip', GzipDecoder)])

        # this is the token that is used to authenticate API requests
        self.xsrf_token = None
        self.auth_token = None

        # who we are
        self.player_nickname = None
        self.player_guid = None
        self.team = None
        self.ap = None
        self.level = None
        self.start_date = None
        self.new_version = False
        self.inventory_done = False
        self.profile_done = False

        # for keeping track of item inventory
        self.inventory = b07.inventory.Inventory()

        # for keeping track of API requests that are delayed until
        # authentication has completed
        self._deferred_api_requests = []

        # for keeping track of periodic inventory refreshes
        self._periodic_inventory_refresh_delayedcall = None

        # list of functions to call every time inventory is refreshed
        self._on_inventory_refreshed = []

        # do an immediate inventory refresh
        self._first_inventory_ready = self._defer_until_authenticated(
            self._inventory0, (), {})

        # do an immediate profile refresh
        self._first_profile_ready = self._defer_until_authenticated(
            self._profile0, (), {})

        # start the authentication process
        self.reactor.callLater(0, self._authenticate0)

    def refreshInventory(self):
        """Manually start an inventory refresh."""

        return self._defer_until_authenticated(self._inventory0, (), {})

    def refreshProfile(self):
        """Manually start a profile refresh."""

        return self._defer_until_authenticated(self._profile0, (), {})

    def onInventoryRefreshed(self, callback, *args, **kw):
        self._on_inventory_refreshed.append((callback, args, kw))

    def err(self, failure):
        log_failure(failure)

    def getInventory(self):
        return self.inventory

    def _defer_until_authenticated(self, func, args, kw):
        kw['finished'] = defer.Deferred()

        if self.xsrf_token is None:
            self._deferred_api_requests.append((func, args, kw))

        else:
            self.reactor.callLater(0, func, *args, **kw)

        return kw['finished']

    def _process_deferred_api_requests(self):
        while self._deferred_api_requests:
            func, args, kw = self._deferred_api_requests.pop(0)
            self.reactor.callLater(0, func, *args, **kw)

    def _authenticate0(self):
        auth_params = {
            'Email': self.email,
            'Passwd': self.password,
            'service': 'ah',
            'source': 'IngressBot',
            'accountType': 'HOSTED_OR_GOOGLE'
        }

        body = b07.utils.StringProducer(urllib.urlencode(auth_params))
        trace('_authenticate0 {}'.format(urllib.urlencode(auth_params)))
        d = self.agent.request(
            'POST', self.URLS.CLIENT_LOGIN,
            Headers({
                'User-Agent': ['Nemesis (gzip)'],
                'Content-Type': ['application/x-www-form-urlencoded'],
                'Accept-Charset': ['utf-8']
            }), body)
        d.addCallback(self._authenticate1)
        d.addErrback(self.err)

    def _authenticate1(self, response):
        finished = defer.Deferred()
        finished.addCallback(self._authenticate2, response.code)
        lp = b07.utils.LoginProtocol(finished)
        response.deliverBody(lp)

    def _authenticate2(self, result, code):
        if code == 200:
            try:
                self.auth_token = result['Auth']

            except KeyError:
                critical('Authentication failed: Bad Response')

        elif code == 403:
            error = result['Error']

            if error == 'BadAuthentication':
                critical('Authentication failed: Username or password wrong')

            elif error == 'NotVerified':
                critical(
                    'Authentication failed: Account email address has not been verified'
                )

            elif error == 'TermsNotAgreed':
                critical(
                    'Authentication failed: User has not agreed to Googles terms of service'
                )

            elif error == 'CaptchaRequired':
                critical('Authentication failed: CAPTCHA required')

            elif error == 'AccountDeleted':
                critical(
                    'Authentication failed: User account has been deleted')

            elif error == 'AccountDisabled':
                critical(
                    'Authentication failed: User account has been disabled')

            elif error == 'ServiceDisabled':
                critical('Authentication failed: Service disabled')

            elif error == 'ServiceUnavailable':
                critical('Authentication failed: Service unavailable')

            else:
                critical('Authentication failed: Unknown reason')

        else:
            critical('Authentication failed: Bad response')

        trace('_authenticate2 {}'.format(self.auth_token))
        d = self.agent.request(
            'GET', self.URLS.GAME_API + self.PATHS.LOGIN + '?' +
            urllib.urlencode({'auth': self.auth_token}),
            Headers({
                'User-Agent': ['Nemesis (gzip)'],
                'Accept-Charset': ['utf-8']
            }), None)
        d.addCallback(self._authenticate3)
        d.addErrback(self.err)

    def _authenticate3(self, response):
        trace('_authenticate3 {}'.format(response.code))
        for cookie in self.cookiejar:
            trace('_authenticate3 {}'.format(cookie))

        urlParams = {'json': json.dumps(self.HANDSHAKE_PARAMS)}
        d = self.agent.request(
            'GET', self.URLS.GAME_API + self.PATHS.API.HANDSHAKE + '?' +
            urllib.urlencode({'json': json.dumps(self.HANDSHAKE_PARAMS)}),
            Headers({
                'User-Agent': ['Nemesis (gzip)'],
                'Accept-Charset': ['utf-8'],
                'Cache-Control': ['max-age=0']
            }), None)
        d.addCallback(self._authenticate4)
        d.addErrback(self.err)

    def _authenticate4(self, response):
        trace('_authenticate4 {}'.format(response.code))

        if response.code == 200:
            finished = defer.Deferred()
            finished.addCallback(self._authenticate5)
            finished.addErrback(self.err)
            jp = b07.utils.JsonProtocol(finished)
            response.deliverBody(jp)

        else:
            critical('Got response code {} after attempting handshake!'.format(
                response.code))

    def _authenticate5(self, result):
        trace('_authenticate5 {}'.format(result))

        result = result['result']

        if 'xsrfToken' not in result:
            critical(
                'Authentication with Ingress severs failed for unknown reason')

        self.xsrf_token = str(result['xsrfToken'])
        self.player_nickname = result['nickname']
        self.player_guid = result['playerEntity'][0]
        self.team = result['playerEntity'][2]['controllingTeam']['team']
        self.ap = result['playerEntity'][2]['playerPersonal']['ap']
        self.level = result['playerEntity'][2]['playerPersonal']['clientLevel']
        start_date = result['storage']['mission_complete_0']
        self.start_date = datetime.datetime.fromtimestamp(
            int(start_date.split(':delim:')[1]) / 1000)

        debug('XSRF Token:      {}'.format(self.xsrf_token))
        debug('Player GUID:     {}'.format(self.player_guid))
        info('Player nickname: {}'.format(self.player_nickname))
        info('Faction:         {}'.format(self.team))
        info('AP:              {}'.format(self.ap))
        info('Level:           {}'.format(self.level))
        info('Start Date:      {}'.format(self.start_date))
        debug('Player info:     {}'.format(result))

        with open(
                os.path.expanduser("~/{}_config.cfg".format(
                    self.player_nickname)), "w") as file:
            json.dump(result, file, indent=1)
        self.new_version = versionCheck(result['serverVersion'],
                                        self.player_nickname)

        self._process_deferred_api_requests()

    def _setup_periodic_inventory_refresh(self, result):
        self._periodic_inventory_refresh_delayedcall = self.reactor.callLater(
            300, self._periodic_inventory_refresh0)

    def _periodic_inventory_refresh0(self):
        finished = self._defer_until_authenticated(self._inventory0, (), {})
        finished.addCallback(self._periodic_inventory_refresh1)

    def _periodic_inventory_refresh1(self, result):
        self._periodic_inventory_refresh_delayedcall = self.reactor.callLater(
            300, self._periodic_inventory_refresh0)

    def _inventory0(self, finished):
        debug('Requesting inventory from server...')
        body = b07.utils.StringProducer(
            json.dumps({
                'params': {
                    'lastQueryTimestamp': self.inventory.last_query_timestamp
                }
            }))
        d = self.agent.request(
            'POST', self.URLS.GAME_API + self.PATHS.API.INVENTORY,
            Headers({
                'User-Agent': ['Nemesis (gzip)'],
                'Content-Type': ['application/json;charset=UTF-8'],
                'X-XsrfToken': [self.xsrf_token]
            }), body)

        d.addCallback(self._inventory1, finished)
        d.addErrback(self.err)

    def _inventory1(self, response, finished):
        if response.code == 500:
            error('Got a 500 SERVER ERROR trying to get the inventory!')

        elif response.code == 200:
            debug('Got 200 OK response to inventory request')
            d = defer.Deferred()
            d.addCallback(self._inventory2, finished)
            jp = b07.utils.JsonProtocol(d)
            response.deliverBody(jp)

        else:
            error(
                'Don\'t know what to do with {} code in response to inventory request!'
                .format(response.code))

    def _inventory2(self, result, finished):
        self.inventory.process_result(result)
        finished.callback(self.inventory)
        for callback, args, kw in self._on_inventory_refreshed:
            self.reactor.callLater(0, callback, self.inventory, *args, **kw)
        self.inventory_done = True

    def _profile0(self, finished):
        debug('Requesting profile from server...')
        body = b07.utils.StringProducer(
            json.dumps({'params': [self.player_nickname]}))
        d = self.agent.request(
            'POST', self.URLS.GAME_API + self.PATHS.API.PROFILE,
            Headers({
                'User-Agent': ['Nemesis (gzip)'],
                'Content-Type': ['application/json;charset=UTF-8'],
                'X-XsrfToken': [self.xsrf_token]
            }), body)

        d.addCallback(self._profile1, finished)
        d.addErrback(self.err)

    def _profile1(self, response, finished):
        if response.code == 500:
            error('Got a 500 SERVER ERROR trying to get the profile!')

        elif response.code == 200:
            debug('Got 200 OK response to profile request')
            d = defer.Deferred()
            d.addCallback(self._profile2, finished)
            d.addCallback(self._inventory0)
            jp = b07.utils.JsonProtocol(d)
            response.deliverBody(jp)

        else:
            error(
                'Don\'t know what to do with {} code in response to profile request!'
                .format(response.code))

    def _profile2(self, result, finished):
        with open(
                os.path.expanduser("~/{}_profile.json".format(
                    self.player_nickname)), "w") as file:
            json.dump(result, file, indent=1)
        self.profile_done = True
Example #52
0
class txBoxAPI(BoxAPIWrapper):
	'Box API client.'

	#: Options to twisted.web.client.HTTPConnectionPool
	request_pool_options = dict(
		persistent = True,
		maxPersistentPerHost = 10,
		cachedConnectionTimeout = 600,
		retryAutomatically = True )

	#: These are timeouts between individual read/write ops
	#: Missing keys will have default values (from HTTPTimeout.state_timeouts)
	request_io_timeouts = dict( req_headers=60,
		req_body=20, res_headers=20, res_body=20, res_end=10 )

	#: Path string or list of strings
	ca_certs_files = b'/etc/ssl/certs/ca-certificates.crt'

	#: Dump HTTP request data in debug log (might contain all sorts of auth tokens!)
	debug_requests = False


	def __init__(self, *argz, **kwz):
		super(txBoxAPI, self).__init__(*argz, **kwz)
		pool = self.request_pool = QuietHTTPConnectionPool( reactor,
				debug_requests=self.debug_requests, **self.request_pool_options )
		self.request_agent = ContentDecoderAgent(RedirectAgent(Agent(
			reactor, TLSContextFactory(self.ca_certs_files), pool=pool )), [('gzip', GzipDecoder)])


	@defer.inlineCallbacks
	def request( self, url, method='get',
			data=None, encode=None, files=None,
			raw=False, headers=dict(), raise_for=dict() ):
		if self.debug_requests:
			url_debug = _dump_trunc(url)
			log.debug('HTTP request: {} {} (h: {}, data: {}, files: {}), raw: {}'.format(
				method, url_debug, headers, _dump_trunc(data), _dump_trunc(files), raw ))

		timeout = HTTPTimeout(**self.request_io_timeouts)

		method, body = method.lower(), None
		headers = dict() if not headers else headers.copy()
		headers.setdefault('User-Agent', 'txBox')

		if data is not None:
			if encode is None:
				encode = 'json' if method != 'post' else 'form'
			if encode == 'form':
				headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
				body = FileBodyProducer(io.BytesIO(urlencode(data)), timer=timeout)
			elif encode == 'json':
				headers.setdefault('Content-Type', 'application/json')
				body = FileBodyProducer(io.BytesIO(json.dumps(data)), timer=timeout)
			else: raise ValueError('Unknown encoding type: {}'.format(encode))

		if files is not None:
			assert not (data or encode),\
				'"files" imply multipart encoding and no other data'
			boundary = os.urandom(16).encode('hex')
			headers.setdefault( 'Content-Type',
				'multipart/form-data; boundary={}'.format(boundary) )
			body = MultipartDataSender(files, boundary, timer=timeout)
			yield body.calculate_length()

		if isinstance(url, unicode): url = url.encode('utf-8')
		if isinstance(method, unicode): method = method.encode('ascii')

		res_deferred = first_result( timeout,
			self.request_agent.request( method.upper(), url,
				Headers(dict((k,[v]) for k,v in (headers or dict()).viewitems())), body ) )
		code = res_body = None
		try:
			res = yield res_deferred
			code = res.code
			if code == http.NO_CONTENT: defer.returnValue(None)

			res_body = defer.Deferred()
			res.deliverBody(DataReceiver(res_body, timer=timeout))
			res_body = yield first_result(timeout, res_body)

			if code not in [http.OK, http.CREATED]:
				if self.debug_requests:
					log.debug('HTTP error response body: {!r}'.format(res_body))
				raise ProtocolError(code, res.phrase, res_body)

			if self.debug_requests:
				log.debug( 'HTTP request done ({} {}): {} {} {}, body_len: {}'\
					.format(method, url_debug, code, res.phrase, res.version, len(res_body)) )
			defer.returnValue(json.loads(res_body) if not raw else res_body)

		except ( timeout.ActivityTimeout, TimeoutError,
				ResponseFailed, RequestNotSent, RequestTransmissionFailed ) as err:
			if isinstance(err, timeout.ActivityTimeout):
				if not res_deferred.called: res_deferred.cancel()
				if res_body and not res_body.called: res_body.cancel()
			if self.debug_requests:
				log.debug(
					'HTTP transport (underlying protocol) error ({} {}): {}'\
					.format(method, url_debug, err.message or repr(err.args)) )
			raise UnderlyingProtocolError(err)

		except ProtocolError as err:
			if self.debug_requests:
				log.debug(
					'HTTP request handling error ({} {}, code: {}): {}'\
					.format(method, url_debug, code, err.message) )
			if code not in raise_for: raise
			raise raise_for[code](code, err.message)

		except RequestGenerationFailed as err:
			err[0][0].raiseException()

		finally: timeout.state_finished()


	@defer.inlineCallbacks
	def __call__( self, url, query=dict(),
			query_filter=True, auth_header=True,
			auto_refresh_token=True, upload=False, **request_kwz ):
		'''Make an arbitrary call to LiveConnect API.
			Shouldn't be used directly under most circumstances.'''
		if query_filter:
			query = dict( (k,v) for k,v in
				query.viewitems() if v is not None )
		if auth_header:
			request_kwz.setdefault('headers', dict())\
				['Authorization'] = 'Bearer {}'.format(self.auth_access_token)
		kwz = request_kwz.copy()
		kwz.setdefault('raise_for', dict())[401] = AuthenticationError
		api_url = ft.partial( self._api_url, url, query,
			pass_access_token=not auth_header, upload=upload )
		try: res = yield self.request(api_url(), **kwz)
		except AuthenticationError:
			if not auto_refresh_token: raise
			yield self.auth_get_token()
			if auth_header: # update auth header with a new token
				request_kwz['headers']['Authorization']\
					= 'Bearer {}'.format(self.auth_access_token)
			# Existing connections seem to hang occasionally after token updates,
			#  though it looks like a twisted issue, since they hang forever (no timeouts in place)
			self.request_pool.closeCachedConnections()
			res = yield self.request(api_url(), **request_kwz)
		defer.returnValue(res)


	@defer.inlineCallbacks
	def auth_get_token(self, check_state=True):
		'Refresh or acquire access_token.'
		res = self.auth_access_data_raw = yield self._auth_token_request()
		defer.returnValue(self._auth_token_process(res, check_state=check_state))
Example #53
0
    def request(
        self,
        method,
        url,
        *,
        params=None,
        headers=None,
        data=None,
        files=None,
        json=_NOTHING,
        auth=None,
        cookies=None,
        allow_redirects=True,
        browser_like_redirects=False,
        unbuffered=False,
        reactor=None,
        timeout=None,
        _stacklevel=2,
    ):
        """
        See :func:`treq.request()`.
        """
        method = method.encode('ascii').upper()

        if isinstance(url, DecodedURL):
            parsed_url = url.encoded_url
        elif isinstance(url, EncodedURL):
            parsed_url = url
        elif isinstance(url, str):
            # We use hyperlink in lazy mode so that users can pass arbitrary
            # bytes in the path and querystring.
            parsed_url = EncodedURL.from_text(url)
        else:
            parsed_url = EncodedURL.from_text(url.decode('ascii'))

        # Join parameters provided in the URL
        # and the ones passed as argument.
        if params:
            parsed_url = parsed_url.replace(
                query=parsed_url.query + tuple(_coerced_query_params(params)))

        url = parsed_url.to_uri().to_text().encode('ascii')

        headers = self._request_headers(headers, _stacklevel + 1)

        bodyProducer, contentType = self._request_body(data,
                                                       files,
                                                       json,
                                                       stacklevel=_stacklevel +
                                                       1)
        if contentType is not None:
            headers.setRawHeaders(b'Content-Type', [contentType])

        if not isinstance(cookies, CookieJar):
            cookies = cookiejar_from_dict(cookies)

        cookies = merge_cookies(self._cookiejar, cookies)
        wrapped_agent = CookieAgent(self._agent, cookies)

        if allow_redirects:
            if browser_like_redirects:
                wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
            else:
                wrapped_agent = RedirectAgent(wrapped_agent)

        wrapped_agent = ContentDecoderAgent(wrapped_agent,
                                            [(b'gzip', GzipDecoder)])

        if auth:
            wrapped_agent = add_auth(wrapped_agent, auth)

        d = wrapped_agent.request(method,
                                  url,
                                  headers=headers,
                                  bodyProducer=bodyProducer)

        if reactor is None:
            from twisted.internet import reactor
        if timeout:
            delayedCall = reactor.callLater(timeout, d.cancel)

            def gotResult(result):
                if delayedCall.active():
                    delayedCall.cancel()
                return result

            d.addBoth(gotResult)

        if not unbuffered:
            d.addCallback(_BufferedResponse)

        return d.addCallback(_Response, cookies)
Example #54
0
    def test_EncodingJSONRPCServer(self):

        DATA = {'foo': 'bar'}
        REQUEST = '{"jsonrpc": "2.0", "method": "test", "params": [], "id": 1}'
        RESPONSE = '{"jsonrpc": "2.0", "id": 1, "result": ' + json.dumps(DATA) + '}'

        class RPCServer(JSONRPCServer):
            def jsonrpc_test(self):
                return defer.succeed(DATA)

        class ReceiverProtocol(Protocol):
            def __init__(self, finished):
                self.finished = finished
                self.body = []

            def dataReceived(self, bytes):
                self.body.append(bytes)

            def connectionLost(self, reason):
                self.finished.callback(b''.join(self.body))

        @implementer(IBodyProducer)
        class StringProducer(object):

            def __init__(self, body):
                self.body = body
                self.length = len(body)

            def startProducing(self, consumer):
                consumer.write(self.body.encode())
                return defer.succeed(None)

            def pauseProducing(self):
                pass

            def stopProducing(self):
                pass

        server = RPCServer()
        resource = EncodingJSONRPCServer(server)
        site = Site(resource)

        port = reactor.listenTCP(8888, site, interface='127.0.0.1')

        agent = ContentDecoderAgent(Agent(reactor), [(b'gzip', GzipDecoder)])

        response = yield agent.request(b'POST', b'http://127.0.0.1:8888',
                                       Headers({'Accept-Encoding': ['gzip']}),
                                       StringProducer(REQUEST))

        self.assertTrue(isinstance(response, GzipDecoder))

        finished = defer.Deferred()

        response.deliverBody(ReceiverProtocol(finished))

        data = yield finished

        self.assert_json(data, RESPONSE)

        port.stopListening()