def startRequest(self, request, url, feed_config = None, selector_defer=None, sanitize=False): downloader = self.downloadercls(self.feed, self.debug, self.snapshot_dir, self.stat_tool, self.memon, request=request, url=url, feed_config=feed_config, selector_defer=selector_defer, sanitize=sanitize, max_size=self.max_size) sresponse = self.tryLocalPage(url) if sresponse: if selector_defer: reactor.callLater(0, selector_defer.callback, sresponse) else: downloader.writeResponse(request, sresponse, feed_config) else: agent = BrowserLikeRedirectAgent( Agent(reactor, contextFactory=ScrapyClientContextFactory(), # skip certificate verification connectTimeout=10), #pool=pool), redirectLimit=5 ) d = agent.request( 'GET', url, twisted_headers({ 'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'], 'Accept-Encoding': ['gzip, deflate, sdch'], 'User-Agent': [self.user_agent] }), None ) print('Request <GET %s> started' % (url,)) d.addCallback(downloader.downloadStarted) d.addErrback(downloader.downloadError)
def push(message): sap_mms_endpoint = SAP_CONFIG.mms_endpoint sap_device_id = SAP_CONFIG.device_id sap_message_type_id = SAP_CONFIG.message_type_id sap_oauth_token = SAP_CONFIG.oauth_token sap_data_endpoint = "{0}/{1}".format(sap_mms_endpoint, sap_device_id) print("SAP MMS data endpoint: {0}".format(sap_data_endpoint)) agent = Agent(reactor) client = BrowserLikeRedirectAgent(agent) headers = Headers({ "Authorization": ["Bearer {0}".format(sap_oauth_token)], "Content-Type": ["application/json;charset=utf-8"] }) body = { "mode": "sync", "messageType": sap_message_type_id, "messages": [{ "sensor": "IntelHowToSample", "value": serialize_json(message), "timestamp": int(time()) }] } print("SAP MMS payload: {0}".format(serialize_json(body))) def handle_response(response): if response.code == 200: print("SAP MMS publish succeeded.") else: print("SAP Error response: {0}".format(response.code)) client.request("POST", sap_data_endpoint, headers, JsonProducer(body)) \ .addCallback(lambda x: print("Published to SAP MMS service.")).addErrback(lambda x: print("SAP publish error:", x))
def url_get_data_async(url, callback=None, data=None, headers=None, timeout=60): def handle_failure(failure): failure.printTraceback() callback(None) def handle_result(data): callback(data) assert data is None, "sorry data is currently not supported" if headers is not None: headers = {k:[v] for k,v in headers.items()} agent = BrowserLikeRedirectAgent(Agent(reactor, connectTimeout=timeout)) d = agent.request('GET', url, Headers(headers)) d.addCallback(readBody) if callback is not None: d.addCallbacks(handle_result, handle_failure) return d
def _request(self, method, url, headers, success, unconditional=None): """Sends an HTTP request. Args: method (str): 'SUBSCRIBE' or 'UNSUBSCRIBE'. url (str): The full endpoint to which the request is being sent. headers (dict): A dict of headers, each key and each value being of type `str`. success (function): A function to be called if the request succeeds. The function will be called with a dict of response headers as its only parameter. unconditional (function): An optional function to be called after the request is complete, regardless of its success. Takes no parameters. """ agent = BrowserLikeRedirectAgent(Agent(reactor)) if headers: for k in headers.keys(): header = headers[k] del headers[k] if isinstance(header, (list, )): header = header[0] if not isinstance(header, (bytes, bytearray)): header = header.encode("latin-1") k = k.encode("latin-1") headers[k] = [header] args = (method.encode("latin-1"), url.encode("latin-1"), Headers(headers)) d = agent.request(*args) # pylint: disable=invalid-name def on_success(response): # pylint: disable=missing-docstring response_headers = {} for header in response.headers.getAllRawHeaders(): decoded_key = header[0].decode("utf8").lower() decoded_header = header[1][0].decode("utf8") response_headers[decoded_key] = decoded_header success(response_headers) return self d.addCallback(on_success) if unconditional: d.addBoth(unconditional) return d
def request(self, method, url, **kwargs): method = method.upper() # Join parameters provided in the URL # and the ones passed as argument. params = kwargs.get("params") if params: url = _combine_query_params(url, params) # Convert headers dictionary to # twisted raw headers format. headers = kwargs.get("headers") if headers: if isinstance(headers, dict): h = Headers({}) for k, v in headers.iteritems(): if isinstance(v, str): h.addRawHeader(k, v) else: h.setRawHeaders(k, v) headers = h else: headers = Headers({}) # Here we choose a right producer # based on the parameters passed in. bodyProducer = None data = kwargs.get("data") files = kwargs.get("files") if files: # If the files keyword is present we will issue a # multipart/form-data request as it suits better for cases # with files and/or large objects. files = list(_convert_files(files)) boundary = uuid.uuid4() headers.setRawHeaders("content-type", ["multipart/form-data; boundary=%s" % (boundary,)]) if data: data = _convert_params(data) else: data = [] bodyProducer = multipart.MultiPartProducer(data + files, boundary=boundary) elif data: # Otherwise stick to x-www-form-urlencoded format # as it's generally faster for smaller requests. if isinstance(data, (dict, list, tuple)): headers.setRawHeaders("content-type", ["application/x-www-form-urlencoded"]) data = urlencode(data, doseq=True) bodyProducer = IBodyProducer(data) wrapped_agent = self._agent if kwargs.get("allow_redirects", True): wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent) wrapped_agent = ContentDecoderAgent(wrapped_agent, [("gzip", GzipDecoder)]) auth = kwargs.get("auth") if auth: wrapped_agent = add_auth(wrapped_agent, auth) d = wrapped_agent.request(method, url, headers=headers, bodyProducer=bodyProducer) timeout = kwargs.get("timeout") if timeout: delayedCall = default_reactor(kwargs.get("reactor")).callLater(timeout, d.cancel) def gotResult(result): if delayedCall.active(): delayedCall.cancel() return result d.addBoth(gotResult) if not kwargs.get("unbuffered", False): d.addCallback(_BufferedResponse) return d.addCallback(_Response)
class TwitchStreamGrid(Screen): TMP_PREVIEW_FILE_PATH = "/tmp/twitch_channel_preview.jpg" SKIN_COMPONENT_KEY = "TwitchStreamGrid" SKIN_COMPONENT_HEADER_HEIGHT = "headerHeight" SKIN_COMPONENT_FOOTER_HEIGHT = "footerHeight" SKIN_COMPONENT_ITEM_PADDING = "itemPadding" def __init__(self, session, windowTitle=_("TwitchTV")): Screen.__init__(self, session, windowTitle=windowTitle) self.skinName = "TwitchStreamGrid" self["actions"] = ActionMap(["OkCancelActions", "ColorActions"], { "ok": self._onOk, "cancel": self.close, "red": self._onRed, "green": self._onGreen, "yellow" : self._onYellow, "blue": self._onBlue, }, -1) self["key_red"] = StaticText() self["key_green"] = StaticText() self["key_blue"] = StaticText() self["key_yellow"] = StaticText() self._setupButtons() sizes = componentSizes[TwitchStreamGrid.SKIN_COMPONENT_KEY] self._itemWidth = sizes.get(ComponentSizes.ITEM_WIDTH, 280) self._itemHeight = sizes.get(ComponentSizes.ITEM_HEIGHT, 162) self._bannerHeight = sizes.get(TwitchStreamGrid.SKIN_COMPONENT_HEADER_HEIGHT, 30) self._footerHeight = sizes.get(TwitchStreamGrid.SKIN_COMPONENT_FOOTER_HEIGHT, 60) self._itemPadding = sizes.get(TwitchStreamGrid.SKIN_COMPONENT_ITEM_PADDING, 5) #one-off calculations pad = self._itemPadding * 2 self._contentWidth = self._itemWidth - pad self._contentHeight = self._itemHeight - pad self._footerOffset = self._itemHeight - self._itemPadding - self._footerHeight self._items = [] self._list = MenuList(self._items, mode=eListbox.layoutGrid, content=eListboxPythonMultiContent, itemWidth=self._itemWidth, itemHeight=self._itemHeight) self["list"] = self._list tlf = TemplatedListFonts() self._list.l.setFont(0, gFont(tlf.face(tlf.MEDIUM), tlf.size(tlf.MEDIUM))) self._list.l.setFont(1, gFont(tlf.face(tlf.SMALLER), tlf.size(tlf.SMALL))) self._list.l.setBuildFunc(self._buildFunc, True) self.twitch = Twitch() self.twitchMiddleware = TwitchMiddleware.instance self._picload = ePicLoad() self._picload.setPara((self._itemWidth, self._itemHeight, self._itemWidth, self._itemHeight, False, 0, '#000000')) self._picload_conn = self._picload.PictureData.connect(self._onDefaultPixmapReady) agent = Agent(reactor, contextFactory=TLSSNIContextFactory(), pool=HTTPConnectionPool(reactor)) self._agent = BrowserLikeRedirectAgent(agent) self._cachingDeferred = None self._loadDefaultPixmap() self._pixmapCache = {} self._currentEntry = 0 self._endEntry = 0 self.onLayoutFinish.append(self._onLayoutFinish) self.onClose.append(self.__onClose) def __onClose(self): if self._cachingDeferred: Log.w("Cancelling pending image download...") self._cachingDeferred.cancel() self._picload_conn = None self._picload = None def _setupButtons(self): pass def _onLayoutFinish(self): self.validateCache(True) def reload(self): self._items = [ ("loading",) ] self._list.setList(self._items) self._loadContent() def _onRed(self): pass def _onGreen(self): pass def _onYellow(self): pass def _onBlue(self): pass def _loadContent(self): raise NotImplementedError def _getCurrent(self): return self._list.getCurrent()[0] current = property(_getCurrent) def _buildFunc(self, stream, selected): raise NotImplementedError def _onOk(self): raise NotImplementedError def goDetails(self): stream = self.current if stream is None or not isinstance(stream, TwitchVideoBase): return self.session.open(TwitchChannelDetails, stream=stream) def validateCache(self, clear=False): if not self._list.instance: return if clear: self._pixmapCache = {} self._currentEntry = -1 self._endEntry = len(self._items) - 1 self._nextForCache() def _nextForCache(self): self._currentEntry += 1 if self._currentEntry > self._endEntry: return if self._currentEntry < len(self._items): item = self._items[self._currentEntry][0] Log.d(item.preview) self._loadPixmapForCache(self._currentEntry, item.preview) def _onDownloadPageResponse(self, response, index, url): self._cachingDeferred = readBody(response) self._cachingDeferred.addCallbacks(self._onDownloadPageBody, self._errorPixmapForCache, callbackArgs=[index, url]) def _onDownloadPageBody(self, body, index, url): with open(self.TMP_PREVIEW_FILE_PATH, 'w') as f: f.write(body) self._gotPixmapForCache(index, url, None) def _loadPixmapForCache(self, index, url): self._cachingDeferred = self._agent.request('GET', url) self._cachingDeferred.addCallbacks(self._onDownloadPageResponse, self._errorPixmapForCache, callbackArgs=[index,url]) def _gotPixmapForCache(self, index, url, data): self._cachingDeferred = None callback = boundFunction(self._decodedPixmapForCache, index, url) self._picload_conn = self._picload.PictureData.connect(callback) self._picload.startDecode(self.TMP_PREVIEW_FILE_PATH) def _decodedPixmapForCache(self, index, url, picInfo=None): Log.d(url) self._pixmapCache[url] = self._picload.getData() self._list.setList(self._items[:]) self._nextForCache() def _errorPixmapForCache(self, *args): Log.w(args) self._cachingDeferred = None if self._picload: self._nextForCache() def _onAllStreams(self, streams): self._items = [] for stream in streams: self._items.append((stream,)) self._list.setList(self._items) if self._list.instance: self.validateCache(True) def addToFavs(self): stream = self.current if stream is None or not isinstance(stream, TwitchVideoBase): return self.twitchMiddleware.addToFavorites(stream.channel) def _loadDefaultPixmap(self, *args): self._picload.startDecode(resolveFilename(SCOPE_PLUGINS, "Extensions/TwitchTV/twitch.svg")) def _errorDefaultPixmap(self, *args): Log.w(args) def _onDefaultPixmapReady(self, picInfo=None): self._defaultPixmap = self._picload.getData() self.reload()
class RadioBrowserClient(object): URL_BASE = "https://de1.api.radio-browser.info/json" URL_COUNTRIES = "/countries" URL_STATIONS = "/stations" URL_STATIONS_BY_NAME = "/stations/byname" URL_STATIONS_BY_COUNTRY = "/stations/bycountry" def __init__(self): self._agent = BrowserLikeRedirectAgent(Agent(reactor)) @property def stations(self): return self._byCountry def request(self, uri, callback, options={}): self._agent.request( 'POST', uri.encode("utf-8"), Headers({'Content-Type': ['application/json']}), StringBodyProducer(json.dumps(options)), ).addCallbacks(self._onResponse, errback=self._onReloadError, callbackArgs=(callback, )) def _parseJson(self, jsonString, callback): def doParse(data): try: data = json.loads(data) return data except Exception as e: Log.w(e) Log.w(jsonString) return {} threads.deferToThread(doParse, jsonString).addCallback(callback) def countries(self, callback): def onReloadFinished(data): def _onJson(countries): callback([Country(data) for data in countries]) self._parseJson(data, _onJson) self.request("{}{}".format(self.URL_BASE, self.URL_COUNTRIES), onReloadFinished, options={ "order": "name", "hidebroken": True }) def stations(self, country, callback, offset=0, limit=2000): def onReloadFinished(data): def _onJson(jsonData): lst = OrderedDict() for entry in jsonData: name = entry["name"] station = lst.get(name, Station(entry)) station.extend(entry) lst[name] = station callback(lst) self._parseJson(data, _onJson) options = { "order": "clickcount", "reverse": True, "offset": offset, "limit": limit, "hidebroken": True } self.request("{}{}/{}".format(self.URL_BASE, self.URL_STATIONS_BY_COUNTRY, quote(country)), onReloadFinished, options=options) def _onResponse(self, response, callback): readBody(response).addCallbacks(callback, errback=self._onReloadError) def _onReloadError(self, failure, *args): Log.w("RELOAD ERROR! {}".format(failure)) if failure.check( RequestGenerationFailed, RequestTransmissionFailed, ResponseFailed, ): failure.value.reasons[0].printTraceback() else: failure.printTraceback()