def startRequest(self, request, url, feed_config = None, selector_defer=None, sanitize=False):
downloader = self.downloadercls(self.feed, self.debug, self.snapshot_dir, self.stat_tool, self.memon,
request=request, url=url, feed_config=feed_config,
selector_defer=selector_defer, sanitize=sanitize, max_size=self.max_size)
sresponse = self.tryLocalPage(url)
if sresponse:
if selector_defer:
reactor.callLater(0, selector_defer.callback, sresponse)
else:
downloader.writeResponse(request, sresponse, feed_config)
else:
agent = BrowserLikeRedirectAgent(
Agent(reactor,
contextFactory=ScrapyClientContextFactory(), # skip certificate verification
connectTimeout=10),
#pool=pool),
redirectLimit=5
)
d = agent.request(
'GET',
url,
twisted_headers({
'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'],
'Accept-Encoding': ['gzip, deflate, sdch'],
'User-Agent': [self.user_agent]
}),
None
)
print('Request <GET %s> started' % (url,))
d.addCallback(downloader.downloadStarted)
d.addErrback(downloader.downloadError)
def push(message):
sap_mms_endpoint = SAP_CONFIG.mms_endpoint
sap_device_id = SAP_CONFIG.device_id
sap_message_type_id = SAP_CONFIG.message_type_id
sap_oauth_token = SAP_CONFIG.oauth_token
sap_data_endpoint = "{0}/{1}".format(sap_mms_endpoint, sap_device_id)
print("SAP MMS data endpoint: {0}".format(sap_data_endpoint))
agent = Agent(reactor)
client = BrowserLikeRedirectAgent(agent)
headers = Headers({
"Authorization": ["Bearer {0}".format(sap_oauth_token)],
"Content-Type": ["application/json;charset=utf-8"]
})
body = {
"mode":
"sync",
"messageType":
sap_message_type_id,
"messages": [{
"sensor": "IntelHowToSample",
"value": serialize_json(message),
"timestamp": int(time())
}]
}
print("SAP MMS payload: {0}".format(serialize_json(body)))
def handle_response(response):
if response.code == 200:
print("SAP MMS publish succeeded.")
else:
print("SAP Error response: {0}".format(response.code))
client.request("POST", sap_data_endpoint, headers, JsonProducer(body)) \
.addCallback(lambda x: print("Published to SAP MMS service.")).addErrback(lambda x: print("SAP publish error:", x))
def url_get_data_async(url, callback=None, data=None, headers=None, timeout=60):
def handle_failure(failure):
failure.printTraceback()
callback(None)
def handle_result(data):
callback(data)
assert data is None, "sorry data is currently not supported"
if headers is not None:
headers = {k:[v] for k,v in headers.items()}
agent = BrowserLikeRedirectAgent(Agent(reactor, connectTimeout=timeout))
d = agent.request('GET', url, Headers(headers))
d.addCallback(readBody)
if callback is not None:
d.addCallbacks(handle_result, handle_failure)
return d
def url_get_data_async(url, callback=None, data=None, headers=None, timeout=60):
def handle_failure(failure):
failure.printTraceback()
callback(None)
def handle_result(data):
callback(data)
assert data is None, "sorry data is currently not supported"
if headers is not None:
headers = {k:[v] for k,v in headers.items()}
agent = BrowserLikeRedirectAgent(Agent(reactor, connectTimeout=timeout))
d = agent.request('GET', url, Headers(headers))
d.addCallback(readBody)
if callback is not None:
d.addCallbacks(handle_result, handle_failure)
return d
def _request(self, method, url, headers, success, unconditional=None):
"""Sends an HTTP request.
Args:
method (str): 'SUBSCRIBE' or 'UNSUBSCRIBE'.
url (str): The full endpoint to which the request is being sent.
headers (dict): A dict of headers, each key and each value being
of type `str`.
success (function): A function to be called if the
request succeeds. The function will be called with a dict
of response headers as its only parameter.
unconditional (function): An optional function to be called after
the request is complete, regardless of its success. Takes
no parameters.
"""
agent = BrowserLikeRedirectAgent(Agent(reactor))
if headers:
for k in headers.keys():
header = headers[k]
del headers[k]
if isinstance(header, (list, )):
header = header[0]
if not isinstance(header, (bytes, bytearray)):
header = header.encode("latin-1")
k = k.encode("latin-1")
headers[k] = [header]
args = (method.encode("latin-1"), url.encode("latin-1"),
Headers(headers))
d = agent.request(*args) # pylint: disable=invalid-name
def on_success(response): # pylint: disable=missing-docstring
response_headers = {}
for header in response.headers.getAllRawHeaders():
decoded_key = header[0].decode("utf8").lower()
decoded_header = header[1][0].decode("utf8")
response_headers[decoded_key] = decoded_header
success(response_headers)
return self
d.addCallback(on_success)
if unconditional:
d.addBoth(unconditional)
return d
def request(self, method, url, **kwargs):
method = method.upper()
# Join parameters provided in the URL
# and the ones passed as argument.
params = kwargs.get("params")
if params:
url = _combine_query_params(url, params)
# Convert headers dictionary to
# twisted raw headers format.
headers = kwargs.get("headers")
if headers:
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.iteritems():
if isinstance(v, str):
h.addRawHeader(k, v)
else:
h.setRawHeaders(k, v)
headers = h
else:
headers = Headers({})
# Here we choose a right producer
# based on the parameters passed in.
bodyProducer = None
data = kwargs.get("data")
files = kwargs.get("files")
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
# with files and/or large objects.
files = list(_convert_files(files))
boundary = uuid.uuid4()
headers.setRawHeaders("content-type", ["multipart/form-data; boundary=%s" % (boundary,)])
if data:
data = _convert_params(data)
else:
data = []
bodyProducer = multipart.MultiPartProducer(data + files, boundary=boundary)
elif data:
# Otherwise stick to x-www-form-urlencoded format
# as it's generally faster for smaller requests.
if isinstance(data, (dict, list, tuple)):
headers.setRawHeaders("content-type", ["application/x-www-form-urlencoded"])
data = urlencode(data, doseq=True)
bodyProducer = IBodyProducer(data)
wrapped_agent = self._agent
if kwargs.get("allow_redirects", True):
wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
wrapped_agent = ContentDecoderAgent(wrapped_agent, [("gzip", GzipDecoder)])
auth = kwargs.get("auth")
if auth:
wrapped_agent = add_auth(wrapped_agent, auth)
d = wrapped_agent.request(method, url, headers=headers, bodyProducer=bodyProducer)
timeout = kwargs.get("timeout")
if timeout:
delayedCall = default_reactor(kwargs.get("reactor")).callLater(timeout, d.cancel)
def gotResult(result):
if delayedCall.active():
delayedCall.cancel()
return result
d.addBoth(gotResult)
if not kwargs.get("unbuffered", False):
d.addCallback(_BufferedResponse)
return d.addCallback(_Response)
class TwitchStreamGrid(Screen):
TMP_PREVIEW_FILE_PATH = "/tmp/twitch_channel_preview.jpg"
SKIN_COMPONENT_KEY = "TwitchStreamGrid"
SKIN_COMPONENT_HEADER_HEIGHT = "headerHeight"
SKIN_COMPONENT_FOOTER_HEIGHT = "footerHeight"
SKIN_COMPONENT_ITEM_PADDING = "itemPadding"
def __init__(self, session, windowTitle=_("TwitchTV")):
Screen.__init__(self, session, windowTitle=windowTitle)
self.skinName = "TwitchStreamGrid"
self["actions"] = ActionMap(["OkCancelActions", "ColorActions"],
{
"ok": self._onOk,
"cancel": self.close,
"red": self._onRed,
"green": self._onGreen,
"yellow" : self._onYellow,
"blue": self._onBlue,
}, -1)
self["key_red"] = StaticText()
self["key_green"] = StaticText()
self["key_blue"] = StaticText()
self["key_yellow"] = StaticText()
self._setupButtons()
sizes = componentSizes[TwitchStreamGrid.SKIN_COMPONENT_KEY]
self._itemWidth = sizes.get(ComponentSizes.ITEM_WIDTH, 280)
self._itemHeight = sizes.get(ComponentSizes.ITEM_HEIGHT, 162)
self._bannerHeight = sizes.get(TwitchStreamGrid.SKIN_COMPONENT_HEADER_HEIGHT, 30)
self._footerHeight = sizes.get(TwitchStreamGrid.SKIN_COMPONENT_FOOTER_HEIGHT, 60)
self._itemPadding = sizes.get(TwitchStreamGrid.SKIN_COMPONENT_ITEM_PADDING, 5)
#one-off calculations
pad = self._itemPadding * 2
self._contentWidth = self._itemWidth - pad
self._contentHeight = self._itemHeight - pad
self._footerOffset = self._itemHeight - self._itemPadding - self._footerHeight
self._items = []
self._list = MenuList(self._items, mode=eListbox.layoutGrid, content=eListboxPythonMultiContent, itemWidth=self._itemWidth, itemHeight=self._itemHeight)
self["list"] = self._list
tlf = TemplatedListFonts()
self._list.l.setFont(0, gFont(tlf.face(tlf.MEDIUM), tlf.size(tlf.MEDIUM)))
self._list.l.setFont(1, gFont(tlf.face(tlf.SMALLER), tlf.size(tlf.SMALL)))
self._list.l.setBuildFunc(self._buildFunc, True)
self.twitch = Twitch()
self.twitchMiddleware = TwitchMiddleware.instance
self._picload = ePicLoad()
self._picload.setPara((self._itemWidth, self._itemHeight, self._itemWidth, self._itemHeight, False, 0, '#000000'))
self._picload_conn = self._picload.PictureData.connect(self._onDefaultPixmapReady)
agent = Agent(reactor, contextFactory=TLSSNIContextFactory(), pool=HTTPConnectionPool(reactor))
self._agent = BrowserLikeRedirectAgent(agent)
self._cachingDeferred = None
self._loadDefaultPixmap()
self._pixmapCache = {}
self._currentEntry = 0
self._endEntry = 0
self.onLayoutFinish.append(self._onLayoutFinish)
self.onClose.append(self.__onClose)
def __onClose(self):
if self._cachingDeferred:
Log.w("Cancelling pending image download...")
self._cachingDeferred.cancel()
self._picload_conn = None
self._picload = None
def _setupButtons(self):
pass
def _onLayoutFinish(self):
self.validateCache(True)
def reload(self):
self._items = [ ("loading",) ]
self._list.setList(self._items)
self._loadContent()
def _onRed(self):
pass
def _onGreen(self):
pass
def _onYellow(self):
pass
def _onBlue(self):
pass
def _loadContent(self):
raise NotImplementedError
def _getCurrent(self):
return self._list.getCurrent()[0]
current = property(_getCurrent)
def _buildFunc(self, stream, selected):
raise NotImplementedError
def _onOk(self):
raise NotImplementedError
def goDetails(self):
stream = self.current
if stream is None or not isinstance(stream, TwitchVideoBase):
return
self.session.open(TwitchChannelDetails, stream=stream)
def validateCache(self, clear=False):
if not self._list.instance:
return
if clear:
self._pixmapCache = {}
self._currentEntry = -1
self._endEntry = len(self._items) - 1
self._nextForCache()
def _nextForCache(self):
self._currentEntry += 1
if self._currentEntry > self._endEntry:
return
if self._currentEntry < len(self._items):
item = self._items[self._currentEntry][0]
Log.d(item.preview)
self._loadPixmapForCache(self._currentEntry, item.preview)
def _onDownloadPageResponse(self, response, index, url):
self._cachingDeferred = readBody(response)
self._cachingDeferred.addCallbacks(self._onDownloadPageBody, self._errorPixmapForCache, callbackArgs=[index, url])
def _onDownloadPageBody(self, body, index, url):
with open(self.TMP_PREVIEW_FILE_PATH, 'w') as f:
f.write(body)
self._gotPixmapForCache(index, url, None)
def _loadPixmapForCache(self, index, url):
self._cachingDeferred = self._agent.request('GET', url)
self._cachingDeferred.addCallbacks(self._onDownloadPageResponse, self._errorPixmapForCache, callbackArgs=[index,url])
def _gotPixmapForCache(self, index, url, data):
self._cachingDeferred = None
callback = boundFunction(self._decodedPixmapForCache, index, url)
self._picload_conn = self._picload.PictureData.connect(callback)
self._picload.startDecode(self.TMP_PREVIEW_FILE_PATH)
def _decodedPixmapForCache(self, index, url, picInfo=None):
Log.d(url)
self._pixmapCache[url] = self._picload.getData()
self._list.setList(self._items[:])
self._nextForCache()
def _errorPixmapForCache(self, *args):
Log.w(args)
self._cachingDeferred = None
if self._picload:
self._nextForCache()
def _onAllStreams(self, streams):
self._items = []
for stream in streams:
self._items.append((stream,))
self._list.setList(self._items)
if self._list.instance:
self.validateCache(True)
def addToFavs(self):
stream = self.current
if stream is None or not isinstance(stream, TwitchVideoBase):
return
self.twitchMiddleware.addToFavorites(stream.channel)
def _loadDefaultPixmap(self, *args):
self._picload.startDecode(resolveFilename(SCOPE_PLUGINS, "Extensions/TwitchTV/twitch.svg"))
def _errorDefaultPixmap(self, *args):
Log.w(args)
def _onDefaultPixmapReady(self, picInfo=None):
self._defaultPixmap = self._picload.getData()
self.reload()
class RadioBrowserClient(object):
URL_BASE = "https://de1.api.radio-browser.info/json"
URL_COUNTRIES = "/countries"
URL_STATIONS = "/stations"
URL_STATIONS_BY_NAME = "/stations/byname"
URL_STATIONS_BY_COUNTRY = "/stations/bycountry"
def __init__(self):
self._agent = BrowserLikeRedirectAgent(Agent(reactor))
@property
def stations(self):
return self._byCountry
def request(self, uri, callback, options={}):
self._agent.request(
'POST',
uri.encode("utf-8"),
Headers({'Content-Type': ['application/json']}),
StringBodyProducer(json.dumps(options)),
).addCallbacks(self._onResponse,
errback=self._onReloadError,
callbackArgs=(callback, ))
def _parseJson(self, jsonString, callback):
def doParse(data):
try:
data = json.loads(data)
return data
except Exception as e:
Log.w(e)
Log.w(jsonString)
return {}
threads.deferToThread(doParse, jsonString).addCallback(callback)
def countries(self, callback):
def onReloadFinished(data):
def _onJson(countries):
callback([Country(data) for data in countries])
self._parseJson(data, _onJson)
self.request("{}{}".format(self.URL_BASE, self.URL_COUNTRIES),
onReloadFinished,
options={
"order": "name",
"hidebroken": True
})
def stations(self, country, callback, offset=0, limit=2000):
def onReloadFinished(data):
def _onJson(jsonData):
lst = OrderedDict()
for entry in jsonData:
name = entry["name"]
station = lst.get(name, Station(entry))
station.extend(entry)
lst[name] = station
callback(lst)
self._parseJson(data, _onJson)
options = {
"order": "clickcount",
"reverse": True,
"offset": offset,
"limit": limit,
"hidebroken": True
}
self.request("{}{}/{}".format(self.URL_BASE,
self.URL_STATIONS_BY_COUNTRY,
quote(country)),
onReloadFinished,
options=options)
def _onResponse(self, response, callback):
readBody(response).addCallbacks(callback, errback=self._onReloadError)
def _onReloadError(self, failure, *args):
Log.w("RELOAD ERROR! {}".format(failure))
if failure.check(
RequestGenerationFailed,
RequestTransmissionFailed,
ResponseFailed,
):
failure.value.reasons[0].printTraceback()
else:
failure.printTraceback()
