def loginToWiki(wiki): if wiki in WIKILOGINS: return WIKILOGINS[wiki] desktop = XSCRIPTCONTEXT.getDesktop() model = desktop.getCurrentComponent() sheet = config("Wikipedia") if not model.Sheets.hasByName(sheet): model.Sheets.insertNewByName(sheet, model.Sheets.getCount() + 1) sheet = model.Sheets.getByName(sheet) sheet.getCellRangeByName("A1").setString("URL") sheet.getCellRangeByName("A2").setString("Username") sheet.getCellRangeByName("A3").setString("Password") else: sheet = model.Sheets.getByName(sheet) urlColumn = getColumn("URL") userColumn = getColumn("Username") passwordColumn = getColumn("Password") index = 2 levelCell = sheet.getCellRangeByName(urlColumn + str(index)) while (levelCell.getType().value != "EMPTY"): if (levelCell.String == wiki): break index = index + 1 levelCell = sheet.getCellRangeByName(urlColumn + str(index)) username = sheet.getCellRangeByName(userColumn + str(index)) password = sheet.getCellRangeByName(passwordColumn + str(index)) session = CacheControl(requests.session()) session.headers.update(USERAGENT) logintokens = session.get(url = wiki, params = {"action": "query", "meta": "tokens", "type": "login", "maxlag": 5, "format": "json"}).json()["query"]["tokens"]["logintoken"] session.post(wiki, data = { "action": "login", "lgname": username, "lgpassword": password, "lgtoken": logintokens}) CSRF = session.get(url=wiki, params = { "action": "query", "meta": "tokens", "maxlag": 5, "format": "json"}).json()["query"]["tokens"]["csrftoken"] WIKILOGINS = {wiki: CSRF} return CSRF
class FeatureRequesterImpl(FeatureRequester): def __init__(self, sdk_key, config): self._sdk_key = sdk_key self._session = CacheControl(requests.Session()) self._config = config def get_all(self): hdrs = _headers(self._sdk_key) uri = self._config.get_latest_features_uri log.debug("Getting all flags using uri: " + uri) r = self._session.get(uri, headers=hdrs, timeout=(self._config.connect_timeout, self._config.read_timeout)) r.raise_for_status() features = r.json() return features def get_one(self, key): hdrs = _headers(self._sdk_key) uri = self._config.get_latest_features_uri + '/' + key log.debug("Getting one feature flag using uri: " + uri) r = self._session.get(uri, headers=hdrs, timeout=(self._config.connect_timeout, self._config.read_timeout)) r.raise_for_status() feature = r.json() return feature
def test_file_cache_recognizes_consumed_file_handle(self, url): s = CacheControl(Session(), FileCache("web_cache")) the_url = url + "cache_60" s.get(the_url) r = s.get(the_url) assert r.from_cache s.close()
class CacheControlHelper(object): def __init__(self): self.sess = CacheControl(requests.session(), heuristic=CustomHeuristic(days=30), cache=FileCache('.web_cache')) self.exceptions = requests.exceptions def get(self, url, params=None, timeout=120, cookies=None, headers={'Accept': 'application/json'}): if cookies: return self.sess.get(url, params=params, timeout=timeout, cookies=cookies, headers=headers) else: return self.sess.get(url, params=params, timeout=timeout, headers=headers) def post(self, url, data, timeout=120, headers={'Accept': 'application/json'}): return self.sess.post(url, data=data, timeout=timeout, headers=headers)
def getTracks(area, bboxInformation): apis = getApis() session = requests.session() session.headers.update({"User-Agent": USER_AGENT}) cached_session = CacheControl(session) for api in apis: if "data" in api: data = api["data"] if "bbTopLeft" in data: data["bbTopLeft"] = "{lat},{lon}".format( lat=bboxInformation[3], lon=bboxInformation[0]) if "bbBottomRight" in data: data["bbBottomRight"] = "{lat},{lon}".format( lat=bboxInformation[1], lon=bboxInformation[2]) response = cached_session.post(api["api"] + api["tracks"], data=data) tJson = response.json() else: turl = api["api"] + api["tracks"] params = api["params"] params["bbox"] = params["bbox"].format( minx=bboxInformation[0], miny=bboxInformation[1], maxx=bboxInformation[2], maxy=bboxInformation[3], ) response = cached_session.get(turl, params=params) print(response.url) while response.status_code != requests.codes.ok: time.sleep(1) response = cached_session.get(turl, params=params) try: tJson = response.json() except json.decoder.JSONDecodeError as e: print(response.url) print(response.text) raise e while "next" in response.links: next_url = response.links["next"]["url"] response = cached_session.get(next_url) while response.status_code != requests.codes.ok: time.sleep(1) response = cached_session.get(next_url) try: tJson["features"] = (tJson["features"] + response.json()["features"]) except json.decoder.JSONDecodeError as e: print(response.url) print(response.text) raise e if response.status_code != requests.codes.ok: raise ValueError("{} gave us a status code of {}".format( response.url, response.status_code)) if api["name"] == "openstreetcam": tJson = convertJson(tJson) save(area, api["name"], tJson)
def full(): sess = requests.session() cached_sess = CacheControl(sess) r_articles = cached_sess.get('http://localhost/articles/retrieve_articles/10/', auth=('email', 'password')) items = [] if r_articles.status_code == 200: articles = r_articles.json() for article in articles: r_comments = cached_sess.get('http://localhost/comments/count/' + str(article['article_id']), auth=('email', 'password')) if r_comments.status_code == 200: commentCount = r_comments.json() else: print(str(r_comments.status_code), file=sys.stderr) return "comments error" r_tags = cached_sess.get('http://localhost/tags/all/' + str(article['article_id']), auth=('email', 'password')) if r_tags.status_code == 200: allTags = r_tags.json() tags_arr = [] for tag in allTags: tags_arr.append( tag['tag'] ) else: print(str(r_tags.status_code), file=sys.stderr) return "tags error" items.append( Item( title = article['title'], author = article['author'], description = article['content'], categories = tags_arr, comments = commentCount['numOfComments'] )) feed = Feed( title = "Full RSS Feed", link = "http://localhost/rss/full", description = "This feed shows a full content for 10 articles", language = "en-US", lastBuildDate = datetime.datetime.now(), items = items ) return feed.rss() else: print(str(r_articles.status_code), file=sys.stderr) return "articles error"
class AzureVerifier: api_version = 'v2.0' authority = 'https://login.microsoftonline.com' tenant = 'common' discovery_document = '.well-known/openid-configuration' audiences = None issuer = None def __init__(self, tenant='common', issuer=None, audiences=None): self.tenant = tenant or self.tenant self.issuer = issuer self.audiences = audiences self.session = CacheControl(requests.Session()) @property def documents_uri(self): return f'{self.authority}/{self.tenant}/{self.api_version}/{self.discovery_document}' @property def jwks_uri(self): return self.session.get(self.documents_uri).json()['jwks_uri'] @property def jwks(self): return self.session.get(self.jwks_uri).json() def get_jwk(self, kid): for jwk in self.jwks.get('keys'): if jwk.get('kid') == kid: return jwk raise InvalidAuthorizationToken('kid not recognized') def get_kid(self, token): headers = jwt.get_unverified_header(token) if not headers: raise InvalidAuthorizationToken('missing headers') try: return headers['kid'] except KeyError: raise InvalidAuthorizationToken('missing kid') def get_public_key(self, token): return rsa_pem_from_jwk(self.get_jwk(self.get_kid(token))) def verify(self, token): public_key = self.get_public_key(token) return jwt.decode(token, public_key, verify=True, algorithms=['RS256'], audience=self.audiences, issuer=self.issuer)
class TestStream(object): def setup(self): self.sess = CacheControl(requests.Session()) def test_stream_is_cached(self, url): resp_1 = self.sess.get(url + 'stream') content_1 = resp_1.content resp_2 = self.sess.get(url + 'stream') content_2 = resp_1.content assert not resp_1.from_cache assert resp_2.from_cache assert content_1 == content_2
class SQLiteCacheTest(unittest.TestCase): def setUp(self): self.url = "https://httpbin.org/cache/60" self.sess = CacheControl(requests.Session(), cache=SQLiteCache(":memory:")) def tearDown(self): self.sess.close() def test_simple(self): response = self.sess.get(self.url) assert not response.from_cache response = self.sess.get(self.url) assert response.from_cache
def test_not_modified_releases_connection(self, server): sess = CacheControl(requests.Session()) etag_url = urljoin(server.application_url, "/etag") sess.get(etag_url) resp = Mock(status=304, headers={}) # This is how the urllib3 response is created in # requests.adapters response_mod = "requests.adapters.HTTPResponse.from_httplib" with patch(response_mod, Mock(return_value=resp)): sess.get(etag_url) assert resp.read.called assert resp.release_conn.called
class JSONLocator(Locator): def __init__(self, url=PYPI_JSON_URL): self.url = url self.session = CacheControl(requests.session()) def versions(self, distribution): url = "{}/{}/json".format(self.url, distribution) response = self.session.get(url) ret = [] j = response.json()['releases'] return [v for v, d in j.items() if len(d) > 0] def get(self, distribution, version): url = "{}/{}/json".format(self.url, distribution) response = self.session.get(url) # Reformat the data... return response.json()['releases'][version]
def test_not_modified_releases_connection(self, server, url): sess = CacheControl(requests.Session()) etag_url = urljoin(url, "/etag") sess.get(etag_url) resp = Mock(status=304, headers={}) # This is how the urllib3 response is created in # requests.adapters response_mod = "requests.adapters.HTTPResponse.from_httplib" with patch(response_mod, Mock(return_value=resp)): sess.get(etag_url) assert resp.read.called assert resp.release_conn.called
def all_sites(sitemap_url='http://library.link/harvest/sitemap.xml'): ''' >>> from librarylink.util import all_sites >>> [ s.host for s in all_sites() if 'denverlibrary' in s.host ] ['link.denverlibrary.org'] ''' #FIXME: Avoid accumulating all the nodes, which will require improvements to xml.treesequence @coroutine def sink(accumulator): while True: e = yield loc = next(select_name(e, 'loc')) lastmod = next(select_name(e, 'lastmod')) s = liblink_site() s.sitemap = loc.xml_value s.url, _, tail = s.sitemap.partition('harvest/sitemap.xml') s.base_url = s.url #Legacy property name #Early warning for funky URLs breaking stuff downstream assert not tail protocol, s.host, path, query, fragment = iri.split_uri_ref(s.sitemap) s.lastmod = lastmod.xml_value accumulator.append(s) nodes = [] ts = xml.treesequence(('sitemapindex', 'sitemap'), sink(nodes)) if hasattr (all_sites, 'cachedir'): sess = CacheControl(requests.Session(), cache=FileCache(all_sites.cachedir)) else: sess = CacheControl(requests.Session()) result = sess.get(sitemap_url) ts.parse(result.text) yield from nodes
def overpass_status(api_status_url="https://overpass-api.de/api/status"): """Get the overpass status -- this returns an int with the time to wait""" session = requests.session() session.headers.update({"User-Agent": USER_AGENT}) cached_session = CacheControl(session) response = cached_session.get(api_status_url) if response.status_code != requests.codes.ok: raise ValueError("Bad Request: {}".format(api_status_url)) parsed_response = {"wait_time": []} for i in response.text.splitlines(): if "Connected as" in i: parsed_response["connected_as"] = i.split(":")[1].strip() elif "Current time" in i: parsed_response["current_time"] = i.split(":")[1].strip() elif "Rate limit" in i: parsed_response["rate_limit"] = int(i.split(":")[1].strip()) elif "slots available now" in i: parsed_response["slots_available"] = int(i.split(" ")[0].strip()) elif "Slot available after" in i: parsed_response["wait_time"].append(int(i.split(" ")[5])) if "slots_available" not in parsed_response: parsed_response["slots_available"] = 0 wait_time = 0 if (parsed_response["rate_limit"] - parsed_response["slots_available"] >= 2 and len(parsed_response["wait_time"]) > 0): return max(parsed_response["wait_time"]) return wait_time
def download(workdir, url): """Download a file, using .cache inside workdir as an HTTP cache.""" logging.debug(u"initializing requests and cache-control") session = CacheControl(requests.Session(), cache=FileCache(os.path.join(workdir, '.cache'))) session.mount('file://', LocalFileAdapter()) req = session.get(url, stream=True) try: downloaded_file = tempfile.TemporaryFile() size = 0 start = datetime.datetime.now() for chunk in req.iter_content(chunk_size=1024000): if chunk: sys.stdout.write('.') sys.stdout.flush() downloaded_file.write(chunk) size += len(chunk) # print newline print() downloaded_file.flush() logging.info(u"downloaded {} - {} o. in {} s.", url, size, (datetime.datetime.now() - start).total_seconds()) logging.debug(u"reset file pointer - seek(0)") downloaded_file.seek(0) return downloaded_file except Exception as exc: logging.debug(u"error on download, closing and deleting file") downloaded_file.close() raise exc
def fetch(self): feed = None if InformantConfig().get_argv_use_cache(): cachefile = InformantConfig().get_cachefile() os.umask( 0o0002 ) # unrestrict umask so we can cache with proper permissions try: session = CacheControl(requests.Session(), cache=FileCache(cachefile, filemode=0o0664, dirmode=0o0775)) feed = feedparser.parse(session.get(self.url).content) except Exception as e: ui.err_print('Unable to read cache information: {}'.format(e)) feed = feedparser.parse(self.url) else: feed = feedparser.parse(self.url) if feed.bozo: ui.err_print('Encountered feed error: {}'.format( feed.bozo_exception)) sys.exit(255) else: return feed
class MAL(BaseLibrary, BaseMetadata): def __init__(self): super().__init__() self.name = "MAL" self.logo_url = 'https://upload.wikimedia.org/wikipedia/commons/7/7a/MyAnimeList_Logo.png' self.website_url = 'https://myanimelist.net/' client_id = "add1ed488bd218c2e10146345377a0b8" url_auth = "https://myanimelist.net/v1/oauth2/authorize" url_token = "https://myanimelist.net/v1/oauth2/token" self.authenticator = OAuth(self.name, client_id, url_auth, url_token) self.requests_session = CacheControl(requests.Session(), cache=FileCache('.Cache/MAL'), heuristic=MALHeuristic()) # self.requests_session = requests.Session() self.rate_limiter = AsyncRateLimiter(max_calls=100, period=1, callback=limited) # Currently MAL is not rate limiting, but if it starts, i'll leave this here. def load(self, url): header = { "Authorization": str(self.authenticator.token_type + " " + self.authenticator.token) } with self.rate_limiter: result = self.requests_session.get(url, headers=header) try: if result.from_cache: # deletes last call if it was cached. only real api calls need to be slowed down self.rate_limiter.calls.pop() except: pass if result.ok: return json.loads(result.content) else: print(result.content) raise RuntimeError('Failed to grab data') def Lists(self): print("MAL: Obtaining User list") all_fields = "?fields=list_status&limit=1000" url = URL_MAIN + URL_ANIME_LIST.format(user_name="@me") + all_fields # Todo: implement paging list = self.load(url)['data'] AnimeList = MALFormatter.AnimeList(list) return AnimeList def PopulateAnime(self, database: Database, anime_hash: str): oldAnimeData = AnimeStruct.Anime.from_db(anime_hash, database) if oldAnimeData.id.getID("MAL"): malID = oldAnimeData.id.getID("MAL") print(f"MAL: Obtaining Anime Metadata: {malID}") url = URL_MAIN + URL_DETAILS.format( id=str(malID)) + ANIME_ALL_FIELDS anime_metadata = self.load(url) properAnime = MALFormatter.AnimeMetadata(anime_metadata, oldAnimeData) # remove edges to stop the anime from keeping some old info like type database.remove_successor_edges(oldAnimeData.hash) properAnime.to_db(database)
def open(self): global SESSION if SESSION is None: SESSION = CacheControl(Session(), cache=FileCache(SESSION_CACHE_PATH)) try: self._response = SESSION.get(self.uri, headers=self.headers) except InvalidSchema as e: raise DocumentNotFoundException( u'document not found: "{0}"'.format(self.uri), cause=e) except ConnectionError as e: raise LoaderException(u'request connection error: "{0}"'.format( self.uri), cause=e) except Exception as e: raise LoaderException(u'request error: "{0}"'.format(self.uri), cause=e) status = self._response.status_code if status == 404: self._response = None raise DocumentNotFoundException( u'document not found: "{0}"'.format(self.uri)) elif status != 200: self._response = None raise LoaderException(u'request error {0:d}: "{1}"'.format( status, self.uri))
class LDClient(object): def __init__(self, apiKey, config=Config.default()): self._apiKey = apiKey self._config = config self._session = CacheControl(requests.Session()) def get_flag(self, key, user, default=False): try: return self._get_flag(key, user, default) except: logging.exception('Unhandled exception in get_flag. Returning default value for flag.') return default def _get_flag(self, key, user, default): hdrs = {'Authorization': 'api_key ' + self._apiKey, 'User-Agent': 'PythonClient/' + __version__} uri = self._config._base_uri + '/api/eval/features/' + key r = self._session.get(uri, headers=hdrs, timeout = (self._config._connect, self._config._read)) dict = r.json() val = _evaluate(dict, user) if val is None: return default else: return val
class SocrataApi: """ Class for starting a session for requests via Socrata APIs. Initialize with a base_url """ def __init__(self, base_url: str): self.session = CacheControl(requests.Session()) self.base_url = base_url self.resource_url = urljoin(self.base_url, '/resource/') self.metadata_url = urljoin(self.base_url, '/api/views/metadata/v1/') def request(self, url:str, **kwargs: Any) -> Dict: try: response = self.session.get(url, **kwargs) response.raise_for_status() return response.json() except requests.exceptions.HTTPError as http_err: try: server_message = response.json()['message'] # see if the API returned message data except Exception: # if no JSON data, re-rasie the original error raise http_err raise BadRequest(server_message, response=response) def resource(self, resource_id: str, **kwargs: Any) -> Dict: return self.request(f'{self.resource_url}{resource_id}', **kwargs) def metadata(self, resource_id: str, **kwargs: Any) -> Dict: return self.request(f'{self.metadata_url}{resource_id}.json', **kwargs)
def summary(): # sess = requests.session() cached_sess = CacheControl(sess) response = cached_sess.get('http://localhost/articles/retrieve_articles/10/', auth=('email', 'password')) articles = response.json() items = [] # if r_articles.status_code == 200: if response.status_code == 200: for article in articles: items.append( Item( title = article['title'], link = "http://localhost/articles/" + str(article['article_id']), author = article['author'], pubDate = datetime.datetime.strptime(str(article['date_published']), "%a, %d %b %Y %H:%M:%S GMT") )) feed = Feed( title = "Summary RSS Feed", link = "http://localhost/rss/summary", description = "This feed shows a summary of the 10 most recent articles", language = "en-US", lastBuildDate = datetime.datetime.now(), items = items ) return feed.rss() else: # print(str(r_articles.status_code), file=sys.stderr) print(str(response.status_code), file=sys.stderr) return "articles error"
def center_iterator(client=None) -> Iterator[Dict]: if not PLATFORM_ENABLED: logger.warning( f"{PLATFORM.capitalize()} scrap is disabled in configuration file." ) return [] session = CacheControl(requests.Session(), cache=FileCache("./cache")) if client: session = client try: url = f'{get_config().get("base_urls").get("github_public_path")}{get_conf_outputs().get("centers_json_path").format(PLATFORM)}' response = session.get(url) # Si on ne vient pas des tests unitaires if not client: if response.from_cache: logger.info( f"Liste des centres pour {PLATFORM} vient du cache") else: logger.info( f"Liste des centres pour {PLATFORM} est une vraie requête") data = response.json() logger.info( f"Found {len(data)} {PLATFORM.capitalize()} centers (external scraper)." ) for center in data: yield center except Exception as e: logger.warning(f"Unable to scrape {PLATFORM} centers: {e}")
def main(): """ Execution begins here. """ # Use our standard logger template logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.DEBUG, ) logger = logging.getLogger() # Specify list of test files to download. # Go to http://njrusmc.net/cache/cache.html to see all test files base_url = "http://njrusmc.net/cache" test_list = [ "zero128k_public60.test", # Cache-Control: public, max-age=60 "zero128k_nostore.test", # Cache-Control: no-store ] # For each file, run two GET requests, and use the logger to print out # the relevant information as requests are processed for test_file in test_list: # Assemble the complete URL to feed into the HTTP GET request url = f"{base_url}/{test_file}" # Create the cached session object, which automatically intereprets # caching-related headers (requests doesn't do it natively) cached_sess = CacheControl(requests.session()) # Print information from first run, include key headers logger.info("First GET to %s", url) resp = cached_sess.get(url) resp.raise_for_status() print_response(resp, dump_body=False) print(f"\n\n{'*' * 80}\n\n") # Slight delay just to show the cache timer countdown # Print information from second run, but focus is on background debugs time.sleep(2) logger.info("Second GET to %s", url) resp = cached_sess.get(url) resp.raise_for_status() print_response(resp, dump_body=False) print(f"\n\n{'*' * 80}\n\n")
def getUserID(user): api_url = "https://www.openstreetmap.org/api/0.6/changesets" data = {"display_name": user} session = CacheControl(requests.session()) result = session.get(api_url, params=data).text root = ElementTree.fromstring(result) changeset = root.find("changeset") return changeset.attrib['uid']
class TestHeuristicWith3xxResponse(object): def setup(self): class DummyHeuristic(BaseHeuristic): def update_headers(self, resp): return {"x-dummy-header": "foobar"} self.sess = CacheControl(Session(), heuristic=DummyHeuristic()) def test_heuristic_applies_to_301(self, url): the_url = url + "permanent_redirect" resp = self.sess.get(the_url) assert "x-dummy-header" in resp.headers def test_heuristic_applies_to_304(self, url): the_url = url + "conditional_get" resp = self.sess.get(the_url) assert "x-dummy-header" in resp.headers
def get_reader(self): sess = CacheControl(requests.Session(), cache=FileCache(gettempdir())) req = sess.get(self.file) # if the response is not 200, an exception will be raised req.raise_for_status() return io.BufferedReader(io.BytesIO(req.content))
def Request(url, method="GET", headers=DEFAULT_HEADERS, additional_headers=None, data=None, session=None, allow_redirects=True, timeout=10, load_cookies=True, mobile=False): if additional_headers: headers.update(additional_headers) try: session = CacheControl(session) except Exception as e: pass # Error("Init web cache failed!!!", e) if mobile: headers["User-Agents"] = MOBILE_IOS_AGENTS xbmc.log("Requests headers: {0}".format(json.dumps(headers)), 1) if session: session.headers.update(headers) domain = re.search("https*\://(.+?)($|/)", url).group(1) if load_cookies: LoadCookies(session, cookies_name=domain) if data: response = session.post(url, data=data, allow_redirects=allow_redirects, timeout=timeout, verify=False) else: if method == "HEAD": response = session.head(url, allow_redirects=allow_redirects, timeout=timeout, verify=False) else: response = session.get(url, allow_redirects=allow_redirects, timeout=timeout, verify=False) response.encoding = "utf8" SaveCookies(session, cookies_name=domain) return response else: if method == "HEAD": return requests.head(url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, verify=False) else: return requests.get(url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, verify=False)
def getURL(url, post_data=None, params=None, headers={}, timeout=30, session=None, json=False): """ Returns a byte-string retrieved from the url provider. """ # request session cache_dir = sickbeard.CACHE_DIR or _getTempDir() session = CacheControl(sess=session, cache=caches.FileCache( os.path.join(cache_dir, 'sessions'))) # request session headers session.headers.update({ 'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate' }) session.headers.update(headers) # request session ssl verify session.verify = False # request session paramaters session.params = params try: # request session proxies if sickbeard.PROXY_SETTING: logger.log("Using proxy for url: " + url, logger.DEBUG) session.proxies = { "http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING, } # decide if we get or post data to server if post_data: resp = session.post(url, data=post_data, timeout=timeout) else: resp = session.get(url, timeout=timeout) if not resp.ok: logger.log( u"Requested url " + url + " returned status code is " + str(resp.status_code) + ': ' + clients.http_error_code[resp.status_code], logger.DEBUG) return except requests.exceptions.HTTPError, e: logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING) return
def comments(): sess = requests.session() cached_sess = CacheControl(sess) r_articles = cached_sess.get('http://localhost/articles/retrieve_articles/10/' , auth=('email', 'password')) if r_articles.status_code == 200: articles = r_articles.json() items = [] for article in articles: r_comments = cached_sess.get('http://localhost/comments/recent/' + str(article['article_id']) + '/10', auth=('email', 'password')) if r_comments.status_code == 200: comments = r_comments.json() all_comments = [] for comment in comments: all_comments.append( comment['comment'] ) elif r_comments.status_code == 404: pass else: print(str(r_comments.status_code), file=sys.stderr) return "comments error" items.append( Item( title = article['title'], pubDate = datetime.datetime.strptime(str(article['date_published']), "%a, %d %b %Y %H:%M:%S GMT"), description = all_comments )) feed = Feed( title = "Comments feed for articles", link = "http://localhost/rss/comments", description = "This feed shows comments for each article.", language = "en-US", lastBuildDate = datetime.datetime.now(), items = items ) return feed.rss() else: print(str(r_articles.status_code), file=sys.stderr)
def main(): """ Execution begins here. """ # Use our standard logger template logging.basicConfig( format="%(asctime)s %(levelname)-8s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.DEBUG, ) logger = logging.getLogger() # Specify list of URLs to perform an HTTP GET against # Author's note: These files don't have "Cache-Control" anymore as I removed # them after the demo. Please replace these URLs with your own! url_list = [ "http://njrusmc.net/jobaid/mpls_pcap.zip", # Cache-Control: public (300s) "http://njrusmc.net/jobaid/ipsec_pcap.zip", # Cache-Control: no-store ] # For each URL, run two GET requests, and use the logger to print out # the relevant information as requests are processed for url in url_list: # Create the cached session object, which automatically intereprets # caching-related headers (requests doesn't do it natively) cached_sess = CacheControl(requests.session()) # Print information from first run, include key headers logger.info("First GET to %s", url) resp = cached_sess.get(url) resp.raise_for_status() print_response(resp, dump_body=False) # Slight delay just to show the cache timer countdown # Print information from second run, but focus is on background debugs time.sleep(2) logger.info("Second GET to %s", url) resp = cached_sess.get(url) resp.raise_for_status() print_response(resp, dump_body=False)
def main(): """ Execution begins here. """ # Create a logger object to let us see what is happening behind the # scenes with the HTTP URLs logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.DEBUG) # Specify list of URLs to perform an HTTP GET against # Author's note: These files don't have "Cache-Control" anymore as I removed # them after the demo. Please replace these URLs with your own! url_list = [ "http://njrusmc.net/jobaid/wlan_pcap.zip", # Cache-Control: public (300s) "http://njrusmc.net/jobaid/lmnop_answers.pdf", # Cache-Control: no-store ] # For each URL, run two GET requests, and use the logger to print out # the relevant information as requests are processed for url in url_list: # Create the cached session object, which automatically intereprets # caching-related headers (requests doesn't do it natively) cached_sess = CacheControl(requests.session()) # Print information from first run, include key headers logger.info( "---------------------------------------------------------") logger.info("First GET to URL: %s", url) resp = cached_sess.get(url) logger.info("Response %s / %s", resp.status_code, resp.reason) logger.info("Cache-Control: %s", resp.headers.get("Cache-Control")) logger.info("ETag: %s", resp.headers.get("ETag")) # Slight delay just to show the cache timer countdown # Print information from second run, but focus is on background debugs time.sleep(1.5) logger.info("Second GET to URL: %s", url) resp = cached_sess.get(url) logger.info("Response %s / %s", resp.status_code, resp.reason)
class reQuiver(object): def __init__(self): self._raw_endpoint = "http://quiver.archerdx.com/results?query=" self._sesh = CacheControl(requests.Session()) def query(self, query): if len(query) == 0: raise EmptyQueryStringException() q_string = self._raw_endpoint + str(query) response = self._sesh.get(q_string) if response.status_code != 200: raise NetworkErrorException(response.status_code) soup = BeautifulSoup(response.content, "html.parser") # parse the panels panels = soup.find(panel_table_filter) panels_list = [] if panels is not None: for row in panels.find_all("tr"): cells = row.find_all("td") if len(cells) == 2: link = cells[0].a['href'] genes = [clean_string(gene) for gene in cells[1].string.split()] panels_list.append(QuiverFushionPlexPanel(link, genes)) # parse the fusions fusions = soup.find_all(fusion_table_filter) fusions_list = [] if fusions is not None: for fusion in fusions: table = fusion.find('table') for row in table.find_all('tr'): cells = row.find_all('td') if len(cells) != 2: # get the link link = cells[0].a['href'] original_annotation = clean_string(cells[1].string) disease = cells[2].string.strip() pubmed_link = cells[3].a['href'] evidence_count = int(cells[4].string) fusions_list.append(QuiverGeneFushion(link, original_annotation, disease, pubmed_link, evidence_count)) return QuiverResultSet(panels_list, fusions_list, query)
def amalgama_lyrics(artist, song): url = amalgama.get_url(artist, song) try: cached_sess = CacheControl(sess, cache=FileCache('.amalgama')) response = cached_sess.get(url) response.raise_for_status() except requests.exceptions.HTTPError: print(f'{artist}-{song} not found in amalgama {url}') return None text = amalgama.get_html(response.text) return text
def get_cached_session(): sess = CacheControl(requests.Session(), cache=FileCache(CACHE_DIR), heuristic=LastModifiedNoDate(require_date=False)) original_get = sess.get def wrapped_get(*args, **kwargs): try: return original_get(*args, **kwargs) except (OSError, IOError) as e: return requests.get(*args, **kwargs) sess.get = wrapped_get return sess
def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=None, json=False): """ Returns a byte-string retrieved from the url provider. """ # request session cache_dir = sickbeard.CACHE_DIR or _getTempDir() session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, "sessions"))) # request session headers req_headers = {"User-Agent": USER_AGENT, "Accept-Encoding": "gzip,deflate"} if headers: req_headers.update(headers) session.headers.update(req_headers) # request session ssl verify session.verify = False # request session paramaters session.params = params try: # Remove double-slashes from url parsed = list(urlparse.urlparse(url)) parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one url = urlparse.urlunparse(parsed) # request session proxies if sickbeard.PROXY_SETTING: logger.log("Using proxy for url: " + url, logger.DEBUG) session.proxies = {"http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING} # decide if we get or post data to server if post_data: resp = session.post(url, data=post_data, timeout=timeout) else: resp = session.get(url, timeout=timeout) if not resp.ok: logger.log( u"Requested url " + url + " returned status code is " + str(resp.status_code) + ": " + clients.http_error_code[resp.status_code], logger.DEBUG, ) return except requests.exceptions.HTTPError, e: logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING) return
def fetch_file(url, encoding=None): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) try: response = s.get(url, headers={}) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection") _assert(response.ok, "%s\nFile request failed: %s (%d)" % (url, response.reason, response.status_code)) if encoding: response.encoding = encoding return response.text
class TwistedHttpFeatureRequester(FeatureRequester): def __init__(self, api_key, config): self._api_key = api_key self._session = CacheControl(txrequests.Session()) self._config = config def get(self, key, callback): d = self.toggle(key) d.addBoth(callback) return d def toggle(self, key): @defer.inlineCallbacks def run(should_retry): # noinspection PyBroadException try: val = yield self._toggle(key) defer.returnValue(val) except ProtocolError as e: inner = e.args[1] if inner.errno == errno.ECONNRESET and should_retry: log.warning( 'ProtocolError exception caught while getting flag. Retrying.') d = yield run(False) defer.returnValue(d) else: log.exception( 'Unhandled exception. Returning default value for flag.') defer.returnValue(None) except Exception: log.exception( 'Unhandled exception. Returning default value for flag.') defer.returnValue(None) return run(True) @defer.inlineCallbacks def _toggle(self, key): hdrs = _headers(self._api_key) uri = self._config.base_uri + '/api/eval/features/' + key r = yield self._session.get(uri, headers=hdrs, timeout=(self._config.connect, self._config.read)) r.raise_for_status() feature = r.json() defer.returnValue(feature)
class TestHeuristicWithoutWarning(object): def setup(self): class NoopHeuristic(BaseHeuristic): warning = Mock() def update_headers(self, resp): return {} self.heuristic = NoopHeuristic() self.sess = CacheControl(Session(), heuristic=self.heuristic) def test_no_header_change_means_no_warning_header(self, url): the_url = url + "optional_cacheable_request" resp = self.sess.get(the_url) assert not self.heuristic.warning.called
def download_file(url, filename, session=None): # create session cache_dir = sickbeard.CACHE_DIR or _getTempDir() session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, "sessions"))) # request session headers session.headers.update({"User-Agent": USER_AGENT, "Accept-Encoding": "gzip,deflate"}) # request session ssl verify session.verify = False # request session streaming session.stream = True # request session proxies if sickbeard.PROXY_SETTING: logger.log("Using proxy for url: " + url, logger.DEBUG) session.proxies = {"http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING} try: resp = session.get(url) if not resp.ok: logger.log( u"Requested url " + url + " returned status code is " + str(resp.status_code) + ": " + clients.http_error_code[resp.status_code], logger.DEBUG, ) return False with open(filename, "wb") as fp: for chunk in resp.iter_content(chunk_size=1024): if chunk: fp.write(chunk) fp.flush() chmodAsParent(filename) except requests.exceptions.HTTPError, e: _remove_file_failed(filename) logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING) return False
def _get_filehandle(filepath_or, *args, **kwargs): """Open file if `filepath_or` looks like a string/unicode/bytes, else pass through. """ if _is_string_or_bytes(filepath_or): if requests.compat.urlparse(filepath_or).scheme in {'http', 'https'}: sess = CacheControl(requests.Session(), cache=FileCache(gettempdir())) req = sess.get(filepath_or, **kwargs) # if the response is not 200, an exception will be raised req.raise_for_status() fh, own_fh = BytesIO(req.content), True else: fh, own_fh = open(filepath_or, *args, **kwargs), True else: fh, own_fh = filepath_or, False return fh, own_fh
def getURL(url, post_data=None, params=None, headers={}, timeout=30, session=None, json=False): """ Returns a byte-string retrieved from the url provider. """ # request session cache_dir = sickbeard.CACHE_DIR or _getTempDir() session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, 'sessions'))) # request session headers session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'}) session.headers.update(headers) # request session ssl verify session.verify = False # request session paramaters session.params = params try: # request session proxies if sickbeard.PROXY_SETTING: logger.log("Using proxy for url: " + url, logger.DEBUG) session.proxies = { "http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING, } # decide if we get or post data to server if post_data: resp = session.post(url, data=post_data, timeout=timeout) else: resp = session.get(url, timeout=timeout) if not resp.ok: logger.log(u"Requested url " + url + " returned status code is " + str( resp.status_code) + ': ' + clients.http_error_code[resp.status_code], logger.DEBUG) return except requests.exceptions.HTTPError, e: logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING) return
def Request( url, method="GET", headers=DEFAULT_HEADERS, additional_headers=None, data=None, session=None, allow_redirects=True, timeout=10, load_cookies=True, mobile=False ): if additional_headers: headers.update(additional_headers) try: session = CacheControl(session) except Exception as e: pass # Error("Init web cache failed!!!", e) if mobile: headers["User-Agents"] = MOBILE_IOS_AGENTS xbmc.log("Requests headers: {0}".format(json.dumps(headers)), 1) if session: session.headers.update(headers) domain = re.search("https*\://(.+?)($|/)", url).group(1) if load_cookies: LoadCookies(session, cookies_name=domain) if data: response = session.post(url, data=data, allow_redirects=allow_redirects, timeout=timeout, verify=False) else: if method == "HEAD": response = session.head(url, allow_redirects=allow_redirects, timeout=timeout, verify=False) else: response = session.get(url, allow_redirects=allow_redirects, timeout=timeout, verify=False) response.encoding = "utf8" SaveCookies(session, cookies_name=domain) return response else: if method == "HEAD": return requests.head(url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, verify=False) else: return requests.get(url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, verify=False)
def main(): current = pkg_resources.get_distribution('tst').version if not sys.stdout.isatty(): print(current) return cprint(WHITE, current, file=sys.stdout) try: s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) response = s.get('https://pypi.org/pypi/tst/json') data = response.json() except requests.ConnectionError: pass latest_version = data['info']['version'] if current != latest_version: cprint(YELLOW, 'Latest version available: %s' % latest_version, file=sys.stdout) cprint(RESET, '---\nUse `pip install --upgrade tst`') cprint(RESET, ' or `pip install --upgrade --user tst`')
class RequestsFeatureRequester(FeatureRequester): def __init__(self, api_key, config): self._api_key = api_key self._session = CacheControl(requests.Session()) self._config = config def get(self, key, callback): # return callback(do_toggle(key)) def do_toggle(should_retry): # noinspection PyBroadException,PyUnresolvedReferences try: val = self._toggle(key) return val except ProtocolError as e: inner = e.args[1] if inner.errno == errno.ECONNRESET and should_retry: log.warning( 'ProtocolError exception caught while getting flag. Retrying.') return do_toggle(False) else: log.exception( 'Unhandled exception. Returning default value for flag.') return None except Exception: log.exception( 'Unhandled exception. Returning default value for flag.') return None return callback(do_toggle(True)) def _toggle(self, key): hdrs = _headers(self._api_key) uri = self._config.base_uri + '/api/eval/features/' + key r = self._session.get(uri, headers=hdrs, timeout=( self._config.connect, self._config.read)) r.raise_for_status() feature = r.json() return feature
def downloadHttpFile(httpurl): # type: (Text) -> Text cache_session = None if "XDG_CACHE_HOME" in os.environ: directory = os.environ["XDG_CACHE_HOME"] elif "HOME" in os.environ: directory = os.environ["HOME"] else: directory = os.path.expanduser('~') cache_session = CacheControl( requests.Session(), cache=FileCache( os.path.join(directory, ".cache", "cwltool"))) r = cache_session.get(httpurl, stream=True) with NamedTemporaryFile(mode='wb', delete=False) as f: for chunk in r.iter_content(chunk_size=16384): if chunk: # filter out keep-alive new chunks f.write(chunk) r.close() return f.name
def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=None, json=False): """ Returns a byte-string retrieved from the url provider. """ # request session session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(sickbeard.CACHE_DIR, 'sessions'))) # request session headers req_headers = {'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'} if headers: req_headers.update(headers) session.headers.update(req_headers) # request session ssl verify session.verify = False # request session paramaters session.params = params try: # Remove double-slashes from url parsed = list(urlparse.urlparse(url)) parsed[2] = re.sub("/{2,}", "/", parsed[2]) # replace two or more / with one url = urlparse.urlunparse(parsed) # request session proxies if sickbeard.PROXY_SETTING: logger.log("Using proxy for url: " + url, logger.DEBUG) session.proxies = { "http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING, } resp = session.get(url, data=post_data, timeout=timeout) except requests.exceptions.HTTPError, e: logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING) return
def download_file(url, filename, session=None): # create session session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(sickbeard.CACHE_DIR, 'sessions'))) # request session headers session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'}) # request session ssl verify session.verify = False # request session streaming session.stream = True # request session proxies if sickbeard.PROXY_SETTING: logger.log("Using proxy for url: " + url, logger.DEBUG) session.proxies = { "http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING, } try: resp = session.get(url) if not resp.ok: return False with open(filename, 'wb') as fp: for chunk in resp.iter_content(chunk_size=1024): if chunk: fp.write(chunk) fp.flush() chmodAsParent(filename) except requests.exceptions.HTTPError, e: _remove_file_failed(filename) logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING) return False
def urls(self): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) headers = {} tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json')) token = tokens.get(self.name) try: response = s.get(self.url, allow_redirects=True) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection") if not response.ok: return None response.encoding = 'utf-8' try: resource = response.json() resource['_response'] = response except ValueError: return None return resource
def get(self, key): s = requests.session() s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache'))) url = "%s/%s" % (self.url, key) headers = {} tokens = JsonFile(os.path.expanduser('~/.tst/tokens.json')) token = tokens.get(self.name) if token: headers['Authorization'] = 'Bearer %s' % token try: response = s.get(url, headers=headers, allow_redirects=True) except requests.ConnectionError: _assert(False, "Connection failed... check your internet connection") if not response.ok: self.last_error = response.status_code self.last_response = response return None response.encoding = 'utf-8' try: resource = response.json() resource['_response'] = response validate_tst_object(resource) except ValueError: #_assert(False, "Resource is not valid json") return None except AssertionError as e: print(resource) _assert(False, "Not a TST Object: %s" % e.message) return resource
class Scrapper: scrape_delay = 1 _requests = requests.session() _last_scrape = 0 def __init__(self): try: from cachecontrol import CacheControl from cachecontrol.caches import FileCache import tempfile self._requests = CacheControl(self._requests, cache=FileCache(tempfile.gettempdir()+'/cagematch-cache', forever=True)) except: logging.warning('CacheControl not available') self._requests.headers.update({'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'}) def get(self, url, **kwargs): if self._last_scrape: sleeptime = max(0, self.scrape_delay - (time.time() - self._last_scrape)) time.sleep(sleeptime) self._last_scrape = time.time() return self._requests.get(url, **kwargs)
class TestPermanentRedirects(object): def setup(self): self.sess = CacheControl(requests.Session()) def test_redirect_response_is_cached(self, url): self.sess.get(url + 'permanent_redirect', allow_redirects=False) resp = self.sess.get(url + 'permanent_redirect', allow_redirects=False) assert resp.from_cache def test_bust_cache_on_redirect(self, url): self.sess.get(url + 'permanent_redirect', allow_redirects=False) resp = self.sess.get(url + 'permanent_redirect', headers={'cache-control': 'no-cache'}, allow_redirects=False) assert not resp.from_cache
def test_max_bytes(self, tmpdir, sess): """ Test that caches the first url but not the second because the maximum bytes have been reached for the cache. """ # use a cache with max_bytes set max_bytes = 1400 self.cache = FileCache(str(tmpdir), max_bytes=max_bytes) sess = CacheControl(requests.Session(), cache=self.cache) url1 = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4))) url2 = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4))) assert url1 != url2 # fill up the cache with url1 response = sess.get(url1) assert not response.from_cache # make sure it got into the cache response = sess.get(url1) assert response.from_cache # do url2 now response = sess.get(url2) assert not response.from_cache # make sure url2 was NOT cached response = sess.get(url2) assert not response.from_cache # clear the cache response = sess.delete(url1) assert not response.from_cache # re-add to cache since bytes should be back to 0 response = sess.get(url1) assert not response.from_cache # verify from cache again response = sess.get(url1) assert response.from_cache
class TestMultipleChoicesRedirects(object): def setup(self): self.sess = CacheControl(requests.Session()) def test_multiple_choices_is_cacheable(self, url): self.sess.get(url + "multiple_choices_redirect", allow_redirects=False) resp = self.sess.get(url + "multiple_choices_redirect", allow_redirects=False) assert resp.from_cache def test_bust_cache_on_redirect(self, url): self.sess.get(url + "multiple_choices_redirect", allow_redirects=False) resp = self.sess.get( url + "multiple_choices_redirect", headers={"cache-control": "no-cache"}, allow_redirects=False, ) assert not resp.from_cache
def test_file_cache_recognizes_consumed_file_handle(self): s = CacheControl(Session(), FileCache('web_cache')) s.get('http://httpbin.org/cache/60') r = s.get('http://httpbin.org/cache/60') assert r.from_cache
class SpotifyPlugin(object): def __init__(self): self.client = None self.server = None self.play_lock = Semaphore(1) self.start_lock = Semaphore(1) self.start_marker = Event() self.last_track_uri = None self.last_track_object = None Dict.Reset() Dict['play_count'] = 0 Dict['last_restart'] = 0 Dict['schedule_restart_each'] = 5*60 # restart each X minutes Dict['play_restart_each'] = 2 # restart each X plays Dict['check_restart_each'] = 5 # check if I should restart each X seconds Dict['radio_salt'] = False # Saves last radio salt so multiple queries return the same radio track list self.start() self.session = requests.session() self.session_cached = CacheControl(self.session) Thread.CreateTimer(Dict['check_restart_each'], self.check_automatic_restart, globalize=True) @property def username(self): return Prefs["username"] @property def password(self): return Prefs["password"] def check_automatic_restart(self): can_restart = False try: diff = time.time() - Dict['last_restart'] scheduled_restart = diff >= Dict['schedule_restart_each'] play_count_restart = Dict['play_count'] >= Dict['play_restart_each'] must_restart = play_count_restart or scheduled_restart if must_restart: can_restart = self.play_lock.acquire(blocking=False) if can_restart: Log.Debug('Automatic restart started') self.start() Log.Debug('Automatic restart finished') finally: if can_restart: self.play_lock.release() Thread.CreateTimer(Dict['check_restart_each'], self.check_automatic_restart, globalize=True) @check_restart def preferences_updated(self): """ Called when the user updates the plugin preferences""" self.start() # Trigger a client restart def start(self): """ Start the Spotify client and HTTP server """ if not self.username or not self.password: Log("Username or password not set: not logging in") return False can_start = self.start_lock.acquire(blocking=False) try: # If there is a start in process, just wait until it finishes, but don't raise another one if not can_start: Log.Debug("Start already in progress, waiting it finishes to return") self.start_lock.acquire() else: Log.Debug("Start triggered, entering private section") self.start_marker.clear() if self.client: self.client.restart(self.username, self.password) else: self.client = SpotifyClient(self.username, self.password) self.last_track_uri = None self.last_track_object = None Dict['play_count'] = 0 Dict['last_restart'] = time.time() self.start_marker.set() Log.Debug("Start finished, leaving private section") finally: self.start_lock.release() return self.client and self.client.is_logged_in() @check_restart def play(self, uri): Log('play(%s)' % repr(uri)) uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") track_url = None if not self.client.is_track_uri_valid(uri): Log("Play track callback invoked with invalid URI (%s). This is very bad :-(" % uri) track_url = "http://www.xamuel.com/blank-mp3-files/2sec.mp3" else: self.play_lock.acquire(blocking=True) try: track_url = self.get_track_url(uri) # If first request failed, trigger re-connection to spotify retry_num = 0 while not track_url and retry_num < 2: Log.Info('get_track_url (%s) failed, re-connecting to spotify...' % uri) time.sleep(retry_num*0.5) # Wait some time based on number of failures if self.start(): track_url = self.get_track_url(uri) retry_num = retry_num + 1 if track_url == False or track_url is None: # Send an empty and short mp3 so player do not fail and we can go on listening next song Log.Error("Play track (%s) couldn't be obtained. This is very bad :-(" % uri) track_url = 'http://www.xamuel.com/blank-mp3-files/2sec.mp3' elif retry_num == 0: # If I didn't restart, add 1 to playcount Dict['play_count'] = Dict['play_count'] + 1 finally: self.play_lock.release() return Redirect(track_url) def get_track_url(self, track_uri): if not self.client.is_track_uri_valid(track_uri): return None track_url = None track = self.client.get(track_uri) if track: track_url = track.getFileURL(urlOnly=True, retries=1) return track_url # # TRACK DETAIL # @check_restart def metadata(self, uri): Log('metadata(%s)' % repr(uri)) uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") oc = ObjectContainer() track_object = None if not self.client.is_track_uri_valid(uri): Log("Metadata callback invoked with invalid URI (%s)" % uri) track_object = self.create_track_object_empty(uri) else: if self.last_track_uri == uri: track_object = self.last_track_object else: track_metadata = self.get_track_metadata(uri) if track_metadata: track_object = self.create_track_object_from_metatada(track_metadata) self.last_track_uri = uri self.last_track_object = track_object else: track_object = self.create_track_object_empty(uri) oc.add(track_object) return oc def get_track_metadata(self, track_uri): if not self.client.is_track_uri_valid(track_uri): return None track = self.client.get(track_uri) if not track: return None #track_uri = track.getURI().decode("utf-8") title = track.getName().decode("utf-8") image_url = self.select_image(track.getAlbumCovers()) track_duration = int(track.getDuration()) track_number = int(track.getNumber()) track_album = track.getAlbum(nameOnly=True).decode("utf-8") track_artists = track.getArtists(nameOnly=True).decode("utf-8") metadata = TrackMetadata(title, image_url, track_uri, track_duration, track_number, track_album, track_artists) return metadata @staticmethod def select_image(images): if images == None: return None if images.get(640): return images[640] elif images.get(320): return images[320] elif images.get(300): return images[300] elif images.get(160): return images[160] elif images.get(60): return images[60] Log.Info('Unable to select image, available sizes: %s' % images.keys()) return None def get_uri_image(self, uri): images = None obj = self.client.get(uri) if isinstance(obj, SpotifyArtist): images = obj.getPortraits() elif isinstance(obj, SpotifyAlbum): images = obj.getCovers() elif isinstance(obj, SpotifyTrack): images = obj.getAlbum().getCovers() elif isinstance(obj, SpotifyPlaylist): images = obj.getImages() return self.select_image(images) @authenticated @check_restart def image(self, uri): if not uri: # TODO media specific placeholders return Redirect(R('placeholder-artist.png')) Log.Debug('Getting image for: %s' % uri) uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") if uri.startswith('spotify:'): # Fetch object for spotify URI and select image image_url = self.get_uri_image(uri) if not image_url: # TODO media specific placeholders return Redirect(R('placeholder-artist.png')) else: # pre-selected image provided Log.Debug('Using pre-selected image URL: "%s"' % uri) image_url = uri return self.session_cached.get(image_url).content # # SECOND_LEVEL_MENU # @authenticated @check_restart def explore(self): Log("explore") """ Explore shared music """ return ObjectContainer( objects=[ DirectoryObject( key=route_path('explore/featured_playlists'), title=L("MENU_FEATURED_PLAYLISTS"), thumb=R("icon-explore-featuredplaylists.png") ), DirectoryObject( key=route_path('explore/top_playlists'), title=L("MENU_TOP_PLAYLISTS"), thumb=R("icon-explore-topplaylists.png") ), DirectoryObject( key=route_path('explore/new_releases'), title=L("MENU_NEW_RELEASES"), thumb=R("icon-explore-newreleases.png") ), DirectoryObject( key=route_path('explore/genres'), title=L("MENU_GENRES"), thumb=R("icon-explore-genres.png") ) ], ) @authenticated @check_restart def discover(self): Log("discover") oc = ObjectContainer( title2=L("MENU_DISCOVER"), view_group=ViewMode.Stories ) stories = self.client.discover() for story in stories: self.add_story_to_directory(story, oc) return oc @authenticated @check_restart def radio(self): Log("radio") """ Show radio options """ return ObjectContainer( objects=[ DirectoryObject( key=route_path('radio/stations'), title=L("MENU_RADIO_STATIONS"), thumb=R("icon-radio-stations.png") ), DirectoryObject( key=route_path('radio/genres'), title=L("MENU_RADIO_GENRES"), thumb=R("icon-radio-genres.png") ) ], ) @authenticated @check_restart def your_music(self): Log("your_music") """ Explore your music """ return ObjectContainer( objects=[ DirectoryObject( key=route_path('your_music/playlists'), title=L("MENU_PLAYLISTS"), thumb=R("icon-playlists.png") ), DirectoryObject( key=route_path('your_music/starred'), title=L("MENU_STARRED"), thumb=R("icon-starred.png") ), DirectoryObject( key=route_path('your_music/albums'), title=L("MENU_ALBUMS"), thumb=R("icon-albums.png") ), DirectoryObject( key=route_path('your_music/artists'), title=L("MENU_ARTISTS"), thumb=R("icon-artists.png") ), ], ) # # EXPLORE # @authenticated @check_restart def featured_playlists(self): Log("featured playlists") oc = ObjectContainer( title2=L("MENU_FEATURED_PLAYLISTS"), content=ContainerContent.Playlists, view_group=ViewMode.Playlists ) playlists = self.client.get_featured_playlists() for playlist in playlists: self.add_playlist_to_directory(playlist, oc) return oc @authenticated @check_restart def top_playlists(self): Log("top playlists") oc = ObjectContainer( title2=L("MENU_TOP_PLAYLISTS"), content=ContainerContent.Playlists, view_group=ViewMode.Playlists ) playlists = self.client.get_top_playlists() for playlist in playlists: self.add_playlist_to_directory(playlist, oc) return oc @authenticated @check_restart def new_releases(self): Log("new releases") oc = ObjectContainer( title2=L("MENU_NEW_RELEASES"), content=ContainerContent.Albums, view_group=ViewMode.Albums ) albums = self.client.get_new_releases() for album in albums: self.add_album_to_directory(album, oc) return oc @authenticated @check_restart def genres(self): Log("genres") oc = ObjectContainer( title2=L("MENU_GENRES"), content=ContainerContent.Playlists, view_group=ViewMode.Playlists ) genres = self.client.get_genres() for genre in genres: self.add_genre_to_directory(genre, oc) return oc @authenticated @check_restart def genre_playlists(self, genre_name): Log("genre playlists") oc = ObjectContainer( title2=genre_name, content=ContainerContent.Playlists, view_group=ViewMode.Playlists ) playlists = self.client.get_playlists_by_genre(genre_name) for playlist in playlists: self.add_playlist_to_directory(playlist, oc) return oc # # RADIO # @authenticated @check_restart def radio_stations(self): Log('radio stations') Dict['radio_salt'] = False oc = ObjectContainer(title2=L("MENU_RADIO_STATIONS")) stations = self.client.get_radio_stations() for station in stations: oc.add(PopupDirectoryObject( key=route_path('radio/stations/' + station.getURI()), title=station.getTitle(), thumb=function_path('image.png', uri=self.select_image(station.getImages())) )) return oc @authenticated @check_restart def radio_genres(self): Log('radio genres') Dict['radio_salt'] = False oc = ObjectContainer(title2=L("MENU_RADIO_GENRES")) genres = self.client.get_radio_genres() for genre in genres: oc.add(PopupDirectoryObject( key=route_path('radio/genres/' + genre.getURI()), title=genre.getTitle(), thumb=function_path('image.png', uri=self.select_image(genre.getImages())) )) return oc @authenticated @check_restart def radio_track_num(self, uri): Log('radio track num') uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") return ObjectContainer( title2=L("MENU_RADIO_TRACK_NUM"), objects=[ DirectoryObject( key=route_path('radio/play/' + uri + '/10'), title=localized_format("MENU_TRACK_NUM", "10"), thumb=R("icon-radio-item.png") ), DirectoryObject( key=route_path('radio/play/' + uri + '/20'), title=localized_format("MENU_TRACK_NUM", "20"), thumb=R("icon-radio-item.png") ), DirectoryObject( key=route_path('radio/play/' + uri + '/50'), title=localized_format("MENU_TRACK_NUM", "50"), thumb=R("icon-radio-item.png") ), DirectoryObject( key=route_path('radio/play/' + uri + '/80'), title=localized_format("MENU_TRACK_NUM", "80"), thumb=R("icon-radio-item.png") ), DirectoryObject( key=route_path('radio/play/' + uri + '/100'), title=localized_format("MENU_TRACK_NUM", "100"), thumb=R("icon-radio-item.png") ) ], ) @authenticated @check_restart def radio_tracks(self, uri, num_tracks): Log('radio tracks') uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") oc = None radio = self.client.get_radio(uri) if not Dict['radio_salt']: Dict['radio_salt'] = radio.generateSalt() salt = Dict['radio_salt'] tracks = radio.getTracks(salt=salt, num_tracks=int(num_tracks)) oc = ObjectContainer( title2 = radio.getTitle().decode("utf-8"), content = ContainerContent.Tracks, view_group = ViewMode.Tracks ) for track in tracks: self.add_track_to_directory(track, oc) return oc # # YOUR_MUSIC # @authenticated @check_restart def playlists(self): Log("playlists") oc = ObjectContainer( title2=L("MENU_PLAYLISTS"), content=ContainerContent.Playlists, view_group=ViewMode.Playlists ) playlists = self.client.get_playlists() for playlist in playlists: self.add_playlist_to_directory(playlist, oc) return oc @authenticated @check_restart def starred(self): Log("starred") oc = ObjectContainer( title2=L("MENU_STARRED"), content=ContainerContent.Tracks, view_group=ViewMode.Tracks ) starred = self.client.get_starred() for x, track in enumerate(starred.getTracks()): self.add_track_to_directory(track, oc, index=x) return oc @authenticated @check_restart def albums(self): Log("albums") oc = ObjectContainer( title2=L("MENU_ALBUMS"), content=ContainerContent.Albums, view_group=ViewMode.Albums ) albums = self.client.get_my_albums() for album in albums: self.add_album_to_directory(album, oc) return oc @authenticated @check_restart def artists(self): Log("artists") oc = ObjectContainer( title2=L("MENU_ARTISTS"), content=ContainerContent.Artists, view_group=ViewMode.Artists ) artists = self.client.get_my_artists() for artist in artists: self.add_artist_to_directory(artist, oc) return oc # # ARTIST DETAIL # @authenticated @check_restart def artist(self, uri): Log("artist") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") artist = self.client.get(uri) return ObjectContainer( title2=artist.getName().decode("utf-8"), objects=[ DirectoryObject( key = route_path('artist/%s/top_tracks' % uri), title=L("MENU_TOP_TRACKS"), thumb=R("icon-artist-toptracks.png") ), DirectoryObject( key = route_path('artist/%s/albums' % uri), title =L("MENU_ALBUMS"), thumb =R("icon-albums.png") ), DirectoryObject( key = route_path('artist/%s/related' % uri), title =L("MENU_RELATED"), thumb =R("icon-artist-related.png") ), DirectoryObject( key=route_path('radio/stations/' + uri), title =L("MENU_RADIO"), thumb =R("icon-radio-custom.png") ) ], ) @authenticated @check_restart def artist_albums(self, uri): Log("artist_albums") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") artist = self.client.get(uri) oc = ObjectContainer( title2=artist.getName().decode("utf-8"), content=ContainerContent.Albums ) for album in artist.getAlbums(): self.add_album_to_directory(album, oc) return oc @authenticated @check_restart def artist_top_tracks(self, uri): Log("artist_top_tracks") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") oc = None artist = self.client.get(uri) top_tracks = artist.getTracks() if top_tracks: oc = ObjectContainer( title2=artist.getName().decode("utf-8"), content=ContainerContent.Tracks, view_group=ViewMode.Tracks ) for track in artist.getTracks(): self.add_track_to_directory(track, oc) else: oc = MessageContainer( header=L("MSG_TITLE_NO_RESULTS"), message=localized_format("MSG_FMT_NO_RESULTS", artist.getName().decode("utf-8")) ) return oc @authenticated @check_restart def artist_related(self, uri): Log("artist_related") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") artist = self.client.get(uri) oc = ObjectContainer( title2=localized_format("MSG_RELATED_TO", artist.getName().decode("utf-8")), content=ContainerContent.Artists ) for artist in artist.getRelatedArtists(): self.add_artist_to_directory(artist, oc) return oc # # ALBUM DETAIL # @authenticated @check_restart def album(self, uri): Log("album") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") album = self.client.get(uri) oc = ObjectContainer( title2=album.getName().decode("utf-8"), content=ContainerContent.Artists ) oc.add(DirectoryObject( key = route_path('album/%s/tracks' % uri), title=L("MENU_ALBUM_TRACKS"), thumb=R("icon-album-tracks.png"))) artists = album.getArtists() for artist in artists: self.add_artist_to_directory(artist, oc) oc.add(DirectoryObject( key=route_path('radio/stations/' + uri), title =L("MENU_RADIO"), thumb =R("icon-radio-custom.png"))) return oc @authenticated @check_restart def album_tracks(self, uri): Log("album_tracks") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") album = self.client.get(uri) oc = ObjectContainer( title2=album.getName().decode("utf-8"), content=ContainerContent.Tracks, view_group=ViewMode.Tracks ) for track in album.getTracks(): self.add_track_to_directory(track, oc) return oc # # PLAYLIST DETAIL # @authenticated @check_restart def playlist(self, uri): Log("playlist") uri = urllib.quote(uri.encode("utf8")).replace("%3A", ":").decode("utf8") pl = self.client.get(uri) if pl is None: # Unable to find playlist return MessageContainer( header=L("MSG_TITLE_UNKNOWN_PLAYLIST"), message='URI: %s' % uri ) Log("Get playlist: %s", pl.getName().decode("utf-8")) Log.Debug('playlist truncated: %s', pl.obj.contents.truncated) oc = ObjectContainer( title2=pl.getName().decode("utf-8"), content=ContainerContent.Tracks, view_group=ViewMode.Tracks, mixed_parents=True ) for x, track in enumerate(pl.getTracks()): self.add_track_to_directory(track, oc, index=x) return oc # # MAIN MENU # def main_menu(self): Log("main_menu") return ObjectContainer( objects=[ InputDirectoryObject( key=route_path('search'), prompt=L("PROMPT_SEARCH"), title=L("MENU_SEARCH"), thumb=R("icon-search.png") ), DirectoryObject( key=route_path('explore'), title=L("MENU_EXPLORE"), thumb=R("icon-explore.png") ), DirectoryObject( key=route_path('discover'), title=L("MENU_DISCOVER"), thumb=R("icon-discover.png") ), DirectoryObject( key=route_path('radio'), title=L("MENU_RADIO"), thumb=R("icon-radio.png") ), DirectoryObject( key=route_path('your_music'), title=L("MENU_YOUR_MUSIC"), thumb=R("icon-yourmusic.png") ), PrefsObject( title=L("MENU_PREFS"), thumb=R("icon-preferences.png") ) ], ) # # Create objects # def create_track_object_from_track(self, track, index=None): if not track: return None # Get metadata info track_uri = track.getURI() title = track.getName().decode("utf-8") image_url = self.select_image(track.getAlbumCovers()) track_duration = int(track.getDuration()) - 500 track_number = int(track.getNumber()) track_album = track.getAlbum(nameOnly=True).decode("utf-8") track_artists = track.getArtists(nameOnly=True).decode("utf-8") metadata = TrackMetadata(title, image_url, track_uri, track_duration, track_number, track_album, track_artists) return self.create_track_object_from_metatada(metadata, index=index) def create_track_object_from_metatada(self, metadata, index=None): if not metadata: return None return self.create_track_object(metadata.uri, metadata.duration, metadata.title, metadata.album, metadata.artists, metadata.number, metadata.image_url, index) def create_track_object_empty(self, uri): if not uri: return None return self.create_track_object(uri, -1, "", "", "", 0, None) def create_track_object(self, uri, duration, title, album, artists, track_number, image_url, index=None): rating_key = uri if index is not None: rating_key = '%s::%s' % (uri, index) art_num = str(randint(1,40)).rjust(2, "0") track_obj = TrackObject( items=[ MediaObject( parts=[PartObject(key=route_path('play/%s' % uri))], duration=duration, container=Container.MP3, audio_codec=AudioCodec.MP3, audio_channels = 2 ) ], key = route_path('metadata', uri), rating_key = rating_key, title = title, album = album, artist = artists, index = index if index != None else track_number, duration = duration, source_title='Spotify', art = R('art-' + art_num + '.png'), thumb = function_path('image.png', uri=image_url) ) Log.Debug('New track object for metadata: --|%s|%s|%s|%s|%s|%s|--' % (image_url, uri, str(duration), str(track_number), album, artists)) return track_obj def create_album_object(self, album, custom_summary=None, custom_image_url=None): """ Factory method for album objects """ title = album.getName().decode("utf-8") if Prefs["displayAlbumYear"] and album.getYear() != 0: title = "%s (%s)" % (title, album.getYear()) artist_name = album.getArtists(nameOnly=True).decode("utf-8") summary = '' if custom_summary == None else custom_summary.decode('utf-8') image_url = self.select_image(album.getCovers()) if custom_image_url == None else custom_image_url return DirectoryObject( key=route_path('album', album.getURI()), title=title + " - " + artist_name, tagline=artist_name, summary=summary, art=function_path('image.png', uri=image_url), thumb=function_path('image.png', uri=image_url), ) #return AlbumObject( # key=route_path('album', album.getURI().decode("utf-8")), # rating_key=album.getURI().decode("utf-8"), # # title=title, # artist=artist_name, # summary=summary, # # track_count=album.getNumTracks(), # source_title='Spotify', # # art=function_path('image.png', uri=image_url), # thumb=function_path('image.png', uri=image_url), #) def create_playlist_object(self, playlist): uri = playlist.getURI() image_url = self.select_image(playlist.getImages()) artist = playlist.getUsername().decode('utf8') title = playlist.getName().decode("utf-8") summary = '' if playlist.getDescription() != None and len(playlist.getDescription()) > 0: summary = playlist.getDescription().decode("utf-8") return DirectoryObject( key=route_path('playlist', uri), title=title + " - " + artist, tagline=artist, summary=summary, art=function_path('image.png', uri=image_url) if image_url != None else R("placeholder-playlist.png"), thumb=function_path('image.png', uri=image_url) if image_url != None else R("placeholder-playlist.png") ) #return AlbumObject( # key=route_path('playlist', uri), # rating_key=uri, # # title=title, # artist=artist, # summary=summary, # # source_title='Spotify', # # art=function_path('image.png', uri=image_url) if image_url != None else R("placeholder-playlist.png"), # thumb=function_path('image.png', uri=image_url) if image_url != None else R("placeholder-playlist.png") #) def create_genre_object(self, genre): uri = genre.getTemplateName() title = genre.getName().decode("utf-8") image_url = genre.getIconUrl() return DirectoryObject( key=route_path('genre', uri), title=title, art=function_path('image.png', uri=image_url) if image_url != None else R("placeholder-playlist.png"), thumb=function_path('image.png', uri=image_url) if image_url != None else R("placeholder-playlist.png") ) def create_artist_object(self, artist, custom_summary=None, custom_image_url=None): image_url = self.select_image(artist.getPortraits()) if custom_image_url == None else custom_image_url artist_name = artist.getName().decode("utf-8") summary = '' if custom_summary == None else custom_summary.decode('utf-8') return DirectoryObject( key=route_path('artist', artist.getURI()), title=artist_name, summary=summary, art=function_path('image.png', uri=image_url), thumb=function_path('image.png', uri=image_url) ) #return ArtistObject( # key=route_path('artist', artist.getURI().decode("utf-8")), # rating_key=artist.getURI().decode("utf-8"), # # title=artist_name, # summary=summary, # source_title='Spotify', # # art=function_path('image.png', uri=image_url), # thumb=function_path('image.png', uri=image_url) # ) # # Insert objects into container # def add_section_header(self, title, oc): oc.add( DirectoryObject( key='', title=title ) ) def add_track_to_directory(self, track, oc, index = None): if not self.client.is_track_playable(track): Log("Ignoring unplayable track: %s" % track.getName()) return track_uri = track.getURI().decode("utf-8") if not self.client.is_track_uri_valid(track_uri): Log("Ignoring unplayable track: %s, invalid uri: %s" % (track.getName(), track_uri)) return oc.add(self.create_track_object_from_track(track, index=index)) def add_album_to_directory(self, album, oc, custom_summary=None, custom_image_url=None): if not self.client.is_album_playable(album): Log("Ignoring unplayable album: %s" % album.getName()) return oc.add(self.create_album_object(album, custom_summary=custom_summary, custom_image_url=custom_image_url)) def add_artist_to_directory(self, artist, oc, custom_summary=None, custom_image_url=None): oc.add(self.create_artist_object(artist, custom_summary=custom_summary, custom_image_url=custom_image_url)) def add_playlist_to_directory(self, playlist, oc): oc.add(self.create_playlist_object(playlist)) def add_genre_to_directory(self, genre, oc): oc.add(self.create_genre_object(genre)) def add_story_to_directory(self, story, oc): content_type = story.getContentType() image_url = self.select_image(story.getImages()) item = story.getObject() if content_type == 'artist': self.add_artist_to_directory(item, oc, custom_summary=story.getDescription(), custom_image_url=image_url) elif content_type == 'album': self.add_album_to_directory(item, oc, custom_summary=story.getDescription(), custom_image_url=image_url) elif content_type == 'track': self.add_album_to_directory(item.getAlbum(), oc, custom_summary=story.getDescription() + " - " + item.getName(), custom_image_url=image_url)