def __init__(self, *args, **kwargs): super(YoutubeHandler, self).__init__(*args, **kwargs) if not self.api_key: raise ApiKeyMissing() self.session = Session()
def do_shorten(self, context): session = Session() params = {"url": unicode(context["url"])} d = session.get(self.base_url, params=params) d.addCallbacks(self.shorten_success, self.shorten_error) return d
def do_shorten(self, context): session = Session() params = {"url": unicode(context["url"])} d = session.get(self.base_url, params=params) d.addCallbacks( self.shorten_success, self.shorten_error ) return d
def test_max_workers(self): """ Tests the `max_workers` shortcut. """ from twisted.python.threadpool import ThreadPool with Session() as session: self.assertEqual(session.pool.max, 4) with Session(maxthreads=5) as session: self.assertEqual(session.pool.max, 5) with Session(pool=ThreadPool(maxthreads=10)) as session: self.assertEqual(session.pool.max, 10) with Session(pool=ThreadPool(maxthreads=10), maxthreads=5) as session: self.assertEqual(session.pool.max, 10)
class LazyRequest(object): result = None _args = [] _kwargs = {} def __init__(self, pool=None, minthreads=1, maxthreads=4, req_args=None, req_kwargs=None, session_kwargs=None): if not req_args: req_args = [] if not req_kwargs: req_kwargs = {} if not session_kwargs: session_kwargs = {} self._args = req_args self._kwargs = req_kwargs self._session = Session( pool=pool, minthreads=minthreads, maxthreads=maxthreads, **session_kwargs ) def get(self): if self.result is None: self.result = self._session.get(*self._args, **self._kwargs) del self._session return self.result
class LazyRequest(object): result = None _args = [] _kwargs = {} def __init__(self, pool=None, minthreads=1, maxthreads=4, req_args=None, req_kwargs=None, session_kwargs=None): if not req_args: req_args = [] if not req_kwargs: req_kwargs = {} if not session_kwargs: session_kwargs = {} self._args = req_args self._kwargs = req_kwargs self._session = Session(pool=pool, minthreads=minthreads, maxthreads=maxthreads, **session_kwargs) def get(self): if self.result is None: self.result = self._session.get(*self._args, **self._kwargs) del self._session return self.result
def test_custom_requests_session(self): from txrequests import Session with Session() as sess: sess.headers["user-agent"] = "spotipy-test" with_custom_session = spotipy_twisted.Spotify( requests_session=sess) user = yield with_custom_session.user(user="******") self.assertTrue(user["uri"] == "spotify:user:akx")
class Crawler: def __init__(self, max_concurrent_requests): self.session = Session(maxthreads=max_concurrent_requests) def add_website_request(self, website: Website, callback: Callable[[Session, requests.Response], requests.Response], query_params: dict, data: dict): self.add_request(website.method, website.api_url, callback, query_params, data) def add_request(self, method: str, url: str, callback: Callable[[Session, requests.Response], requests.Response], query_params: dict, data: dict): logging.debug(f"{method} {url} [params: {query_params}, data: {data}]") self.session.request(method=method, url=url, params=query_params, #data=data, background_callback=callback) def add_cookies(self, cookies: CookieJar): self.session.cookies.update(cookies) def stop(self): self.session.close()
def new_instance(cls, enabled=None): """Initialize an instance using values from the configuration""" session = Session() if enabled is None: enabled = conf.settings['share_usage_data'] return cls( session, conf.settings['ANALYTICS_ENDPOINT'], utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']), enabled, )
def test_redirect(self): """ Tests for the ability to cleanly handle redirects. """ with Session() as sess: d = sess.get(httpbin('redirect-to?url=get')) resp = yield d self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) d = sess.get(httpbin('redirect-to?url=status/404')) resp = yield d self.assertEqual(404, resp.status_code)
def __init__(self, pool=None, minthreads=1, maxthreads=4, req_args=None, req_kwargs=None, session_kwargs=None): if not req_args: req_args = [] if not req_kwargs: req_kwargs = {} if not session_kwargs: session_kwargs = {} self._args = req_args self._kwargs = req_kwargs self._session = Session(pool=pool, minthreads=minthreads, maxthreads=maxthreads, **session_kwargs)
def __init__(self, url, fqdn=False, localname=None, facility=None, session=None): logging.Handler.__init__(self) self.url = url self.fqdn = fqdn self.localname = localname self.facility = facility self.session = session if session is not None else Session()
def do_request(): proxies = {} if proxy_url: proxies['http'] = proxies['https'] = proxy_url elif proxy_host: proxies['http'] = proxies['https'] = '{}:{}'.format( proxy_host, proxy_port) headers = kwargs.get('headers') body = kwargs.get('body') disable_tls_verification = kwargs.get('disable_tls_verification', False) allow_redirects = kwargs.get('allow_redirects', False) params = kwargs.get('params') cookies = kwargs.get('cookies') auth = kwargs.get('auth') digest_auth = kwargs.get('digest_auth') args = { 'method': method, 'url': url, 'verify': not disable_tls_verification, 'timeout': timeout, 'allow_redirects': allow_redirects, } if headers: args['headers'] = headers if body: args['data'] = body if proxies: args['proxies'] = proxies if params: args['params'] = params if cookies: args['cookies'] = cookies if auth: args['auth'] = auth if digest_auth: args['auth'] = HTTPDigestAuth(digest_auth) if disable_tls_verification: disable_warnings() with Session() as session: request = session.request(**args) response = yield request if response.status_code != expected_code: raise RuntimeError("Unexpected response code: {}".format( response.status_code))
def new_instance(cls, enabled=None): """Initialize an instance using values from the configuration""" # Session是requests库的twisted的异步版本 session = Session() if enabled is None: # 是否与LBRY共享使用统计信息和诊断信息。 enabled = conf.settings['share_usage_data'] return cls( session, # 下面这两个配置的值是https://segment.com/网站的api访问 # 此站是网站主上传用户数据后, 可提供200+的工具用于数据分析 conf.settings['ANALYTICS_ENDPOINT'], utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']), enabled, )
def __init__(self, pool=None, minthreads=1, maxthreads=4, req_args=None, req_kwargs=None, session_kwargs=None): if not req_args: req_args = [] if not req_kwargs: req_kwargs = {} if not session_kwargs: session_kwargs = {} self._args = req_args self._kwargs = req_kwargs self._session = Session( pool=pool, minthreads=minthreads, maxthreads=maxthreads, **session_kwargs )
def reload(self): self.teardown() self.group_sessions = {} self.resolver = AddressResolver() proxy = self.plugin.get_proxy() if not proxy: self.global_session = Session() else: self.global_session = ProxySession(proxy) try: self.global_session.cookies = self.get_cookie_jar("/global.txt") self.global_session.session_type = "global" self.global_session.cookies.set_mode( self.plugin.config.get("sessions", {}).get("cookies", {}).get("global", "discard")) except ValueError as e: self.urls_plugin.logger.error( "Failed to create global cookie jar: {0}".format(e))
def test_session(self): # basic futures get with Session() as sess: d = sess.get(httpbin('get')) self.assertIsInstance(d, defer.Deferred) resp = yield d self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) # non-200, 404 d = sess.get(httpbin('status/404')) resp = yield d self.assertEqual(404, resp.status_code) def cb(s, r): self.assertIsInstance(s, Session) self.assertIsInstance(r, Response) # add the parsed json data to the response r.data = r.json() return r d = sess.get(httpbin('get'), background_callback=cb) # this should block until complete resp = yield d self.assertEqual(200, resp.status_code) # make sure the callback was invoked self.assertTrue(hasattr(resp, 'data')) def rasing_cb(s, r): raise Exception('boom') d = sess.get(httpbin('get'), background_callback=rasing_cb) raised = False try: resp = yield d except Exception as e: self.assertEqual('boom', e.args[0]) raised = True self.assertTrue(raised)
class YoutubeHandler(URLHandler): name = "youtube" criteria = { "protocol": re.compile(r"http|https", re.I), "domain": re.compile(r"(www\.)?(youtube\.com|youtu\.be)", re.I), } VIDEO_LINK, CHANNEL_LINK, PLAYLIST_LINK = xrange(3) BASE_URL = "https://www.googleapis.com/youtube/v3/" VIDEOS_URL = BASE_URL + "videos" CHANNELS_URL = BASE_URL + "channels" PLAYLISTS_URL = BASE_URL + "playlists" DEFAULT_FORMATS = { "video": u'[YouTube Video] "{title}" by {channel} - {description} - ' u'length {duration_formatted} - rated {rating_percent:.0f}%' u' - {views} views', "channel": u'[YouTube Channel] {title} - {description} - {videos} ' u'videos - {subscribers} subscribers - {views} views', "playlist": u'[YouTube Playlist] "{title}" by {channel} - ' u'{description} - {videos} videos', } def __init__(self, *args, **kwargs): super(YoutubeHandler, self).__init__(*args, **kwargs) if not self.api_key: raise ApiKeyMissing() self.session = Session() @property def api_key(self): return self.plugin.config.get("youtube", {}).get("api_key", "") @property def api_key_referrer(self): youtube_conf = self.plugin.config.get("youtube", {}) return youtube_conf.get("api_key_referrer", "") @property def description_length(self): youtube_conf = self.plugin.config.get("youtube", {}) return youtube_conf.get("description_length", 75) def get_format_string(self, key): youtube_conf = self.plugin.config.get("youtube", {}) format_conf = youtube_conf.get("formatting", {}) if key not in format_conf: return self.DEFAULT_FORMATS[key] return format_conf[key] def call(self, url, context): domain = url.domain.lower() if domain.startswith(u"www."): domain = domain[4:] if domain == u"youtu.be": link_type, data = self._parse_youtu_be(url) else: link_type, data = self._parse_youtube_com(url) if link_type == self.VIDEO_LINK: self.handle_video(data, context) return STOP_HANDLING elif link_type == self.CHANNEL_LINK: self.handle_channel(data, context) return STOP_HANDLING elif link_type == self.PLAYLIST_LINK: self.handle_playlist(data, context) return STOP_HANDLING else: return CASCADE def _parse_youtu_be(self, url): return self.VIDEO_LINK, url.path.strip("/") def _parse_youtube_com(self, url): # Video: https://www.youtube.com/watch?v=orvJo3nNZuI # Channel: # Username: https://www.youtube.com/user/Mtvnoob # Channel ID: https://www.youtube.com/channel/UCmkoMt2VCc3TaFSE5MKrkpQ # Playlist: https://www.youtube.com/playlist?list=PLE6Wd9FR--EfW8dtjAuPoTuPcqmOV53Fu # noqa try: path_split = url.path.strip("/").split("/") root_path = path_split[0] if root_path == u"watch": return self.VIDEO_LINK, url.query["v"] elif root_path == u"user": return self.CHANNEL_LINK, {u"username": path_split[1]} elif root_path == u"channel": return self.CHANNEL_LINK, {u"channel_id": path_split[1]} elif root_path == u"playlist": return self.PLAYLIST_LINK, url.query[u"list"] except Exception: self.plugin.logger.exception("Error parsing youtube.com URL") return None, None def _get(self, url, params, **kwargs): referrer = self.api_key_referrer if referrer: headers = {"referer": referrer} if "headers" in kwargs: headers.update(kwargs["headers"]) kwargs["headers"] = headers params["key"] = self.api_key return self.session.get(url, params=params, **kwargs) def handle_video(self, video_id, context): req_def = self._get(self.VIDEOS_URL, params={ "part": "snippet,contentDetails,statistics", "id": video_id, }) return self._add_callbacks(self._handle_video_response, self._handle_request_failure, context, req_def) def handle_channel(self, data, context): params = { "part": "snippet,statistics", } if "channel_id" in data: params["id"] = data["channel_id"] elif "username" in data: params["forUsername"] = data["username"] else: raise ValueError("Must specify channel_id or username") req_def = self._get(self.CHANNELS_URL, params=params) return self._add_callbacks(self._handle_channel_response, self._handle_request_failure, context, req_def) def handle_playlist(self, playlist_id, context): req_def = self._get(self.PLAYLISTS_URL, params={ "part": "snippet,contentDetails", "id": playlist_id, }) return self._add_callbacks(self._handle_playlist_response, self._handle_request_failure, context, req_def) def _add_callbacks(self, callback, errback, context, req_def): result_def = Deferred() req_def.addCallback(callback, context, result_def) req_def.addErrback(errback, context, result_def) return result_def def _handle_video_response(self, response, context, result_def): data = response.json() items = self._get_items(data) content_details = items["contentDetails"] snippet = items["snippet"] statistics = items["statistics"] description = snippet["description"].strip() if len(description) == 0: description = "No description" description_snippet = self.snip_description(description) duration = isodate.parse_duration(content_details["duration"]) likes_count = int(statistics["likeCount"]) dislike_count = int(statistics["dislikeCount"]) ratings_total = likes_count + dislike_count rating_percentage = (float(likes_count) / ratings_total) * 100 tags = snippet.get("tags", []) if len(tags) > 0: tags_formatted = ", ".join(tags[:5]) else: tags_formatted = "No tags" duration_formatted = self.format_time_period(duration) format_data = { "full_response": data, "title": snippet["title"], "channel": snippet["channelTitle"], "duration": duration, "duration_formatted": duration_formatted, "description": description_snippet, "full_description": description, "tags": tags, "tags_formatted": tags_formatted, "likes": likes_count, "dislikes": dislike_count, "favourites": int(statistics["favoriteCount"]), "views": int(statistics["viewCount"]), "comments": int(statistics["commentCount"]), "rating_percent": rating_percentage, "rating_total": ratings_total } message = self.get_format_string("video").format(**format_data) self._handle_message(message, context) result_def.callback(STOP_HANDLING) def _handle_channel_response(self, response, context, result_def): data = response.json() items = self._get_items(data) snippet = items["snippet"] statistics = items["statistics"] description = snippet["description"] if len(description) == 0: description = "No description" description_snippet = self.snip_description(description) try: # I'm not sure what happens here if hiddenSubscriberCount is true subscribers = int(statistics["subscriberCount"]) except ValueError: subscribers = 0 hidden_subscribers = statistics["hiddenSubscriberCount"] # noqa country = snippet.get("country", "Unknown") format_data = { "full_response": data, "title": snippet["title"], "subscribers": subscribers, "videos": statistics["videoCount"], "views": statistics["viewCount"], "comments": statistics["commentCount"], "country": country, "description": description_snippet, "full_description": description, } message = self.get_format_string("channel").format(**format_data) self._handle_message(message, context) result_def.callback(STOP_HANDLING) def _handle_playlist_response(self, response, context, result_def): data = response.json() items = self._get_items(data) content_details = items["contentDetails"] snippet = items["snippet"] description = snippet["description"].strip() if len(description) == 0: description = "No description" description_snippet = self.snip_description(description) format_data = { "full_response": data, "title": snippet["title"], "channel": snippet["channelTitle"], "videos": content_details["itemCount"], "description": description_snippet, "full_description": description, } message = self.get_format_string("playlist").format(**format_data) self._handle_message(message, context) result_def.callback(STOP_HANDLING) def _get_items(self, data): if "error" in data: error = data["error"] raise YoutubeAPIError( error["message"], error["code"], error["errors"]) try: return data["items"][0] except LookupError: raise YoutubeMissingItemError() def _handle_request_failure(self, fail, context, result_def): if fail.check(YoutubeAPIError): self.plugin.logger.error(fail.getErrorMessage()) elif fail.check(YoutubeMissingItemError): # It's a 404, basically, so don't bother with a title result_def.callback(STOP_HANDLING) return else: self.plugin.logger.error(fail.getTraceback()) result_def.callback(CASCADE) def _handle_message(self, message, context): context["event"].target.respond(message) def reload(self): self.teardown() self.session = Session() def teardown(self): if self.session is not None: self.session.close() def format_time_period(self, duration): secs = duration.total_seconds() m, s = divmod(secs, 60) if m >= 60: h, m = divmod(m, 60) return "%d:%02d:%02d" % (h, m, s) else: return "%d:%02d" % (m, s) def snip_description(self, description, length=0): if not length: length = self.description_length split = description.strip().split(u"\n") desc = split[0].strip() if len(desc) > length: return desc[:length - 3].strip() + u"..." return desc
def reload(self): self.teardown() self.session = Session()
class OsuHandler(URLHandler): criteria = { "protocol": re.compile(r"http|https", str_to_regex_flags("iu")), "domain": re.compile(r"osu\.ppy\.sh", str_to_regex_flags("iu")) } session = None name = "osu" @property def api_key(self): return self.plugin.config.get("osu", {}).get("api_key", "") def __init__(self, plugin): super(OsuHandler, self).__init__(plugin) if not self.api_key: raise ApiKeyMissing() self.reload() def reload(self): self.teardown() self.session = Session() def teardown(self): if self.session is not None: self.session.close() def get_string(self, string): formatting = self.plugin.config.get("osu", {}).get("formatting", {}) if string not in formatting: return strings[string] return formatting[string] @inlineCallbacks def get(self, *args, **kwargs): params = kwargs.get("params", {}) kwargs["params"] = self.merge_params(params) r = yield self.session.get(*args, **kwargs) returnValue(r) def parse_fragment(self, url): """ Sometimes osu pages have query-style fragments for some reason :param url: URL object to parse fragment from :type url: plugins.urls.url.URL :return: Parsed fragment as a dict :rtype: dict """ parsed = {} if not url.fragment: return parsed for element in url.fragment.split("&"): if "=" in element: left, right = element.split("=", 1) parsed[left] = right else: parsed[element] = None return parsed def merge_params(self, params): params.update({ "k": self.api_key }) return params @inlineCallbacks def call(self, url, context): target = url.path while target.endswith("/"): target = target[:-1] target = target.split("/") if "" in target: target.remove("") if " " in target: target.remove(" ") message = "" try: if len(target) < 2: # It's the front page or invalid, don't bother returnValue(CASCADE) elif target[0] in [ # Special cases we don't care about "forum", "wiki", "news" ]: returnValue(True) elif target[0].lower() == "p": # Old-style page URL if target[1].lower() == "beatmap": if "b" in url.query: message = yield self.beatmap(url, url.query["b"]) elif target[0].lower() == "u": # User page message = yield self.user(url, target[1]) elif target[0].lower() == "s": # Beatmap set message = yield self.mapset(url, target[1]) elif target[0].lower() == "b": # Specific beatmap message = yield self.beatmap(url, target[1]) except Exception: self.plugin.logger.exception("Error handling URL: {}".format(url)) returnValue(CASCADE) # At this point, if `message` isn't set then we don't understand the # url, and so we'll just allow it to pass down to the other handlers if message: context["event"].target.respond(message) returnValue(STOP_HANDLING) else: returnValue(CASCADE) @inlineCallbacks def beatmap(self, url, beatmap): fragment = self.parse_fragment(url) params = {} if url.query: params.update(url.query) if fragment: params.update(fragment) params["b"] = beatmap r = yield self.get(URL_BEATMAPS, params=params) beatmap = r.json()[0] if "m" not in params: params["m"] = beatmap["mode"] for key in ["favourite_count", "playcount", "passcount"]: beatmap[key] = locale.format( "%d", int(beatmap[key]), grouping=True ) for key in ["difficultyrating"]: beatmap[key] = int(round(float(beatmap[key]))) if "approved" in beatmap: beatmap["approved"] = OSU_APPROVALS.get( beatmap["approved"], u"Unknown approval" ) beatmap["mode"] = OSU_MODES[beatmap["mode"]] scores = None try: r = yield self.get(URL_SCORES, params=params) scores = r.json() for score in scores: for key in ["score", "count50", "count100", "count300", "countmiss", "countkatu", "countgeki"]: score[key] = locale.format( "%d", int(score[key]), grouping=True ) for key in ["pp"]: score[key] = int(round(float(score[key]))) score["enabled_mods"] = ", ".join( get_mods(int(score["enabled_mods"])) ) except Exception: pass data = beatmap if beatmap["approved"] in [ u"Pending", u"WIP", u"Graveyard", u"Unknown approval" ]: message = self.get_string("beatmap-unapproved") elif scores is None: message = self.get_string("beatmap-mode-mismatch") elif not scores: message = self.get_string("beatmap-no-scores") else: data["scores"] = scores message = self.get_string("beatmap") returnValue(message.format(**data)) @inlineCallbacks def mapset(self, url, mapset): params = { "s": mapset } r = yield self.get(URL_BEATMAPS, params=params) data = r.json() modes = {} to_join = [] for beatmap in data: modes[beatmap["mode"]] = modes.get(beatmap["mode"], 0) + 1 beatmap["mode"] = OSU_MODES[beatmap["mode"]] for key in ["favourite_count", "playcount", "passcount"]: beatmap[key] = locale.format( "%d", int(beatmap[key]), grouping=True ) for key in ["difficultyrating"]: beatmap[key] = int(round(float(beatmap[key]))) if "approved" in beatmap: beatmap["approved"] = OSU_APPROVALS.get( beatmap["approved"], u"Unknown approval: {}".format( beatmap["approved"] ) ) for k, v in modes.iteritems(): if v: to_join.append("{} x{}".format(OSU_MODES[k], v)) first = data[0] data = { "beatmaps": data, "counts": ", ".join(to_join) } data.update(first) returnValue(self.get_string("mapset").format(**data)) @inlineCallbacks def user(self, url, user): fragment = self.parse_fragment(url) params = { "u": user, } if "m" in fragment: # Focused mode m = fragment["m"].lower() if m in OSU_MODES: params["m"] = OSU_MODES[m] else: try: params["m"] = int(m) except ValueError: pass # This logic is down to being able to specify either a username or ID. # The osu backend has to deal with this and so the api lets us specify # either "string" or "id" for usernames and IDs respectively. This # may be useful for usernames that are numerical, so we allow users # to add this to the fragment if they wish. if "t" in fragment: # This once was called "t".. params["type"] = fragment["t"] elif "type" in fragment: # ..but now is "type" for some reason params["type"] = fragment["type"] r = yield self.get(URL_USER, params=params) data = r.json()[0] # It's a list for some reason for key in ["level", "accuracy"]: # Round floats data[key] = int(round(float(data[key]))) for key in ["ranked_score", "pp_raw", "pp_rank", "count300", "count100", "count50", "playcount", "total_score", "pp_country_rank"]: # Localisé number formatting data[key] = locale.format( "%d", int(data[key]), grouping=True ) epic_factors = [ int(event["epicfactor"]) for event in data["events"] ] epic_total = reduce(sum, epic_factors, 0) epic_avg = 0 if epic_total: epic_avg = round( epic_total / (1.0 * len(epic_factors)), 2 ) data["events"] = "{} events at an average of {}/32 epicness".format( len(epic_factors), epic_avg ) returnValue(self.get_string("user").format(**data))
def new_instance(cls, session=None): """Initialize an instance using values from the configuration""" if not session: session = Session() return cls(session, conf.settings['ANALYTICS_ENDPOINT'], utils.deobfuscate(conf.settings['ANALYTICS_TOKEN']))
class OsuHandler(URLHandler): criteria = { "protocol": re.compile(r"http|https", str_to_regex_flags("iu")), "domain": re.compile(r"osu\.ppy\.sh", str_to_regex_flags("iu")) } session = None name = "osu" @property def api_key(self): return self.plugin.config.get("osu", {}).get("api_key", "") def __init__(self, plugin): super(OsuHandler, self).__init__(plugin) if not self.api_key: raise ApiKeyMissing() self.reload() def reload(self): self.teardown() self.session = Session() def teardown(self): if self.session is not None: self.session.close() def get_string(self, string): formatting = self.plugin.config.get("osu", {}).get("formatting", {}) if string not in formatting: return strings[string] return formatting[string] @inlineCallbacks def get(self, *args, **kwargs): params = kwargs.get("params", {}) kwargs["params"] = self.merge_params(params) r = yield self.session.get(*args, **kwargs) returnValue(r) def parse_fragment(self, url): """ Sometimes osu pages have query-style fragments for some reason :param url: URL object to parse fragment from :type url: plugins.urls.url.URL :return: Parsed fragment as a dict :rtype: dict """ parsed = {} if not url.fragment: return parsed for element in url.fragment.split("&"): if "=" in element: left, right = element.split("=", 1) parsed[left] = right else: parsed[element] = None return parsed def merge_params(self, params): params.update({"k": self.api_key}) return params @inlineCallbacks def call(self, url, context): target = url.path while target.endswith("/"): target = target[:-1] target = target.split("/") if "" in target: target.remove("") if " " in target: target.remove(" ") message = "" try: if len(target) < 2: # It's the front page or invalid, don't bother returnValue(CASCADE) elif target[0] in [ # Special cases we don't care about "forum", "wiki", "news" ]: returnValue(True) elif target[0].lower() == "p": # Old-style page URL if target[1].lower() == "beatmap": if "b" in url.query: message = yield self.beatmap(url, url.query["b"]) elif target[0].lower() == "u": # User page message = yield self.user(url, target[1]) elif target[0].lower() == "s": # Beatmap set message = yield self.mapset(url, target[1]) elif target[0].lower() == "b": # Specific beatmap message = yield self.beatmap(url, target[1]) except Exception: self.plugin.logger.exception("Error handling URL: {}".format(url)) returnValue(CASCADE) # At this point, if `message` isn't set then we don't understand the # url, and so we'll just allow it to pass down to the other handlers if message: context["event"].target.respond(message) returnValue(STOP_HANDLING) else: returnValue(CASCADE) @inlineCallbacks def beatmap(self, url, beatmap): fragment = self.parse_fragment(url) params = {} if url.query: params.update(url.query) if fragment: params.update(fragment) params["b"] = beatmap r = yield self.get(URL_BEATMAPS, params=params) beatmap = r.json()[0] if "m" not in params: params["m"] = beatmap["mode"] for key in ["favourite_count", "playcount", "passcount"]: beatmap[key] = locale.format("%d", int(beatmap[key]), grouping=True) for key in ["difficultyrating"]: beatmap[key] = int(round(float(beatmap[key]))) if "approved" in beatmap: beatmap["approved"] = OSU_APPROVALS.get(beatmap["approved"], u"Unknown approval") beatmap["mode"] = OSU_MODES[beatmap["mode"]] scores = None try: r = yield self.get(URL_SCORES, params=params) scores = r.json() for score in scores: for key in [ "score", "count50", "count100", "count300", "countmiss", "countkatu", "countgeki" ]: score[key] = locale.format("%d", int(score[key]), grouping=True) for key in ["pp"]: score[key] = int(round(float(score[key]))) score["enabled_mods"] = ", ".join( get_mods(int(score["enabled_mods"]))) except Exception: pass data = beatmap if beatmap["approved"] in [ u"Pending", u"WIP", u"Graveyard", u"Unknown approval" ]: message = self.get_string("beatmap-unapproved") elif scores is None: message = self.get_string("beatmap-mode-mismatch") elif not scores: message = self.get_string("beatmap-no-scores") else: data["scores"] = scores message = self.get_string("beatmap") returnValue(message.format(**data)) @inlineCallbacks def mapset(self, url, mapset): params = {"s": mapset} r = yield self.get(URL_BEATMAPS, params=params) data = r.json() modes = {} to_join = [] for beatmap in data: modes[beatmap["mode"]] = modes.get(beatmap["mode"], 0) + 1 beatmap["mode"] = OSU_MODES[beatmap["mode"]] for key in ["favourite_count", "playcount", "passcount"]: beatmap[key] = locale.format("%d", int(beatmap[key]), grouping=True) for key in ["difficultyrating"]: beatmap[key] = int(round(float(beatmap[key]))) if "approved" in beatmap: beatmap["approved"] = OSU_APPROVALS.get( beatmap["approved"], u"Unknown approval: {}".format(beatmap["approved"])) for k, v in modes.iteritems(): if v: to_join.append("{} x{}".format(OSU_MODES[k], v)) first = data[0] data = {"beatmaps": data, "counts": ", ".join(to_join)} data.update(first) returnValue(self.get_string("mapset").format(**data)) @inlineCallbacks def user(self, url, user): fragment = self.parse_fragment(url) params = { "u": user, } if "m" in fragment: # Focused mode m = fragment["m"].lower() if m in OSU_MODES: params["m"] = OSU_MODES[m] else: try: params["m"] = int(m) except ValueError: pass # This logic is down to being able to specify either a username or ID. # The osu backend has to deal with this and so the api lets us specify # either "string" or "id" for usernames and IDs respectively. This # may be useful for usernames that are numerical, so we allow users # to add this to the fragment if they wish. if "t" in fragment: # This once was called "t".. params["type"] = fragment["t"] elif "type" in fragment: # ..but now is "type" for some reason params["type"] = fragment["type"] r = yield self.get(URL_USER, params=params) data = r.json()[0] # It's a list for some reason for key in ["level", "accuracy"]: # Round floats data[key] = int(round(float(data[key]))) for key in [ "ranked_score", "pp_raw", "pp_rank", "count300", "count100", "count50", "playcount", "total_score", "pp_country_rank" ]: # Localisé number formatting data[key] = locale.format("%d", int(data[key]), grouping=True) epic_factors = [int(event["epicfactor"]) for event in data["events"]] epic_total = reduce(sum, epic_factors, 0) epic_avg = 0 if epic_total: epic_avg = round(epic_total / (1.0 * len(epic_factors)), 2) data["events"] = "{} events at an average of {}/32 epicness".format( len(epic_factors), epic_avg) returnValue(self.get_string("user").format(**data))
def __init__(self, api_key=None, client_id=None): self.api_key = api_key self.client_id = client_id self._session = Session()
def __init__(self, max_concurrent_requests): self.session = Session(maxthreads=max_concurrent_requests)
import os import random import sys import time import boto import psycopg2 import requests from faker import Faker from twisted.internet import defer from twisted.internet.task import react from txrequests import Session import push_helper as ph session = Session(maxthreads=10) responses = [] # Instantiate logger logger = logging.getLogger(__name__) logger.setLevel(logging.ERROR) # Create a file handler handler = logging.FileHandler('/home/ubuntu/push_engine/push_engine.log') handler.setLevel(logging.ERROR) # Create a logging format formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter)
# coding=utf-8 from txrequests import Session from utils.html import unescape_html_entities __author__ = "Gareth Coles" url = "http://ajax.googleapis.com/ajax/services/search/web" session = Session() def get_results(query, page=0, limit=None): if limit is None: limit = 4 start = int(page * limit) # In case some fool passes a float if start > 0: start -= 1 return session.get(url, params={"v": "1.0", "start": start, "q": query}) def parse_results(json, limit=None): if limit is None: limit = 4 result = {} i = 1
def message_handler(self, event): """ Event handler for general messages """ protocol = event.caller source = event.source target = event.target message = event.message if protocol.TYPE == "irc": message = protocol.utils.strip_formatting(message) allowed = self.commands.perm_handler.check("urls.trigger", source, target, protocol) if not allowed: return if isinstance(target, Channel): self.ensure_channel(protocol.name, target.name) status = self.channels.get(protocol.name, {})\ .get(target.name, {})\ .get("status", True) if not status or status == "off": return matches = extract_urls(message) for match in matches: self.logger.trace("match: {0}", match) _url = self.match_to_url(match) if _url is None: continue # Check redirects, following as necessary redirects = 0 max_redirects = self.config.get("redirects", {}).get("max", 15) domains = self.config.get("redirects", {}).get("domains", []) self.logger.debug("Checking redirects...") while _url.domain in domains and redirects < max_redirects: redirects += 1 session = Session() #: :type: requests.Response r = yield session.get(unicode(_url), allow_redirects=False) if r.is_redirect: # This only ever happens when we have a well-formed # redirect that could have been handled automatically redirect_url = r.headers["location"] self.logger.debug( "Redirect [{0:03d}] {1}".format( redirects, redirect_url ) ) _url = self.match_to_url(extract_urls(redirect_url)[0]) else: break if redirects >= max_redirects: self.logger.debug("URL has exceeded the redirects limit") return lazy_request = LazyRequest(req_args=[unicode(_url)]) if isinstance(target, Channel): with self.channels: self.channels[protocol.name][target.name]["last"] = ( unicode(_url) ) yield self.run_handlers(_url, { "event": event, "config": self.config, "get_request": lazy_request, "redirects": redirects, "max_redirects": max_redirects })
class Domainr(object): """ Basic Domainr API wrapper. Returns parsed JSON response. """ # Availability responses AVAILABLE = "available" TAKEN = "taken" UNAVAILABLE = "unavailable" MAYBE = "maybe" TLD = "tld" # API key and client_id auth use different domains API_URL_CID = "https://api.domainr.com/v1/" API_URL_KEY = "https://domainr.p.mashape.com/v1/" def __init__(self, api_key=None, client_id=None): self.api_key = api_key self.client_id = client_id self._session = Session() def _handle_response(self, response): result = response.json() if "error" in result: raise DomainrError(**result["error"]) elif "error_message" in result: # Apparently the API doesn't follow the docs... raise DomainrError(message=result["error_message"]) else: return result # I'll have to play around to see what the best limit/buffer is, but it # should be ~60 per minute anyway. # Sod it, the rate limiting plugin (coming soon) can deal with # burst/slowdown - we'll just set this to 60 per 60. # 2015/10/07 - This is way over (~260x) what the free tier allows, but this # has to work with the paid tier too. Additionally, limiting the free tier # would have to be done in terms of at least daily time periods to allow # for bursts. I'll consider how best to deal with this. It's not like we # were making 10,000 calls per month before anyway, but it's definitely # something that's used in rapid bursts between long periods of non-use. # Config options would likely be best. @RateLimiter(limit=60, buffer=10, time_period=60) def _make_request(self, method, payload): """ Actually make the HTTP request. :rtype : twisted.internet.defer.Deferred """ url = self.API_URL_KEY if self.client_id is not None: payload["client_id"] = self.client_id url = self.API_URL_CID elif self.api_key is not None: payload["mashape-key"] = self.api_key deferred = self._session.get(url + method, params=payload) deferred.addCallback(self._handle_response) return deferred def search(self, query): """ Search for domain suggestions for the given query. :rtype : twisted.internet.defer.Deferred """ payload = { "q": query } return self._make_request("search", payload) def info(self, domain): """ Get info for given domain. :rtype : twisted.internet.defer.Deferred """ payload = { "q": domain } return self._make_request("info", payload)
def message_handler(self, event): """ Event handler for general messages """ protocol = event.caller source = event.source target = event.target message = event.message if protocol.TYPE == "irc": message = protocol.utils.strip_formatting(message) allowed = self.commands.perm_handler.check("urls.trigger", source, target, protocol) if not allowed: return if isinstance(target, Channel): self.ensure_channel(protocol.name, target.name) status = self.channels.get(protocol.name, {})\ .get(target.name, {})\ .get("status", True) if not status or status == "off": return matches = extract_urls(message) for match in matches: self.logger.trace("match: {0}", match) _url = self.match_to_url(match) if _url is None: continue # Check redirects, following as necessary redirects = 0 max_redirects = self.config.get("redirects", {}).get("max", 15) domains = self.config.get("redirects", {}).get("domains", []) self.logger.debug("Checking redirects...") while _url.domain in domains and redirects < max_redirects: redirects += 1 session = Session() #: :type: requests.Response r = yield session.get(unicode(_url), allow_redirects=False) if r.is_redirect: # This only ever happens when we have a well-formed # redirect that could have been handled automatically redirect_url = r.headers["location"] self.logger.debug("Redirect [{0:03d}] {1}".format( redirects, redirect_url)) _url = self.match_to_url(extract_urls(redirect_url)[0]) else: break if redirects >= max_redirects: self.logger.debug("URL has exceeded the redirects limit") return lazy_request = LazyRequest(req_args=[unicode(_url)]) if isinstance(target, Channel): with self.channels: self.channels[protocol.name][target.name]["last"] = ( unicode(_url)) yield self.run_handlers( _url, { "event": event, "config": self.config, "get_request": lazy_request, "redirects": redirects, "max_redirects": max_redirects })
class FListHandler(URLHandler): criteria = { "protocol": re.compile(r"http|https", str_to_regex_flags("iu")), "domain": re.compile(r"(www\.f-list\.net)|(f-list\.net)", str_to_regex_flags("iu")), "path": re.compile(r"/c/.*", str_to_regex_flags("iu")), "permission": "urls.trigger.nsfw" } ticket = "" # API auth ticket; needs manual renewing last_renewal = None # So we know when we renewed last session = None name = "f-list" @property def username(self): return self.plugin.config.get("f-list", {}).get("username", "") @property def password(self): return self.plugin.config.get("f-list", {}).get("password", "") @property def kinks_limit(self): return self.plugin.config.get("f-list", {}).get("kink-sample", 2) def __init__(self, plugin): super(FListHandler, self).__init__(plugin) if not (self.username and self.password): raise ApiKeyMissing() self.reload() self.get_ticket() def reload(self): self.teardown() self.session = Session() def teardown(self): if self.session is not None: self.session.close() def get_string(self, string): formatting = self.plugin.config.get("osu", {}).get("formatting", {}) if string not in formatting: return strings[string] return formatting[string] @inlineCallbacks def get(self, *args, **kwargs): r = yield self.session.get(*args, **kwargs) data = r.json() if "error" in data and data["error"]: raise FListError(data["error"]) returnValue(data) @inlineCallbacks def post(self, *args, **kwargs): r = yield self.session.post(*args, **kwargs) data = r.json() if "error" in data and data["error"]: raise FListError(data["error"]) returnValue(data) @inlineCallbacks def get_ticket(self): now = datetime.now() then = now - timedelta(minutes=4) if not self.last_renewal or then > self.last_renewal: data = yield self.post(URL_TICKET, params={ "account": self.username, "password": self.password }) self.ticket = data["ticket"] self.last_renewal = datetime.now() returnValue(self.ticket) def get_sample(self, items, count): if not items: return ["Nothing"] if len(items) <= count: return items return [i for i in random.sample(items, count)] @inlineCallbacks def call(self, url, context): target = url.path while target.endswith("/"): target = target[:-1] target = target.split("/") if "" in target: target.remove("") if " " in target: target.remove(" ") message = "" try: if len(target) < 2: # It's the front page or invalid, don't bother returnValue(CASCADE) elif target[0].lower() == "c": # Character page message = yield self.character(target[1]) except Exception: self.plugin.logger.exception("Error handling URL: {}".format(url)) returnValue(CASCADE) # At this point, if `message` isn't set then we don't understand the # url, and so we'll just allow it to pass down to the other handlers if message: context["event"].target.respond(message) returnValue(STOP_HANDLING) else: returnValue(CASCADE) @inlineCallbacks def character(self, char_name): char_name = urlparse.unquote(char_name) ticket = yield self.get_ticket() params = { "ticket": ticket, "name": char_name, "account": self.username } char_info = yield self.post(URL_CHAR_INFO, params=params) char_kinks = yield self.post(URL_CHAR_KINKS, params=params) char_info = flatten_character(char_info) char_kinks = flatten_kinks(char_kinks) data = char_info["info"] data["sample_kinks"] = { "fave": ", ".join( self.get_sample(char_kinks["preferences"]["fave"], self.kinks_limit)), "yes": ", ".join( self.get_sample(char_kinks["preferences"]["yes"], self.kinks_limit)), "maybe": ", ".join( self.get_sample(char_kinks["preferences"]["maybe"], self.kinks_limit)), "no": ", ".join( self.get_sample(char_kinks["preferences"]["no"], self.kinks_limit)), } data["given"] = {"name": char_name} returnValue( self.get_string("character").format(**data).replace( u"&", u"&"))
def get_session(self, url, context): sessions = context.get("config", {}).get("sessions", {}) if not sessions.get("enable", False): self.urls_plugin.logger.debug("Sessions are disabled.") proxy = self.urls_plugin.get_proxy(url) if not proxy: s = Session() else: s = ProxySession(proxy) s.session_type = None return s for entry in sessions["never"]: if re.match(entry, url.domain, flags=str_to_regex_flags("ui")): self.urls_plugin.logger.debug( "Domain {0} is blacklisted for sessions.".format( url.domain ) ) proxy = self.urls_plugin.get_proxy(url) if not proxy: s = Session() else: s = ProxySession(proxy) s.session_type = None return s for group, entries in sessions["group"].iteritems(): for entry in entries: try: if re.match( entry, url.domain, flags=str_to_regex_flags("ui") ): self.urls_plugin.logger.debug( "Domain {0} uses the '{1}' group sessions.".format( url.domain, group ) ) if group not in self.group_sessions: proxy = self.urls_plugin.get_proxy(group=group) if not proxy: s = Session() else: s = ProxySession(proxy) s.cookies = ( self.get_cookie_jar( "/groups/{0}.txt".format( group ) ) ) s.session_type = "group" s.cookies.set_mode( context.get("config") .get("sessions") .get("cookies") .get("group") ) self.group_sessions[group] = s return self.group_sessions[group] except ValueError as e: self.urls_plugin.logger.error( "Failed to create cookie jar: {0}".format(e) ) continue self.urls_plugin.logger.debug( "Domain {0} uses the global session storage.".format( url.domain ) ) proxy = self.urls_plugin.get_proxy(url) if not proxy: return self.global_session else: s = ProxySession(proxy) s.cookies = self.get_cookie_jar("/global.txt") s.session_type = "global" s.cookies.set_mode( self.plugin.config.get("sessions", {}) .get("cookies", {}) .get("global", "discard") ) return s
class Domainr(object): """ Basic Domainr API wrapper. Returns parsed JSON response. """ # Availability responses AVAILABLE = "available" TAKEN = "taken" UNAVAILABLE = "unavailable" MAYBE = "maybe" TLD = "tld" # API key and client_id auth use different domains API_URL_CID = "https://api.domainr.com/v1/" API_URL_KEY = "https://domainr.p.mashape.com/v1/" def __init__(self, api_key=None, client_id=None): self.api_key = api_key self.client_id = client_id self._session = Session() def _handle_response(self, response): result = response.json() if "error" in result: raise DomainrError(**result["error"]) elif "error_message" in result: # Apparently the API doesn't follow the docs... raise DomainrError(message=result["error_message"]) else: return result # I'll have to play around to see what the best limit/buffer is, but it # should be ~60 per minute anyway. # Sod it, the rate limiting plugin (coming soon) can deal with # burst/slowdown - we'll just set this to 60 per 60. # 2015/10/07 - This is way over (~260x) what the free tier allows, but this # has to work with the paid tier too. Additionally, limiting the free tier # would have to be done in terms of at least daily time periods to allow # for bursts. I'll consider how best to deal with this. It's not like we # were making 10,000 calls per month before anyway, but it's definitely # something that's used in rapid bursts between long periods of non-use. # Config options would likely be best. @RateLimiter(limit=60, buffer=10, time_period=60) def _make_request(self, method, payload): """ Actually make the HTTP request. :rtype : twisted.internet.defer.Deferred """ url = self.API_URL_KEY if self.client_id is not None: payload["client_id"] = self.client_id url = self.API_URL_CID elif self.api_key is not None: payload["mashape-key"] = self.api_key deferred = self._session.get(url + method, params=payload) deferred.addCallback(self._handle_response) return deferred def search(self, query): """ Search for domain suggestions for the given query. :rtype : twisted.internet.defer.Deferred """ payload = {"q": query} return self._make_request("search", payload) def info(self, domain): """ Get info for given domain. :rtype : twisted.internet.defer.Deferred """ payload = {"q": domain} return self._make_request("info", payload)
class FListHandler(URLHandler): criteria = { "protocol": re.compile(r"http|https", str_to_regex_flags("iu")), "domain": re.compile( r"(www\.f-list\.net)|(f-list\.net)", str_to_regex_flags("iu") ), "path": re.compile(r"/c/.*", str_to_regex_flags("iu")), "permission": "urls.trigger.nsfw" } ticket = "" # API auth ticket; needs manual renewing last_renewal = None # So we know when we renewed last session = None name = "f-list" @property def username(self): return self.plugin.config.get("f-list", {}).get("username", "") @property def password(self): return self.plugin.config.get("f-list", {}).get("password", "") @property def kinks_limit(self): return self.plugin.config.get("f-list", {}).get("kink-sample", 2) def __init__(self, plugin): super(FListHandler, self).__init__(plugin) if not (self.username and self.password): raise ApiKeyMissing() self.reload() self.get_ticket() def reload(self): self.teardown() self.session = Session() def teardown(self): if self.session is not None: self.session.close() def get_string(self, string): formatting = self.plugin.config.get("osu", {}).get("formatting", {}) if string not in formatting: return strings[string] return formatting[string] @inlineCallbacks def get(self, *args, **kwargs): r = yield self.session.get(*args, **kwargs) data = r.json() if "error" in data and data["error"]: raise FListError(data["error"]) returnValue(data) @inlineCallbacks def post(self, *args, **kwargs): r = yield self.session.post(*args, **kwargs) data = r.json() if "error" in data and data["error"]: raise FListError(data["error"]) returnValue(data) @inlineCallbacks def get_ticket(self): now = datetime.now() then = now - timedelta(minutes=4) if not self.last_renewal or then > self.last_renewal: data = yield self.post( URL_TICKET, params={ "account": self.username, "password": self.password } ) self.ticket = data["ticket"] self.last_renewal = datetime.now() returnValue(self.ticket) def get_sample(self, items, count): if not items: return ["Nothing"] if len(items) <= count: return items return [i for i in random.sample(items, count)] @inlineCallbacks def call(self, url, context): target = url.path while target.endswith("/"): target = target[:-1] target = target.split("/") if "" in target: target.remove("") if " " in target: target.remove(" ") message = "" try: if len(target) < 2: # It's the front page or invalid, don't bother returnValue(CASCADE) elif target[0].lower() == "c": # Character page message = yield self.character(target[1]) except Exception: self.plugin.logger.exception("Error handling URL: {}".format(url)) returnValue(CASCADE) # At this point, if `message` isn't set then we don't understand the # url, and so we'll just allow it to pass down to the other handlers if message: context["event"].target.respond(message) returnValue(STOP_HANDLING) else: returnValue(CASCADE) @inlineCallbacks def character(self, char_name): char_name = urlparse.unquote(char_name) ticket = yield self.get_ticket() params = { "ticket": ticket, "name": char_name, "account": self.username } char_info = yield self.post(URL_CHAR_INFO, params=params) char_kinks = yield self.post(URL_CHAR_KINKS, params=params) char_info = flatten_character(char_info) char_kinks = flatten_kinks(char_kinks) data = char_info["info"] data["sample_kinks"] = { "fave": ", ".join(self.get_sample( char_kinks["preferences"]["fave"], self.kinks_limit )), "yes": ", ".join(self.get_sample( char_kinks["preferences"]["yes"], self.kinks_limit )), "maybe": ", ".join(self.get_sample( char_kinks["preferences"]["maybe"], self.kinks_limit )), "no": ", ".join(self.get_sample( char_kinks["preferences"]["no"], self.kinks_limit )), } data["given"] = { "name": char_name } returnValue( self.get_string("character").format(**data).replace(u"&", u"&") )