class Crawler:
def __init__(self, max_concurrent_requests):
self.session = Session(maxthreads=max_concurrent_requests)
def add_website_request(self, website: Website, callback: Callable[[Session, requests.Response], requests.Response],
query_params: dict, data: dict):
self.add_request(website.method, website.api_url, callback, query_params, data)
def add_request(self, method: str, url: str, callback: Callable[[Session, requests.Response], requests.Response],
query_params: dict, data: dict):
logging.debug(f"{method} {url} [params: {query_params}, data: {data}]")
self.session.request(method=method, url=url, params=query_params, #data=data,
background_callback=callback)
def add_cookies(self, cookies: CookieJar):
self.session.cookies.update(cookies)
def stop(self):
self.session.close()
class YoutubeHandler(URLHandler):
name = "youtube"
criteria = {
"protocol": re.compile(r"http|https", re.I),
"domain": re.compile(r"(www\.)?(youtube\.com|youtu\.be)", re.I),
}
VIDEO_LINK, CHANNEL_LINK, PLAYLIST_LINK = xrange(3)
BASE_URL = "https://www.googleapis.com/youtube/v3/"
VIDEOS_URL = BASE_URL + "videos"
CHANNELS_URL = BASE_URL + "channels"
PLAYLISTS_URL = BASE_URL + "playlists"
DEFAULT_FORMATS = {
"video": u'[YouTube Video] "{title}" by {channel} - {description} - '
u'length {duration_formatted} - rated {rating_percent:.0f}%'
u' - {views} views',
"channel": u'[YouTube Channel] {title} - {description} - {videos} '
u'videos - {subscribers} subscribers - {views} views',
"playlist": u'[YouTube Playlist] "{title}" by {channel} - '
u'{description} - {videos} videos',
}
def __init__(self, *args, **kwargs):
super(YoutubeHandler, self).__init__(*args, **kwargs)
if not self.api_key:
raise ApiKeyMissing()
self.session = Session()
@property
def api_key(self):
return self.plugin.config.get("youtube", {}).get("api_key", "")
@property
def api_key_referrer(self):
youtube_conf = self.plugin.config.get("youtube", {})
return youtube_conf.get("api_key_referrer", "")
@property
def description_length(self):
youtube_conf = self.plugin.config.get("youtube", {})
return youtube_conf.get("description_length", 75)
def get_format_string(self, key):
youtube_conf = self.plugin.config.get("youtube", {})
format_conf = youtube_conf.get("formatting", {})
if key not in format_conf:
return self.DEFAULT_FORMATS[key]
return format_conf[key]
def call(self, url, context):
domain = url.domain.lower()
if domain.startswith(u"www."):
domain = domain[4:]
if domain == u"youtu.be":
link_type, data = self._parse_youtu_be(url)
else:
link_type, data = self._parse_youtube_com(url)
if link_type == self.VIDEO_LINK:
self.handle_video(data, context)
return STOP_HANDLING
elif link_type == self.CHANNEL_LINK:
self.handle_channel(data, context)
return STOP_HANDLING
elif link_type == self.PLAYLIST_LINK:
self.handle_playlist(data, context)
return STOP_HANDLING
else:
return CASCADE
def _parse_youtu_be(self, url):
return self.VIDEO_LINK, url.path.strip("/")
def _parse_youtube_com(self, url):
# Video: https://www.youtube.com/watch?v=orvJo3nNZuI
# Channel:
# Username: https://www.youtube.com/user/Mtvnoob
# Channel ID: https://www.youtube.com/channel/UCmkoMt2VCc3TaFSE5MKrkpQ
# Playlist: https://www.youtube.com/playlist?list=PLE6Wd9FR--EfW8dtjAuPoTuPcqmOV53Fu # noqa
try:
path_split = url.path.strip("/").split("/")
root_path = path_split[0]
if root_path == u"watch":
return self.VIDEO_LINK, url.query["v"]
elif root_path == u"user":
return self.CHANNEL_LINK, {u"username": path_split[1]}
elif root_path == u"channel":
return self.CHANNEL_LINK, {u"channel_id": path_split[1]}
elif root_path == u"playlist":
return self.PLAYLIST_LINK, url.query[u"list"]
except Exception:
self.plugin.logger.exception("Error parsing youtube.com URL")
return None, None
def _get(self, url, params, **kwargs):
referrer = self.api_key_referrer
if referrer:
headers = {"referer": referrer}
if "headers" in kwargs:
headers.update(kwargs["headers"])
kwargs["headers"] = headers
params["key"] = self.api_key
return self.session.get(url, params=params, **kwargs)
def handle_video(self, video_id, context):
req_def = self._get(self.VIDEOS_URL, params={
"part": "snippet,contentDetails,statistics",
"id": video_id,
})
return self._add_callbacks(self._handle_video_response,
self._handle_request_failure,
context, req_def)
def handle_channel(self, data, context):
params = {
"part": "snippet,statistics",
}
if "channel_id" in data:
params["id"] = data["channel_id"]
elif "username" in data:
params["forUsername"] = data["username"]
else:
raise ValueError("Must specify channel_id or username")
req_def = self._get(self.CHANNELS_URL, params=params)
return self._add_callbacks(self._handle_channel_response,
self._handle_request_failure,
context, req_def)
def handle_playlist(self, playlist_id, context):
req_def = self._get(self.PLAYLISTS_URL, params={
"part": "snippet,contentDetails",
"id": playlist_id,
})
return self._add_callbacks(self._handle_playlist_response,
self._handle_request_failure,
context, req_def)
def _add_callbacks(self, callback, errback, context, req_def):
result_def = Deferred()
req_def.addCallback(callback, context, result_def)
req_def.addErrback(errback, context, result_def)
return result_def
def _handle_video_response(self, response, context, result_def):
data = response.json()
items = self._get_items(data)
content_details = items["contentDetails"]
snippet = items["snippet"]
statistics = items["statistics"]
description = snippet["description"].strip()
if len(description) == 0:
description = "No description"
description_snippet = self.snip_description(description)
duration = isodate.parse_duration(content_details["duration"])
likes_count = int(statistics["likeCount"])
dislike_count = int(statistics["dislikeCount"])
ratings_total = likes_count + dislike_count
rating_percentage = (float(likes_count) / ratings_total) * 100
tags = snippet.get("tags", [])
if len(tags) > 0:
tags_formatted = ", ".join(tags[:5])
else:
tags_formatted = "No tags"
duration_formatted = self.format_time_period(duration)
format_data = {
"full_response": data,
"title": snippet["title"],
"channel": snippet["channelTitle"],
"duration": duration,
"duration_formatted": duration_formatted,
"description": description_snippet,
"full_description": description,
"tags": tags,
"tags_formatted": tags_formatted,
"likes": likes_count,
"dislikes": dislike_count,
"favourites": int(statistics["favoriteCount"]),
"views": int(statistics["viewCount"]),
"comments": int(statistics["commentCount"]),
"rating_percent": rating_percentage,
"rating_total": ratings_total
}
message = self.get_format_string("video").format(**format_data)
self._handle_message(message, context)
result_def.callback(STOP_HANDLING)
def _handle_channel_response(self, response, context, result_def):
data = response.json()
items = self._get_items(data)
snippet = items["snippet"]
statistics = items["statistics"]
description = snippet["description"]
if len(description) == 0:
description = "No description"
description_snippet = self.snip_description(description)
try:
# I'm not sure what happens here if hiddenSubscriberCount is true
subscribers = int(statistics["subscriberCount"])
except ValueError:
subscribers = 0
hidden_subscribers = statistics["hiddenSubscriberCount"] # noqa
country = snippet.get("country", "Unknown")
format_data = {
"full_response": data,
"title": snippet["title"],
"subscribers": subscribers,
"videos": statistics["videoCount"],
"views": statistics["viewCount"],
"comments": statistics["commentCount"],
"country": country,
"description": description_snippet,
"full_description": description,
}
message = self.get_format_string("channel").format(**format_data)
self._handle_message(message, context)
result_def.callback(STOP_HANDLING)
def _handle_playlist_response(self, response, context, result_def):
data = response.json()
items = self._get_items(data)
content_details = items["contentDetails"]
snippet = items["snippet"]
description = snippet["description"].strip()
if len(description) == 0:
description = "No description"
description_snippet = self.snip_description(description)
format_data = {
"full_response": data,
"title": snippet["title"],
"channel": snippet["channelTitle"],
"videos": content_details["itemCount"],
"description": description_snippet,
"full_description": description,
}
message = self.get_format_string("playlist").format(**format_data)
self._handle_message(message, context)
result_def.callback(STOP_HANDLING)
def _get_items(self, data):
if "error" in data:
error = data["error"]
raise YoutubeAPIError(
error["message"], error["code"], error["errors"])
try:
return data["items"][0]
except LookupError:
raise YoutubeMissingItemError()
def _handle_request_failure(self, fail, context, result_def):
if fail.check(YoutubeAPIError):
self.plugin.logger.error(fail.getErrorMessage())
elif fail.check(YoutubeMissingItemError):
# It's a 404, basically, so don't bother with a title
result_def.callback(STOP_HANDLING)
return
else:
self.plugin.logger.error(fail.getTraceback())
result_def.callback(CASCADE)
def _handle_message(self, message, context):
context["event"].target.respond(message)
def reload(self):
self.teardown()
self.session = Session()
def teardown(self):
if self.session is not None:
self.session.close()
def format_time_period(self, duration):
secs = duration.total_seconds()
m, s = divmod(secs, 60)
if m >= 60:
h, m = divmod(m, 60)
return "%d:%02d:%02d" % (h, m, s)
else:
return "%d:%02d" % (m, s)
def snip_description(self, description, length=0):
if not length:
length = self.description_length
split = description.strip().split(u"\n")
desc = split[0].strip()
if len(desc) > length:
return desc[:length - 3].strip() + u"..."
return desc
class FListHandler(URLHandler):
criteria = {
"protocol":
re.compile(r"http|https", str_to_regex_flags("iu")),
"domain":
re.compile(r"(www\.f-list\.net)|(f-list\.net)",
str_to_regex_flags("iu")),
"path":
re.compile(r"/c/.*", str_to_regex_flags("iu")),
"permission":
"urls.trigger.nsfw"
}
ticket = "" # API auth ticket; needs manual renewing
last_renewal = None # So we know when we renewed last
session = None
name = "f-list"
@property
def username(self):
return self.plugin.config.get("f-list", {}).get("username", "")
@property
def password(self):
return self.plugin.config.get("f-list", {}).get("password", "")
@property
def kinks_limit(self):
return self.plugin.config.get("f-list", {}).get("kink-sample", 2)
def __init__(self, plugin):
super(FListHandler, self).__init__(plugin)
if not (self.username and self.password):
raise ApiKeyMissing()
self.reload()
self.get_ticket()
def reload(self):
self.teardown()
self.session = Session()
def teardown(self):
if self.session is not None:
self.session.close()
def get_string(self, string):
formatting = self.plugin.config.get("osu", {}).get("formatting", {})
if string not in formatting:
return strings[string]
return formatting[string]
@inlineCallbacks
def get(self, *args, **kwargs):
r = yield self.session.get(*args, **kwargs)
data = r.json()
if "error" in data and data["error"]:
raise FListError(data["error"])
returnValue(data)
@inlineCallbacks
def post(self, *args, **kwargs):
r = yield self.session.post(*args, **kwargs)
data = r.json()
if "error" in data and data["error"]:
raise FListError(data["error"])
returnValue(data)
@inlineCallbacks
def get_ticket(self):
now = datetime.now()
then = now - timedelta(minutes=4)
if not self.last_renewal or then > self.last_renewal:
data = yield self.post(URL_TICKET,
params={
"account": self.username,
"password": self.password
})
self.ticket = data["ticket"]
self.last_renewal = datetime.now()
returnValue(self.ticket)
def get_sample(self, items, count):
if not items:
return ["Nothing"]
if len(items) <= count:
return items
return [i for i in random.sample(items, count)]
@inlineCallbacks
def call(self, url, context):
target = url.path
while target.endswith("/"):
target = target[:-1]
target = target.split("/")
if "" in target:
target.remove("")
if " " in target:
target.remove(" ")
message = ""
try:
if len(target) < 2: # It's the front page or invalid, don't bother
returnValue(CASCADE)
elif target[0].lower() == "c": # Character page
message = yield self.character(target[1])
except Exception:
self.plugin.logger.exception("Error handling URL: {}".format(url))
returnValue(CASCADE)
# At this point, if `message` isn't set then we don't understand the
# url, and so we'll just allow it to pass down to the other handlers
if message:
context["event"].target.respond(message)
returnValue(STOP_HANDLING)
else:
returnValue(CASCADE)
@inlineCallbacks
def character(self, char_name):
char_name = urlparse.unquote(char_name)
ticket = yield self.get_ticket()
params = {
"ticket": ticket,
"name": char_name,
"account": self.username
}
char_info = yield self.post(URL_CHAR_INFO, params=params)
char_kinks = yield self.post(URL_CHAR_KINKS, params=params)
char_info = flatten_character(char_info)
char_kinks = flatten_kinks(char_kinks)
data = char_info["info"]
data["sample_kinks"] = {
"fave":
", ".join(
self.get_sample(char_kinks["preferences"]["fave"],
self.kinks_limit)),
"yes":
", ".join(
self.get_sample(char_kinks["preferences"]["yes"],
self.kinks_limit)),
"maybe":
", ".join(
self.get_sample(char_kinks["preferences"]["maybe"],
self.kinks_limit)),
"no":
", ".join(
self.get_sample(char_kinks["preferences"]["no"],
self.kinks_limit)),
}
data["given"] = {"name": char_name}
returnValue(
self.get_string("character").format(**data).replace(
u"&", u"&"))
class OsuHandler(URLHandler):
criteria = {
"protocol": re.compile(r"http|https", str_to_regex_flags("iu")),
"domain": re.compile(r"osu\.ppy\.sh", str_to_regex_flags("iu"))
}
session = None
name = "osu"
@property
def api_key(self):
return self.plugin.config.get("osu", {}).get("api_key", "")
def __init__(self, plugin):
super(OsuHandler, self).__init__(plugin)
if not self.api_key:
raise ApiKeyMissing()
self.reload()
def reload(self):
self.teardown()
self.session = Session()
def teardown(self):
if self.session is not None:
self.session.close()
def get_string(self, string):
formatting = self.plugin.config.get("osu", {}).get("formatting", {})
if string not in formatting:
return strings[string]
return formatting[string]
@inlineCallbacks
def get(self, *args, **kwargs):
params = kwargs.get("params", {})
kwargs["params"] = self.merge_params(params)
r = yield self.session.get(*args, **kwargs)
returnValue(r)
def parse_fragment(self, url):
"""
Sometimes osu pages have query-style fragments for some reason
:param url: URL object to parse fragment from
:type url: plugins.urls.url.URL
:return: Parsed fragment as a dict
:rtype: dict
"""
parsed = {}
if not url.fragment:
return parsed
for element in url.fragment.split("&"):
if "=" in element:
left, right = element.split("=", 1)
parsed[left] = right
else:
parsed[element] = None
return parsed
def merge_params(self, params):
params.update({
"k": self.api_key
})
return params
@inlineCallbacks
def call(self, url, context):
target = url.path
while target.endswith("/"):
target = target[:-1]
target = target.split("/")
if "" in target:
target.remove("")
if " " in target:
target.remove(" ")
message = ""
try:
if len(target) < 2: # It's the front page or invalid, don't bother
returnValue(CASCADE)
elif target[0] in [ # Special cases we don't care about
"forum", "wiki", "news"
]:
returnValue(True)
elif target[0].lower() == "p": # Old-style page URL
if target[1].lower() == "beatmap":
if "b" in url.query:
message = yield self.beatmap(url, url.query["b"])
elif target[0].lower() == "u": # User page
message = yield self.user(url, target[1])
elif target[0].lower() == "s": # Beatmap set
message = yield self.mapset(url, target[1])
elif target[0].lower() == "b": # Specific beatmap
message = yield self.beatmap(url, target[1])
except Exception:
self.plugin.logger.exception("Error handling URL: {}".format(url))
returnValue(CASCADE)
# At this point, if `message` isn't set then we don't understand the
# url, and so we'll just allow it to pass down to the other handlers
if message:
context["event"].target.respond(message)
returnValue(STOP_HANDLING)
else:
returnValue(CASCADE)
@inlineCallbacks
def beatmap(self, url, beatmap):
fragment = self.parse_fragment(url)
params = {}
if url.query:
params.update(url.query)
if fragment:
params.update(fragment)
params["b"] = beatmap
r = yield self.get(URL_BEATMAPS, params=params)
beatmap = r.json()[0]
if "m" not in params:
params["m"] = beatmap["mode"]
for key in ["favourite_count", "playcount", "passcount"]:
beatmap[key] = locale.format(
"%d", int(beatmap[key]), grouping=True
)
for key in ["difficultyrating"]:
beatmap[key] = int(round(float(beatmap[key])))
if "approved" in beatmap:
beatmap["approved"] = OSU_APPROVALS.get(
beatmap["approved"], u"Unknown approval"
)
beatmap["mode"] = OSU_MODES[beatmap["mode"]]
scores = None
try:
r = yield self.get(URL_SCORES, params=params)
scores = r.json()
for score in scores:
for key in ["score", "count50", "count100", "count300",
"countmiss", "countkatu", "countgeki"]:
score[key] = locale.format(
"%d", int(score[key]), grouping=True
)
for key in ["pp"]:
score[key] = int(round(float(score[key])))
score["enabled_mods"] = ", ".join(
get_mods(int(score["enabled_mods"]))
)
except Exception:
pass
data = beatmap
if beatmap["approved"] in [
u"Pending", u"WIP", u"Graveyard", u"Unknown approval"
]:
message = self.get_string("beatmap-unapproved")
elif scores is None:
message = self.get_string("beatmap-mode-mismatch")
elif not scores:
message = self.get_string("beatmap-no-scores")
else:
data["scores"] = scores
message = self.get_string("beatmap")
returnValue(message.format(**data))
@inlineCallbacks
def mapset(self, url, mapset):
params = {
"s": mapset
}
r = yield self.get(URL_BEATMAPS, params=params)
data = r.json()
modes = {}
to_join = []
for beatmap in data:
modes[beatmap["mode"]] = modes.get(beatmap["mode"], 0) + 1
beatmap["mode"] = OSU_MODES[beatmap["mode"]]
for key in ["favourite_count", "playcount", "passcount"]:
beatmap[key] = locale.format(
"%d", int(beatmap[key]), grouping=True
)
for key in ["difficultyrating"]:
beatmap[key] = int(round(float(beatmap[key])))
if "approved" in beatmap:
beatmap["approved"] = OSU_APPROVALS.get(
beatmap["approved"], u"Unknown approval: {}".format(
beatmap["approved"]
)
)
for k, v in modes.iteritems():
if v:
to_join.append("{} x{}".format(OSU_MODES[k], v))
first = data[0]
data = {
"beatmaps": data,
"counts": ", ".join(to_join)
}
data.update(first)
returnValue(self.get_string("mapset").format(**data))
@inlineCallbacks
def user(self, url, user):
fragment = self.parse_fragment(url)
params = {
"u": user,
}
if "m" in fragment: # Focused mode
m = fragment["m"].lower()
if m in OSU_MODES:
params["m"] = OSU_MODES[m]
else:
try:
params["m"] = int(m)
except ValueError:
pass
# This logic is down to being able to specify either a username or ID.
# The osu backend has to deal with this and so the api lets us specify
# either "string" or "id" for usernames and IDs respectively. This
# may be useful for usernames that are numerical, so we allow users
# to add this to the fragment if they wish.
if "t" in fragment: # This once was called "t"..
params["type"] = fragment["t"]
elif "type" in fragment: # ..but now is "type" for some reason
params["type"] = fragment["type"]
r = yield self.get(URL_USER, params=params)
data = r.json()[0] # It's a list for some reason
for key in ["level", "accuracy"]: # Round floats
data[key] = int(round(float(data[key])))
for key in ["ranked_score", "pp_raw", "pp_rank", "count300",
"count100", "count50", "playcount", "total_score",
"pp_country_rank"]: # Localisé number formatting
data[key] = locale.format(
"%d", int(data[key]), grouping=True
)
epic_factors = [
int(event["epicfactor"]) for event in data["events"]
]
epic_total = reduce(sum, epic_factors, 0)
epic_avg = 0
if epic_total:
epic_avg = round(
epic_total / (1.0 * len(epic_factors)), 2
)
data["events"] = "{} events at an average of {}/32 epicness".format(
len(epic_factors),
epic_avg
)
returnValue(self.get_string("user").format(**data))
class OsuHandler(URLHandler):
criteria = {
"protocol": re.compile(r"http|https", str_to_regex_flags("iu")),
"domain": re.compile(r"osu\.ppy\.sh", str_to_regex_flags("iu"))
}
session = None
name = "osu"
@property
def api_key(self):
return self.plugin.config.get("osu", {}).get("api_key", "")
def __init__(self, plugin):
super(OsuHandler, self).__init__(plugin)
if not self.api_key:
raise ApiKeyMissing()
self.reload()
def reload(self):
self.teardown()
self.session = Session()
def teardown(self):
if self.session is not None:
self.session.close()
def get_string(self, string):
formatting = self.plugin.config.get("osu", {}).get("formatting", {})
if string not in formatting:
return strings[string]
return formatting[string]
@inlineCallbacks
def get(self, *args, **kwargs):
params = kwargs.get("params", {})
kwargs["params"] = self.merge_params(params)
r = yield self.session.get(*args, **kwargs)
returnValue(r)
def parse_fragment(self, url):
"""
Sometimes osu pages have query-style fragments for some reason
:param url: URL object to parse fragment from
:type url: plugins.urls.url.URL
:return: Parsed fragment as a dict
:rtype: dict
"""
parsed = {}
if not url.fragment:
return parsed
for element in url.fragment.split("&"):
if "=" in element:
left, right = element.split("=", 1)
parsed[left] = right
else:
parsed[element] = None
return parsed
def merge_params(self, params):
params.update({"k": self.api_key})
return params
@inlineCallbacks
def call(self, url, context):
target = url.path
while target.endswith("/"):
target = target[:-1]
target = target.split("/")
if "" in target:
target.remove("")
if " " in target:
target.remove(" ")
message = ""
try:
if len(target) < 2: # It's the front page or invalid, don't bother
returnValue(CASCADE)
elif target[0] in [ # Special cases we don't care about
"forum", "wiki", "news"
]:
returnValue(True)
elif target[0].lower() == "p": # Old-style page URL
if target[1].lower() == "beatmap":
if "b" in url.query:
message = yield self.beatmap(url, url.query["b"])
elif target[0].lower() == "u": # User page
message = yield self.user(url, target[1])
elif target[0].lower() == "s": # Beatmap set
message = yield self.mapset(url, target[1])
elif target[0].lower() == "b": # Specific beatmap
message = yield self.beatmap(url, target[1])
except Exception:
self.plugin.logger.exception("Error handling URL: {}".format(url))
returnValue(CASCADE)
# At this point, if `message` isn't set then we don't understand the
# url, and so we'll just allow it to pass down to the other handlers
if message:
context["event"].target.respond(message)
returnValue(STOP_HANDLING)
else:
returnValue(CASCADE)
@inlineCallbacks
def beatmap(self, url, beatmap):
fragment = self.parse_fragment(url)
params = {}
if url.query:
params.update(url.query)
if fragment:
params.update(fragment)
params["b"] = beatmap
r = yield self.get(URL_BEATMAPS, params=params)
beatmap = r.json()[0]
if "m" not in params:
params["m"] = beatmap["mode"]
for key in ["favourite_count", "playcount", "passcount"]:
beatmap[key] = locale.format("%d",
int(beatmap[key]),
grouping=True)
for key in ["difficultyrating"]:
beatmap[key] = int(round(float(beatmap[key])))
if "approved" in beatmap:
beatmap["approved"] = OSU_APPROVALS.get(beatmap["approved"],
u"Unknown approval")
beatmap["mode"] = OSU_MODES[beatmap["mode"]]
scores = None
try:
r = yield self.get(URL_SCORES, params=params)
scores = r.json()
for score in scores:
for key in [
"score", "count50", "count100", "count300",
"countmiss", "countkatu", "countgeki"
]:
score[key] = locale.format("%d",
int(score[key]),
grouping=True)
for key in ["pp"]:
score[key] = int(round(float(score[key])))
score["enabled_mods"] = ", ".join(
get_mods(int(score["enabled_mods"])))
except Exception:
pass
data = beatmap
if beatmap["approved"] in [
u"Pending", u"WIP", u"Graveyard", u"Unknown approval"
]:
message = self.get_string("beatmap-unapproved")
elif scores is None:
message = self.get_string("beatmap-mode-mismatch")
elif not scores:
message = self.get_string("beatmap-no-scores")
else:
data["scores"] = scores
message = self.get_string("beatmap")
returnValue(message.format(**data))
@inlineCallbacks
def mapset(self, url, mapset):
params = {"s": mapset}
r = yield self.get(URL_BEATMAPS, params=params)
data = r.json()
modes = {}
to_join = []
for beatmap in data:
modes[beatmap["mode"]] = modes.get(beatmap["mode"], 0) + 1
beatmap["mode"] = OSU_MODES[beatmap["mode"]]
for key in ["favourite_count", "playcount", "passcount"]:
beatmap[key] = locale.format("%d",
int(beatmap[key]),
grouping=True)
for key in ["difficultyrating"]:
beatmap[key] = int(round(float(beatmap[key])))
if "approved" in beatmap:
beatmap["approved"] = OSU_APPROVALS.get(
beatmap["approved"],
u"Unknown approval: {}".format(beatmap["approved"]))
for k, v in modes.iteritems():
if v:
to_join.append("{} x{}".format(OSU_MODES[k], v))
first = data[0]
data = {"beatmaps": data, "counts": ", ".join(to_join)}
data.update(first)
returnValue(self.get_string("mapset").format(**data))
@inlineCallbacks
def user(self, url, user):
fragment = self.parse_fragment(url)
params = {
"u": user,
}
if "m" in fragment: # Focused mode
m = fragment["m"].lower()
if m in OSU_MODES:
params["m"] = OSU_MODES[m]
else:
try:
params["m"] = int(m)
except ValueError:
pass
# This logic is down to being able to specify either a username or ID.
# The osu backend has to deal with this and so the api lets us specify
# either "string" or "id" for usernames and IDs respectively. This
# may be useful for usernames that are numerical, so we allow users
# to add this to the fragment if they wish.
if "t" in fragment: # This once was called "t"..
params["type"] = fragment["t"]
elif "type" in fragment: # ..but now is "type" for some reason
params["type"] = fragment["type"]
r = yield self.get(URL_USER, params=params)
data = r.json()[0] # It's a list for some reason
for key in ["level", "accuracy"]: # Round floats
data[key] = int(round(float(data[key])))
for key in [
"ranked_score", "pp_raw", "pp_rank", "count300", "count100",
"count50", "playcount", "total_score", "pp_country_rank"
]: # Localisé number formatting
data[key] = locale.format("%d", int(data[key]), grouping=True)
epic_factors = [int(event["epicfactor"]) for event in data["events"]]
epic_total = reduce(sum, epic_factors, 0)
epic_avg = 0
if epic_total:
epic_avg = round(epic_total / (1.0 * len(epic_factors)), 2)
data["events"] = "{} events at an average of {}/32 epicness".format(
len(epic_factors), epic_avg)
returnValue(self.get_string("user").format(**data))
class FListHandler(URLHandler):
criteria = {
"protocol": re.compile(r"http|https", str_to_regex_flags("iu")),
"domain": re.compile(
r"(www\.f-list\.net)|(f-list\.net)",
str_to_regex_flags("iu")
),
"path": re.compile(r"/c/.*", str_to_regex_flags("iu")),
"permission": "urls.trigger.nsfw"
}
ticket = "" # API auth ticket; needs manual renewing
last_renewal = None # So we know when we renewed last
session = None
name = "f-list"
@property
def username(self):
return self.plugin.config.get("f-list", {}).get("username", "")
@property
def password(self):
return self.plugin.config.get("f-list", {}).get("password", "")
@property
def kinks_limit(self):
return self.plugin.config.get("f-list", {}).get("kink-sample", 2)
def __init__(self, plugin):
super(FListHandler, self).__init__(plugin)
if not (self.username and self.password):
raise ApiKeyMissing()
self.reload()
self.get_ticket()
def reload(self):
self.teardown()
self.session = Session()
def teardown(self):
if self.session is not None:
self.session.close()
def get_string(self, string):
formatting = self.plugin.config.get("osu", {}).get("formatting", {})
if string not in formatting:
return strings[string]
return formatting[string]
@inlineCallbacks
def get(self, *args, **kwargs):
r = yield self.session.get(*args, **kwargs)
data = r.json()
if "error" in data and data["error"]:
raise FListError(data["error"])
returnValue(data)
@inlineCallbacks
def post(self, *args, **kwargs):
r = yield self.session.post(*args, **kwargs)
data = r.json()
if "error" in data and data["error"]:
raise FListError(data["error"])
returnValue(data)
@inlineCallbacks
def get_ticket(self):
now = datetime.now()
then = now - timedelta(minutes=4)
if not self.last_renewal or then > self.last_renewal:
data = yield self.post(
URL_TICKET, params={
"account": self.username,
"password": self.password
}
)
self.ticket = data["ticket"]
self.last_renewal = datetime.now()
returnValue(self.ticket)
def get_sample(self, items, count):
if not items:
return ["Nothing"]
if len(items) <= count:
return items
return [i for i in random.sample(items, count)]
@inlineCallbacks
def call(self, url, context):
target = url.path
while target.endswith("/"):
target = target[:-1]
target = target.split("/")
if "" in target:
target.remove("")
if " " in target:
target.remove(" ")
message = ""
try:
if len(target) < 2: # It's the front page or invalid, don't bother
returnValue(CASCADE)
elif target[0].lower() == "c": # Character page
message = yield self.character(target[1])
except Exception:
self.plugin.logger.exception("Error handling URL: {}".format(url))
returnValue(CASCADE)
# At this point, if `message` isn't set then we don't understand the
# url, and so we'll just allow it to pass down to the other handlers
if message:
context["event"].target.respond(message)
returnValue(STOP_HANDLING)
else:
returnValue(CASCADE)
@inlineCallbacks
def character(self, char_name):
char_name = urlparse.unquote(char_name)
ticket = yield self.get_ticket()
params = {
"ticket": ticket,
"name": char_name,
"account": self.username
}
char_info = yield self.post(URL_CHAR_INFO, params=params)
char_kinks = yield self.post(URL_CHAR_KINKS, params=params)
char_info = flatten_character(char_info)
char_kinks = flatten_kinks(char_kinks)
data = char_info["info"]
data["sample_kinks"] = {
"fave": ", ".join(self.get_sample(
char_kinks["preferences"]["fave"], self.kinks_limit
)),
"yes": ", ".join(self.get_sample(
char_kinks["preferences"]["yes"], self.kinks_limit
)),
"maybe": ", ".join(self.get_sample(
char_kinks["preferences"]["maybe"], self.kinks_limit
)),
"no": ", ".join(self.get_sample(
char_kinks["preferences"]["no"], self.kinks_limit
)),
}
data["given"] = {
"name": char_name
}
returnValue(
self.get_string("character").format(**data).replace(u"&", u"&")
)