def _akari_fetch_retry(self, user, listurl, listname, times=5):
"""
Jikan.moe is susceptible to randomly failing. This method allows us to try multiple times before really "failing"
Params: See self._akari_list
Returns: See self._akari_list if successful, or raises an Exception() otherwise
"""
for i in range(times):
try:
LoggingUtils.debug(
"Attempt #{} to contact Jikan.moe for {}{}{}".format(
i + 1, CYAN, listname, ENDC))
anime = self._akari_list(user, listurl, listname)
LoggingUtils.debug("Attempt #{} {}succeeded{}".format(
i + 1, GREEN, ENDC))
return anime
except:
# Sleep 5 seconds, and then try again
LoggingUtils.debug(
"Attempt #{} {}failed{}, sleeping 5 seconds and trying again..."
.format(i + 1, RED, ENDC))
time.sleep(5)
# If this point is reached, then there has been too many errors. Raise an exception
LoggingUtils.error("Akari was unable to contact Jikan.moe")
raise Exception()
def _kitsu_basic_search(self, title):
"""
This is a quick Kitsu search implementation from the Hitsu 2A module.
Params:
title - the title of the show (in provided request) to search for
Returns: Kitsu's JSON response
"""
title_lower = title.lower()
request_url = requests.utils.requote_uri(KITSU_API_URL + title_lower)
LoggingUtils.debug("Created Kitsu url - {}".format(request_url))
try:
kitsu_res = requests.get(request_url)
try:
kitsu_res_json = kitsu_res.json()
except:
LoggingUtils.warning("Kitsu response did not properly parse into JSON", color=LoggingUtils.YELLOW)
raise Exception()
return kitsu_res_json
except:
LoggingUtils.error("There was an error when attempting to contact Kitsu", color=LoggingUtils.RED)
raise Exception()
def create_from_json(data: dict) -> Job:
try:
LoggingUtils.debug(
"Creating new Job instance with the following info:",
color=LoggingUtils.MAGENTA)
LoggingUtils.debug("Job.show: {}".format(data['show']),
color=LoggingUtils.MAGENTA)
LoggingUtils.debug("Job.episode: {}".format(data['episode']),
color=LoggingUtils.MAGENTA)
LoggingUtils.debug("Job.filesize: {}".format(data['filesize']),
color=LoggingUtils.MAGENTA)
LoggingUtils.debug("Job.sub: {}".format(data['sub']),
color=LoggingUtils.MAGENTA)
return Job(data['show'], data['episode'], int(data['filesize']),
data['sub'])
except:
LoggingUtils.error(
"Failed to create Job from request - json body is malformed",
color=LoggingUtils.RED)
return None
def _anilist(self, query, search, status):
"""
This helper method handles making requests to Anilist
Returns the response in JSON form
Params:
query: The type of query (single, page) to request for
search: the name of the show to search for
status: the status of the show to search for
"""
try:
# Make request to Anlist and substitute the variables properly
ani = requests.post(self._API_URL,
json={'query': query,
'variables': {
'search': search,
'status': status}
})
if ani.status_code != 200:
LoggingUtils.warning("Anilist returned a bad HTTP code when attempting to connect")
raise Exception()
try:
# Try to get the response as a JSON object
ani_json = ani.json()
except:
LoggingUtils.warning("Anilist response did not properly parse into JSON", color=LoggingUtils.YELLOW)
raise Exception()
# Return the data provided by the request response
return ani_json['data']
except:
LoggingUtils.error("There was an error when attempting to contact Anilist", color=LoggingUtils.RED)
raise Exception()
def work(self,
burst=False,
logging_level="INFO",
date_format=DEFAULT_LOGGING_DATE_FORMAT,
log_format=DEFAULT_LOGGING_FORMAT,
max_jobs=None):
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
setup_loghandlers(logging_level, date_format, log_format)
self._install_signal_handlers()
completed_jobs = 0
self.register_birth()
self.log.info("Worker %s: started, version %s", self.key, VERSION)
#LoggingUtils.info("Worker {}: started, version {}".format(self.key, VERSION))
self.set_state(WorkerStatus.STARTED)
qnames = self.queue_names()
self.log.info('*** Listening on %s...', green(', '.join(qnames)))
# LoggingUtils.info("*** Listening on {}...".format(', '.join(qnames)), color=LoggingUtils.LGREEN)
try:
while True:
try:
self.check_for_suspension(burst)
if self.should_run_maintenance_tasks:
self.clean_registries()
if self._stop_requested:
self.log.info('Worker %s: stopping on request',
self.key)
LoggingUtils.info(
'Worker {}: stopping on request'.format(self.key),
color=LoggingUtils.LCYAN)
break
timeout = None if burst else max(
1, self.default_worker_ttl - 15)
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None:
if burst:
self.log.info("Worker %s: done, quitting",
self.key)
LoggingUtils.info(
"Worker {}: done, quitting".format(self.key),
color=LoggingUtils.LCYAN)
break
job, queue = result
self.execute_job(job, queue)
self.heartbeat()
completed_jobs += 1
if max_jobs is not None:
if completed_jobs >= max_jobs:
self.log.info(
"Worker %s: finished executing %d jobs, quitting",
self.key, completed_jobs)
LoggingUtils.info(
"Worker {}: finished executing {} jobs, quitting"
.format(self.key, completed_jobs))
break
except StopRequested:
#break
raise WorkerCancelledError
except SystemExit:
# Cold shutdown detected
#raise
raise WorkerCancelledError
# These are our custom changes
except TimeoutError:
# This is an expected error thrown by us almost always
# Catch it in the external loop so we can re-raise to not get stuck in a loop
raise TimeoutError
except: # noqa
self.log.error(
'Worker %s: found an unhandled exception, quitting...',
self.key,
exc_info=True)
LoggingUtils.error(
"Worker {}: found an unhandled exception, quitting...".
format(self.key))
break
except WorkerCancelledError:
if not self.is_horse:
self.register_death()
raise WorkerCancelledError()
except TimeoutError:
if not self.is_horse:
self.register_death()
raise TimeoutError()
finally:
if not self.is_horse:
self.register_death()
return bool(completed_jobs)
def _akari_list(self, user, listurl, listname):
"""
Helper method that gets a user's specific anime list
!! Doesn't handle the random errors thrown by Jikan.moe
Params:
user: String, is the username of the MAL user
listurl: The URL to use when fetching information
listname: Watching/Plan To Watch/etc
Returns a list that holds a bunch of anime entries
"""
anime = list() # Stores every anime entry that is paginated
# Step 1: Get the first page (which is assumed to exist)
try:
jikan_res = requests.get(listurl.format(user, ""))
# Check the Status code
if jikan_res.status_code != 200:
LoggingUtils.debug(
"Jikan.moe returned a bad status code when attempting to get {}'s {} list"
.format(user, listname),
color=LoggingUtils.YELLOW)
raise Exception()
# Make sure there actually was an anime
try:
jikan_res_json = jikan_res.json()
except:
jikan_res_json = dict() # For clean handling
if 'anime' not in jikan_res_json:
LoggingUtils.debug(
"Jikan.moe did not have an anime list when attempting to get {}'s {} list page"
.format(user, listname),
color=LoggingUtils.YELLOW)
raise Exception()
# Add all anime in the first page
for entry in jikan_res_json['anime']:
anime.append(entry)
LoggingUtils.debug("Added {} show: {}".format(
listname, entry['title']))
# Now, process the rest of pages that are there
page = 2
while (len(jikan_res_json['anime']) == 300):
jikan_res = requests.get(listurl.format(user, str(page)))
if jikan_res.status_code != 200 or 'anime' not in jikan_res:
LoggingUtils.debug(
"Jikan.moe returned a bad status code when attempting to get {}'s {} list"
.format(user, listname))
raise Exception()
try:
jikan_res_json = jikan_res.json()
except:
jikan_res_json = dict()
if 'anime' not in jikan_res_json:
LoggingUtils.debug(
"Jikan.moe did not have an anime list when attempting to get {}'s {} list page"
.format(user, listname))
raise Exception()
for entry in jikan_res_json['anime']:
anime.append(entry)
LoggingUtils.debug("Added {} show: {}".format(
listname, entry['title']))
page += 1
return anime
except:
# raise some kind of exception - somehow Jikan couldn't be reached
LoggingUtils.error(
"Akari encountered an error when attempting to fetch {}'s {} list"
.format(user, listname),
color=LoggingUtils.RED)
raise Exception()
