start_id = int(args.start)
end_id = int(args.end)
print "ID range: " + str(start_id) + " to " + str(end_id)
if start_id >= end_id:
print "ID range is empty, halting."
sys.exit(0)
if int(args.per_minute) < 1 or int(args.per_minute) > 60:
print "Users per minute must be between 1 and 60, inclusive"
sys.exit(0)
list_insert_queue = DbConn.DbInsertQueue(db, table='mal_anime_lists', fields=[
'user_id',
'anime_id',
'started',
'time',
'finished',
'status',
'score',
'episode'
]).update('started=VALUES(started), time=VALUES(time), finished=VALUES(finished), status=VALUES(status), score=VALUES(score), episode=VALUES(episode)')
mal_statuses_to_int = {
'Unknown': 0,
'Watching': 1,
'Completed': 2,
'On-Hold': 3,
'Dropped': 4,
'Plan to Watch': 6
}
sleep_time = 60 / int(args.per_minute)
def import_mal_profiles(self):
'''
Processes the MAL profile scraping queue.
'''
if (datetime.datetime.now(tz=pytz.utc) -
self.info['last_run_time']) < datetime.timedelta(hours=1):
return
self.info['last_run_time'] = datetime.datetime.now(tz=pytz.utc)
self.daemon.log.info("Processing MAL profile queue.")
mal_session = myanimelist.session.Session()
import_requests = self.dbs['animurecs'].table('users').fields(
'id', 'mal_username',
'last_import').where('mal_username IS NOT NULL').where(
'last_import_failed = 0').order('last_import ASC').list()
entries_to_add = []
entry_insert_queue = DbConn.DbInsertQueue(
self.dbs['animurecs'], 'anime_lists',
['user_id', 'anime_id', 'time', 'status', 'score', 'episode'
]).ignore(True)
requested_ids = map(lambda r: int(r['id']), import_requests)
broken_ids = []
for request in import_requests:
# process MAL profile import request.
self.daemon.log.info("Processing MAL profile for user ID " +
str(request['id']) + ".")
try:
anime_list = mal_session.anime_list(
request['mal_username']).load()
except myanimelist.media_list.InvalidMediaListError as e:
self.daemon.log.error("Invalid MAL username provided: " +
request['mal_username'] +
". Marking as broken and skipping.")
broken_ids.append(int(request['id']))
continue
curr_time = datetime.datetime.now(tz=pytz.timezone(
self.daemon.config['timezone'])).strftime('%Y-%m-%d %H:%M:%S')
for anime in anime_list.list:
if anime_list.list[anime]['score'] is None:
anime_list.list[anime]['score'] = 0
entry_insert_queue.queue({
'user_id':
request['id'],
'anime_id':
int(anime.id),
'time':
pytz.timezone(self.daemon.config['timezone']).localize(
anime_list.list[anime]['last_updated']).strftime(
'%Y-%m-%d %H:%M:%S'),
'status':
self.mal_statuses_to_int[anime_list.list[anime]['status']],
'score':
anime_list.list[anime]['score'],
'episode':
anime_list.list[anime]['episodes_watched']
})
# flush insert queue.
entry_insert_queue.flush()
# update last-import time and broken usernames.
if requested_ids:
self.dbs['animurecs'].table('users').set(
last_import=datetime.datetime.now(
tz=pytz.timezone(self.daemon.config['timezone'])).strftime(
'%Y-%m-%d %H:%M:%S')).where(id=requested_ids).update()
if broken_ids:
self.dbs['animurecs'].table('users').set(
last_import_failed=1).where(id=broken_ids).update()
self.daemon.log.info("Inserted entries for " +
str(len(requested_ids)) + " users.")
