start_id = int(args.start) end_id = int(args.end) print "ID range: " + str(start_id) + " to " + str(end_id) if start_id >= end_id: print "ID range is empty, halting." sys.exit(0) if int(args.per_minute) < 1 or int(args.per_minute) > 60: print "Users per minute must be between 1 and 60, inclusive" sys.exit(0) list_insert_queue = DbConn.DbInsertQueue(db, table='mal_anime_lists', fields=[ 'user_id', 'anime_id', 'started', 'time', 'finished', 'status', 'score', 'episode' ]).update('started=VALUES(started), time=VALUES(time), finished=VALUES(finished), status=VALUES(status), score=VALUES(score), episode=VALUES(episode)') mal_statuses_to_int = { 'Unknown': 0, 'Watching': 1, 'Completed': 2, 'On-Hold': 3, 'Dropped': 4, 'Plan to Watch': 6 } sleep_time = 60 / int(args.per_minute)
def import_mal_profiles(self): ''' Processes the MAL profile scraping queue. ''' if (datetime.datetime.now(tz=pytz.utc) - self.info['last_run_time']) < datetime.timedelta(hours=1): return self.info['last_run_time'] = datetime.datetime.now(tz=pytz.utc) self.daemon.log.info("Processing MAL profile queue.") mal_session = myanimelist.session.Session() import_requests = self.dbs['animurecs'].table('users').fields( 'id', 'mal_username', 'last_import').where('mal_username IS NOT NULL').where( 'last_import_failed = 0').order('last_import ASC').list() entries_to_add = [] entry_insert_queue = DbConn.DbInsertQueue( self.dbs['animurecs'], 'anime_lists', ['user_id', 'anime_id', 'time', 'status', 'score', 'episode' ]).ignore(True) requested_ids = map(lambda r: int(r['id']), import_requests) broken_ids = [] for request in import_requests: # process MAL profile import request. self.daemon.log.info("Processing MAL profile for user ID " + str(request['id']) + ".") try: anime_list = mal_session.anime_list( request['mal_username']).load() except myanimelist.media_list.InvalidMediaListError as e: self.daemon.log.error("Invalid MAL username provided: " + request['mal_username'] + ". Marking as broken and skipping.") broken_ids.append(int(request['id'])) continue curr_time = datetime.datetime.now(tz=pytz.timezone( self.daemon.config['timezone'])).strftime('%Y-%m-%d %H:%M:%S') for anime in anime_list.list: if anime_list.list[anime]['score'] is None: anime_list.list[anime]['score'] = 0 entry_insert_queue.queue({ 'user_id': request['id'], 'anime_id': int(anime.id), 'time': pytz.timezone(self.daemon.config['timezone']).localize( anime_list.list[anime]['last_updated']).strftime( '%Y-%m-%d %H:%M:%S'), 'status': self.mal_statuses_to_int[anime_list.list[anime]['status']], 'score': anime_list.list[anime]['score'], 'episode': anime_list.list[anime]['episodes_watched'] }) # flush insert queue. entry_insert_queue.flush() # update last-import time and broken usernames. if requested_ids: self.dbs['animurecs'].table('users').set( last_import=datetime.datetime.now( tz=pytz.timezone(self.daemon.config['timezone'])).strftime( '%Y-%m-%d %H:%M:%S')).where(id=requested_ids).update() if broken_ids: self.dbs['animurecs'].table('users').set( last_import_failed=1).where(id=broken_ids).update() self.daemon.log.info("Inserted entries for " + str(len(requested_ids)) + " users.")