def __addMoviesToTraktWatched(self, kodiMovies, traktMovies, fromPercent,
toPercent):
if utilities.getSettingAsBool(
'trakt_movie_playcount') and not self.sync.IsCanceled():
updateTraktTraktMovies = copy.deepcopy(traktMovies)
updateTraktKodiMovies = copy.deepcopy(kodiMovies)
traktMoviesToUpdate = self.__compareMovies(updateTraktKodiMovies,
updateTraktTraktMovies,
watched=True)
self.sanitizeMovies(traktMoviesToUpdate)
if len(traktMoviesToUpdate) == 0:
self.sync.UpdateProgress(toPercent,
line2=utilities.getString(32086))
logger.debug(
"[Movies Sync] Trakt.tv movie playcount is up to date")
return
titles = ", ".join(
["%s" % (m['title']) for m in traktMoviesToUpdate])
logger.debug(
"[Movies Sync] %i movie(s) playcount will be updated on Trakt.tv"
% len(traktMoviesToUpdate))
logger.debug("[Movies Sync] Movies updated: %s" % titles)
self.sync.UpdateProgress(fromPercent,
line2=utilities.getString(32064) %
len(traktMoviesToUpdate))
# Send request to update playcounts on Trakt.tv
chunksize = 200
chunked_movies = utilities.chunks(
[movie for movie in traktMoviesToUpdate], chunksize)
errorcount = 0
i = 0
x = float(len(traktMoviesToUpdate))
for chunk in chunked_movies:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent - fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y),
line2=utilities.getString(32093) %
((i) * chunksize if
(i) * chunksize < x else x, x))
params = {'movies': chunk}
# logger.debug("moviechunk: %s" % params)
try:
self.sync.traktapi.addToHistory(params)
except Exception as ex:
message = utilities.createError(ex)
logging.fatal(message)
errorcount += 1
logger.debug("[Movies Sync] Movies updated: %d error(s)" %
errorcount)
self.sync.UpdateProgress(toPercent,
line2=utilities.getString(32087) %
len(traktMoviesToUpdate))
def get_month_assignments():
"""Get all canvas assignments in all courses for the next month"""
courses = get_all_courses()
course_contexts = ['course_' + str(course.id) for course in courses]
# set date range for getting assignments
date_now = datetime.datetime.utcnow()
month_delta = datetime.timedelta(days=365 / 12)
day_delta = datetime.timedelta(days=1)
next_month = date_now + month_delta
yesterday = date_now - day_delta
# Canvas API only takes at max 10 contexts
# break up contexts into chunks
course_chunks = list(chunks(course_contexts, CHUNK_SIZE))
all_assignments: List[CanvasAssignment] = list()
# for each chunk, request the assignment calendar events
for course_context_chunk in course_chunks:
cal_params = {
'type': 'assignment',
'context_codes[]': course_context_chunk,
'start_date': yesterday.isoformat(),
'end_date': next_month.isoformat(),
'per_page': PER_PAGE,
}
assignment_events_res = SESSION.get(config["CANVAS_URL"] +
'/api/v1/calendar_events',
params=cal_params)
#pull the assignment out of the assignment event and put them into the list
all_assignments.extend([
CanvasAssignment.from_dict(event['assignment'])
for event in assignment_events_res.json()
])
# assignments
return all_assignments
def __kodiUpdateEpisodes(self, shows):
if len(shows['shows']) == 0:
self.__updateProgress(82, line1=utilities.getString(1441), line2=utilities.getString(1493))
Debug("[Episodes Sync] Kodi episode playcounts are up to date.")
return
Debug("[Episodes Sync] %i show(s) shows are missing playcounts on Kodi" % len(shows['shows']))
for s in ["%s" % self.__getShowAsString(s, short=True) for s in shows['shows']]:
Debug("[Episodes Sync] Episodes updated: %s" % s)
self.__updateProgress(64, line1=utilities.getString(1441), line2="%i %s" % (len(shows['shows']), utilities.getString(1439)), line3=" ")
episodes = []
for show in shows['shows']:
for season in show['seasons']:
for episode in season['episodes']:
episodes.append({'episodeid': episode['ids']['episodeid'], 'playcount': 1})
#split episode list into chunks of 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "method": "VideoLibrary.SetEpisodeDetails", "params": episodes[i], "id": i} for i in range(len(episodes))], 50)
i = 0
x = float(len(chunked_episodes))
for chunk in chunked_episodes:
if self.__isCanceled():
return
Debug("[Episodes Sync] chunk %s" % str(chunk))
result = utilities.kodiJsonRequest(chunk)
Debug("[Episodes Sync] result %s" % str(result))
i += 1
y = ((i / x) * 18) + 64
self.__updateProgress(int(y), line2=utilities.getString(1494))
self.__updateProgress(82, line2=utilities.getString(1495) % len(episodes))
def __addMoviesToKodiWatched(self, traktMovies, kodiMovies):
if utilities.getSettingAsBool('kodi_movie_playcount') and not self.__isCanceled():
updateKodiTraktMovies = copy.deepcopy(traktMovies)
updateKodiKodiMovies = copy.deepcopy(kodiMovies)
kodiMoviesToUpdate = self.__compareMovies(updateKodiTraktMovies, updateKodiKodiMovies, watched=True, restrict=True)
if len(kodiMoviesToUpdate) == 0:
self.__updateProgress(84, line2=utilities.getString(32088))
logger.debug("[Movies Sync] Kodi movie playcount is up to date.")
return
titles = ", ".join(["%s" % (m['title']) for m in kodiMoviesToUpdate])
logger.debug("[Movies Sync] %i movie(s) playcount will be updated in Kodi" % len(kodiMoviesToUpdate))
logger.debug("[Movies Sync] Movies to add: %s" % titles)
self.__updateProgress(73, line2=utilities.getString(32065) % len(kodiMoviesToUpdate))
#split movie list into chunks of 50
chunksize = 50
chunked_movies = utilities.chunks([{"jsonrpc": "2.0", "method": "VideoLibrary.SetMovieDetails", "params": {"movieid": kodiMoviesToUpdate[i]['movieid'], "playcount": kodiMoviesToUpdate[i]['plays'], "lastplayed": utilities.convertUtcToDateTime(kodiMoviesToUpdate[i]['last_watched_at'])}, "id": i} for i in range(len(kodiMoviesToUpdate))], chunksize)
i = 0
x = float(len(kodiMoviesToUpdate))
for chunk in chunked_movies:
if self.__isCanceled():
return
i += 1
y = ((i / x) * 11) + 73
self.__updateProgress(int(y), line2=utilities.getString(32089) % ((i)*chunksize if (i)*chunksize < x else x, x))
utilities.kodiJsonRequest(chunk)
self.__updateProgress(84, line2=utilities.getString(32090) % len(kodiMoviesToUpdate))
def __traktUpdateMovies(self, movies):
if len(movies) == 0:
self.__updateProgress(60, line2=utilities.getString(1469))
Debug("[Movies Sync] trakt.tv movie playcount is up to date")
return
titles = ", ".join(["%s" % (m['title']) for m in movies])
Debug(
"[Movies Sync] %i movie(s) playcount will be updated on trakt.tv" %
len(movies))
Debug("[Movies Sync] Movies updated: %s" % titles)
self.__updateProgress(40,
line2="%i %s" %
(len(movies), utilities.getString(1428)))
# Send request to update playcounts on trakt.tv
chunked_movies = utilities.chunks([movie for movie in movies], 200)
i = 0
x = float(len(chunked_movies))
for chunk in chunked_movies:
if self.__isCanceled():
return
params = {'movies': chunk}
self.traktapi.addToHistory(params)
i += 1
y = ((i / x) * 20) + 40
self.__updateProgress(int(y), line2=utilities.getString(1478))
self.__updateProgress(60,
line2=utilities.getString(1470) % len(movies))
def __addMovieProgressToKodi(self, traktMovies, kodiMovies):
if utilities.getSettingAsBool('trakt_movie_playback') and traktMovies and not self.__isCanceled():
updateKodiTraktMovies = copy.deepcopy(traktMovies)
updateKodiKodiMovies = copy.deepcopy(kodiMovies)
kodiMoviesToUpdate = self.__compareMovies(updateKodiTraktMovies['movies'], updateKodiKodiMovies, restrict=True, playback=True)
if len(kodiMoviesToUpdate) == 0:
self.__updateProgress(99, line2=utilities.getString(32125))
logger.debug("[Movies Sync] Kodi movie playbacks are up to date.")
return
logger.debug("[Movies Sync] %i movie(s) playbacks will be updated in Kodi" % len(kodiMoviesToUpdate))
self.__updateProgress(85, line2=utilities.getString(32126) % len(kodiMoviesToUpdate))
#need to calculate the progress in int from progress in percent from Trakt
#split movie list into chunks of 50
chunksize = 50
chunked_movies = utilities.chunks([{"jsonrpc": "2.0", "id": i, "method": "VideoLibrary.SetMovieDetails", "params": {"movieid": kodiMoviesToUpdate[i]['movieid'], "resume": {"position": kodiMoviesToUpdate[i]['runtime']/100.0*kodiMoviesToUpdate[i]['progress']}}} for i in range(len(kodiMoviesToUpdate))], chunksize)
i = 0
x = float(len(kodiMoviesToUpdate))
for chunk in chunked_movies:
if self.__isCanceled():
return
i += 1
y = ((i / x) * 14) + 85
self.__updateProgress(int(y), line2=utilities.getString(32127) % ((i)*chunksize if (i)*chunksize < x else x, x))
utilities.kodiJsonRequest(chunk)
self.__updateProgress(99, line2=utilities.getString(32128) % len(kodiMoviesToUpdate))
def xbmcUpdateEpisodes(self, shows):
if len(shows) == 0:
self.updateProgress(82, line1=utilities.getString(1441), line2=utilities.getString(1493))
Debug("[Episodes Sync] XBMC episode playcounts are up to date.")
return
Debug("[Episodes Sync] %i show(s) shows are missing playcounts on XBMC" % len(shows))
for s in ["%s" % self.getShowAsString(s, short=True) for s in shows]:
Debug("[Episodes Sync] Episodes updated: %s" % s)
self.updateProgress(64, line1=utilities.getString(1441), line2="%i %s" % (len(shows), utilities.getString(1439)), line3=" ")
episodes = []
for show in shows:
for season in show['seasons']:
for episode in show['seasons'][season]:
episodes.append({'episodeid': show['seasons'][season][episode]['id'], 'playcount': 1})
#split episode list into chunks of 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "method": "VideoLibrary.SetEpisodeDetails", "params": episodes[i], "id": i} for i in range(len(episodes))], 50)
i = 0
x = float(len(chunked_episodes))
for chunk in chunked_episodes:
if self.isCanceled():
return
if self.simulate:
Debug("[Episodes Sync] %s" % str(chunk))
else:
utilities.xbmcJsonRequest(chunk)
i = i + 1
y = ((i / x) * 18) + 64
self.updateProgress(int(y), line2=utilities.getString(1494))
self.updateProgress(82, line2=utilities.getString(1495) % len(episodes))
def traktAddMovies(self, movies):
if len(movies) == 0:
self.updateProgress(40, line2=utilities.getString(1467))
Debug("[Movies Sync] trakt.tv movie collection is up to date.")
return
titles = ", ".join(["%s (%s)" % (m['title'], m['imdb_id']) for m in movies])
Debug("[Movies Sync] %i movie(s) will be added to trakt.tv collection." % len(movies))
Debug("[Movies Sync] Movies added: %s" % titles)
self.updateProgress(20, line2="%i %s" % (len(movies), utilities.getString(1426)))
chunked_movies = utilities.chunks([self.sanitizeMovieData(movie) for movie in movies], 50)
i = 0
x = float(len(chunked_movies))
for chunk in chunked_movies:
if self.isCanceled():
return
params = {'movies': chunk}
if self.simulate:
Debug("[Movies Sync] %s" % str(params))
else:
self.traktapi.addMovie(params)
i = i + 1
y = ((i / x) * 20) + 20
self.updateProgress(int(y), line2=utilities.getString(1477))
self.updateProgress(40, line2=utilities.getString(1468) % len(movies))
def traktRemoveMovies(self, movies):
if len(movies) == 0:
self.updateProgress(98, line2=utilities.getString(1474))
Debug("[Movies Sync] trakt.tv movie collection is clean, no movies to remove.")
return
titles = ", ".join(["%s (%s)" % (m['title'], m['imdb_id']) for m in movies])
Debug("[Movies Sync] %i movie(s) will be removed from trakt.tv collection." % len(movies))
Debug("[Movies Sync] Movies removed: %s" % titles)
self.updateProgress(80, line2="%i %s" % (len(movies), utilities.getString(1444)))
chunked_movies = utilities.chunks([self.sanitizeMovieData(movie) for movie in movies], 50)
i = 0
x = float(len(chunked_movies))
for chunk in chunked_movies:
if self.isCanceled():
return
params = {'movies': chunk}
if self.simulate:
Debug("[Movies Sync] %s" % str(params))
else:
self.traktapi.removeMovie(params)
i = i + 1
y = ((i / x) * 20) + 80
self.updateProgress(int(y), line2=utilities.getString(1476))
self.updateProgress(98, line2=utilities.getString(1475) % len(movies))
def traktUpdateMovies(self, movies):
if len(movies) == 0:
self.updateProgress(60, line2=utilities.getString(1469))
Debug("[Movies Sync] trakt.tv movie playcount is up to date")
return
titles = ", ".join(["%s (%s)" % (m['title'], m['imdb_id']) for m in movies])
Debug("[Movies Sync] %i movie(s) playcount will be updated on trakt.tv" % len(movies))
Debug("[Movies Sync] Movies updated: %s" % titles)
self.updateProgress(40, line2="%i %s" % (len(movies), utilities.getString(1428)))
# Send request to update playcounts on trakt.tv
chunked_movies = utilities.chunks([self.sanitizeMovieData(movie) for movie in movies], 50)
i = 0
x = float(len(chunked_movies))
for chunk in chunked_movies:
if self.isCanceled():
return
params = {'movies': chunk}
if self.simulate:
Debug("[Movies Sync] %s" % str(params))
else:
self.traktapi.updateSeenMovie(params)
i = i + 1
y = ((i / x) * 20) + 40
self.updateProgress(int(y), line2=utilities.getString(1478))
self.updateProgress(60, line2=utilities.getString(1470) % len(movies))
def xbmcUpdateMovies(self, movies):
if len(movies) == 0:
self.updateProgress(80, line2=utilities.getString(1471))
Debug("[Movies Sync] XBMC movie playcount is up to date.")
return
titles = ", ".join(["%s (%s)" % (m['title'], m['imdb_id']) for m in movies])
Debug("[Movies Sync] %i movie(s) playcount will be updated in XBMC" % len(movies))
Debug("[Movies Sync] Movies updated: %s" % titles)
self.updateProgress(60, line2="%i %s" % (len(movies), utilities.getString(1430)))
#split movie list into chunks of 50
chunked_movies = utilities.chunks([{"jsonrpc": "2.0", "method": "VideoLibrary.SetMovieDetails", "params": {"movieid": movies[i]['movieid'], "playcount": movies[i]['plays']}, "id": i} for i in range(len(movies))], 50)
i = 0
x = float(len(chunked_movies))
for chunk in chunked_movies:
if self.isCanceled():
return
if self.simulate:
Debug("[Movies Sync] %s" % str(chunk))
else:
utilities.xbmcJsonRequest(chunk)
i = i + 1
y = ((i / x) * 20) + 60
self.updateProgress(int(y), line2=utilities.getString(1472))
self.updateProgress(80, line2=utilities.getString(1473) % len(movies))
def tags():
page_title = 'Tag collection'
tags = Tag.query.all()
admin = False
if g.user is not None and g.user.is_authenticated():
admin = g.user.is_admin()
tags = list(chunks(tags, COLS_IN_TAG_TABLE))
return render_template('tags.html', page_title = page_title, tags = tags, admin = bool(admin))
def run_multiply_process(files, tokens_path, test_mode=False, cpu_number=None):
if cpu_number is None:
cpu_number = multiprocessing.cpu_count()
results = Pool(processes=cpu_number).starmap(change_file_batches, zip(chunks(files, cpu_number), repeat(tokens_path), repeat(test_mode)))
result = reduce(reduce_merge_result, results, Counter())
return result
def __addMoviesToKodiWatched(self, traktMovies, kodiMovies):
if utilities.getSettingAsBool(
'kodi_movie_playcount') and not self.__isCanceled():
updateKodiTraktMovies = copy.deepcopy(traktMovies)
updateKodiKodiMovies = copy.deepcopy(kodiMovies)
kodiMoviesToUpdate = self.__compareMovies(updateKodiTraktMovies,
updateKodiKodiMovies,
watched=True,
restrict=True)
if len(kodiMoviesToUpdate) == 0:
self.__updateProgress(99, line2=utilities.getString(32088))
logger.debug(
"[Movies Sync] Kodi movie playcount is up to date.")
return
titles = ", ".join(
["%s" % (m['title']) for m in kodiMoviesToUpdate])
logger.debug(
"[Movies Sync] %i movie(s) playcount will be updated in Kodi" %
len(kodiMoviesToUpdate))
logger.debug("[Movies Sync] Movies to add: %s" % titles)
self.__updateProgress(83,
line2=utilities.getString(32065) %
len(kodiMoviesToUpdate))
#split movie list into chunks of 50
chunksize = 50
chunked_movies = utilities.chunks([{
"jsonrpc": "2.0",
"method": "VideoLibrary.SetMovieDetails",
"params": {
"movieid": kodiMoviesToUpdate[i]['movieid'],
"playcount": kodiMoviesToUpdate[i]['plays']
},
"id": i
} for i in range(len(kodiMoviesToUpdate))], chunksize)
i = 0
x = float(len(kodiMoviesToUpdate))
for chunk in chunked_movies:
if self.__isCanceled():
return
i += 1
y = ((i / x) * 16) + 83
self.__updateProgress(int(y),
line2=utilities.getString(32089) %
((i) * chunksize if
(i) * chunksize < x else x, x))
utilities.kodiJsonRequest(chunk)
self.__updateProgress(99,
line2=utilities.getString(32090) %
len(kodiMoviesToUpdate))
def UpdatePlaysXBMC(self):
Debug("[Movies Sync] Checking if XBMC playcount is up to date")
if self.show_progress:
progress.update(85, line1=__getstring__(1429), line2=" ", line3=" ")
update_playcount = []
trakt_playcounts = {}
for trakt_movie in self.trakt_movies_seen:
if "tmdb_id" in trakt_movie:
trakt_playcounts[trakt_movie["tmdb_id"]] = trakt_movie["plays"]
if "imdb_id" in trakt_movie:
trakt_playcounts[trakt_movie["imdb_id"]] = trakt_movie["plays"]
trakt_playcounts[trakt_movie["title"]] = trakt_movie["plays"]
for xbmc_movie in self.xbmc_movies:
if xbmc_movie["imdbnumber"] in trakt_playcounts:
if trakt_playcounts[xbmc_movie["imdbnumber"]] > xbmc_movie["playcount"]:
xbmc_movie["playcount"] = trakt_playcounts[xbmc_movie["imdbnumber"]]
update_playcount.append(xbmc_movie)
elif xbmc_movie["title"] in trakt_playcounts:
if trakt_playcounts[xbmc_movie["title"]] > xbmc_movie["playcount"]:
xbmc_movie["playcount"] = trakt_playcounts[xbmc_movie["title"]]
update_playcount.append(xbmc_movie)
if update_playcount:
Debug("[Movies Sync] %i movie(s) playcount will be updated on XBMC" % len(update_playcount))
if self.show_progress:
progress.update(90, line2="%i %s" % (len(update_playcount), __getstring__(1430)))
# split movie list into chunks of 50
chunked_movies = chunks(
[
{
"jsonrpc": "2.0",
"method": "VideoLibrary.SetMovieDetails",
"params": {
"movieid": update_playcount[i]["movieid"],
"playcount": update_playcount[i]["playcount"],
},
"id": i,
}
for i in range(len(update_playcount))
],
50,
)
for chunk in chunked_movies:
if self.Canceled():
return
xbmcJsonRequest(chunk)
else:
Debug("[Movies Sync] XBMC movie playcount is up to date")
def tags():
page_title = 'Tag collection'
tags = Tag.query.all()
admin = False
if g.user is not None and g.user.is_authenticated():
admin = g.user.is_admin()
tags = list(chunks(tags, COLS_IN_TAG_TABLE))
return render_template('tags.html',
page_title=page_title,
tags=tags,
admin=bool(admin))
def update_summoner_names(riot_cache, riot_connection, queued_counts, min_players=100, chunk_size=40):
logger = logging.getLogger(__name__)
max_players = max(min_players, queued_counts["player"] * 2) * 40
logger.info("Fetching summoner names, up to %d", max_players)
players = []
for player in riot_cache.get_queued_players(max_players):
players.append(player)
for player_chunk in chunks(players, chunk_size):
players = riot_connection.get_summoners(ids=[p.id for p in player_chunk])
riot_cache.update_player_names(players)
def add_entry():
form = EntryForm(request.form)
page_title = 'Add new blog entry'
if request.method == 'POST' and form.validate_on_submit():
entry = Entry()
entry.user_id = g.user.user_id
entry.create_time = datetime.datetime.utcnow()
form.populate_obj(entry)
db.session.add(entry)
db.session.commit()
return redirect(url_for('entries'))
tags = Tag.query.all()
tags = list(chunks(tags, COLS_IN_TAG_TABLE))
return render_template('entry_editor.html', form_action = 'add_entry', form = form, tags = tags, page_title = page_title)
def __syncEpisodeRatings(self, traktShows, kodiShows, fromPercent, toPercent):
if kodiUtilities.getSettingAsBool('trakt_sync_ratings') and traktShows and not self.sync.IsCanceled():
updateKodiTraktShows = copy.deepcopy(traktShows)
updateKodiKodiShows = copy.deepcopy(kodiShows)
traktShowsToUpdate = self.__compareEpisodes(updateKodiKodiShows, updateKodiTraktShows, rating=True)
if len(traktShowsToUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1='', line2=kodiUtilities.getString(32181))
logger.debug("[Episodes Sync] Trakt episode ratings are up to date.")
else:
logger.debug("[Episodes Sync] %i show(s) will have episode ratings added on Trakt" % len(traktShowsToUpdate['shows']))
self.sync.UpdateProgress(fromPercent, line1='', line2=kodiUtilities.getString(32182) % len(traktShowsToUpdate['shows']))
self.sync.traktapi.addRating(traktShowsToUpdate)
kodiShowsUpdate = self.__compareEpisodes(updateKodiTraktShows, updateKodiKodiShows, restrict=True, rating=True)
if len(kodiShowsUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1='', line2=kodiUtilities.getString(32173))
logger.debug("[Episodes Sync] Kodi episode ratings are up to date.")
else:
logger.debug("[Episodes Sync] %i show(s) will have episode ratings added in Kodi" % len(kodiShowsUpdate['shows']))
for s in ["%s" % self.__getShowAsString(s, short=True) for s in kodiShowsUpdate['shows']]:
logger.debug("[Episodes Sync] Episodes updated: %s" % s)
episodes = []
for show in kodiShowsUpdate['shows']:
for season in show['seasons']:
for episode in season['episodes']:
episodes.append({'episodeid': episode['ids']['episodeid'], 'rating': episode['rating']})
# split episode list into chunks of 50
chunksize = 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "id": i, "method": "VideoLibrary.SetEpisodeDetails",
"params": {"episodeid": episodes[i]['episodeid'],
"userrating": episodes[i]['rating']}} for i in range(len(episodes))],
chunksize)
i = 0
x = float(len(episodes))
for chunk in chunked_episodes:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent-fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y), line1='', line2=kodiUtilities.getString(32174) % ((i) * chunksize if (i) * chunksize < x else x, x))
kodiUtilities.kodiJsonRequest(chunk)
self.sync.UpdateProgress(toPercent, line2=kodiUtilities.getString(32175) % len(episodes))
def __syncEpisodeRatings(self, traktShows, kodiShows, fromPercent, toPercent):
if utilities.getSettingAsBool('trakt_sync_ratings') and traktShows and not self.sync.IsCanceled():
updateKodiTraktShows = copy.deepcopy(traktShows)
updateKodiKodiShows = copy.deepcopy(kodiShows)
traktShowsToUpdate = self.__compareEpisodes(updateKodiKodiShows, updateKodiTraktShows, rating=True)
if len(traktShowsToUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1='', line2=utilities.getString(32181))
logger.debug("[Episodes Sync] Trakt episode ratings are up to date.")
else:
logger.debug("[Episodes Sync] %i show(s) will have episode ratings added on Trakt" % len(traktShowsToUpdate['shows']))
self.sync.UpdateProgress(fromPercent, line1='', line2=utilities.getString(32182) % len(traktShowsToUpdate['shows']))
self.sync.traktapi.addRating(traktShowsToUpdate)
kodiShowsUpdate = self.__compareEpisodes(updateKodiTraktShows, updateKodiKodiShows, restrict=True, rating=True)
if len(kodiShowsUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1='', line2=utilities.getString(32173))
logger.debug("[Episodes Sync] Kodi episode ratings are up to date.")
else:
logger.debug("[Episodes Sync] %i show(s) will have episode ratings added in Kodi" % len(kodiShowsUpdate['shows']))
for s in ["%s" % self.__getShowAsString(s, short=True) for s in kodiShowsUpdate['shows']]:
logger.debug("[Episodes Sync] Episodes updated: %s" % s)
episodes = []
for show in kodiShowsUpdate['shows']:
for season in show['seasons']:
for episode in season['episodes']:
episodes.append({'episodeid': episode['ids']['episodeid'], 'rating': episode['rating']})
# split episode list into chunks of 50
chunksize = 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "id": i, "method": "VideoLibrary.SetEpisodeDetails",
"params": {"episodeid": episodes[i]['episodeid'],
"userrating": episodes[i]['rating']}} for i in range(len(episodes))],
chunksize)
i = 0
x = float(len(episodes))
for chunk in chunked_episodes:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent-fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y), line1='', line2=utilities.getString(32174) % ((i) * chunksize if (i) * chunksize < x else x, x))
utilities.kodiJsonRequest(chunk)
self.sync.UpdateProgress(toPercent, line2=utilities.getString(32175) % len(episodes))
def get_caches(self, codes, fields):
print("-- downloading cache details...")
url = self._base_url + 'services/caches/geocaches'
caches = {}
for code_chunk in chunks(codes, 100):
data = {
'consumer_key': self._consumer_key,
'cache_codes': '|'.join(code_chunk),
'fields': '|'.join(fields)
}
# headers = {'Content-type': 'application/json', 'Accept': 'application/json', 'Accept-Charset': 'utf-8'}
headers = {'User-agent': self._user_agent}
r = requests.post(url, data=data, headers=headers)
caches.update(r.json())
return caches
def __syncShowsRatings(self, traktShows, kodiShows, fromPercent, toPercent):
if kodiUtilities.getSettingAsBool('trakt_sync_ratings') and traktShows and not self.sync.IsCanceled():
updateKodiTraktShows = copy.deepcopy(traktShows)
updateKodiKodiShows = copy.deepcopy(kodiShows)
traktShowsToUpdate = self.__compareShows(updateKodiKodiShows, updateKodiTraktShows, rating=True)
if len(traktShowsToUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1='', line2=kodiUtilities.getString(32181))
logger.debug("[Episodes Sync] Trakt show ratings are up to date.")
else:
logger.debug("[Episodes Sync] %i show(s) will have show ratings added on Trakt" % len(traktShowsToUpdate['shows']))
self.sync.UpdateProgress(fromPercent, line1='', line2=kodiUtilities.getString(32182) % len(traktShowsToUpdate['shows']))
self.sync.traktapi.addRating(traktShowsToUpdate)
# needs to be restricted, because we can't add a rating to an episode which is not in our Kodi collection
kodiShowsUpdate = self.__compareShows(updateKodiTraktShows, updateKodiKodiShows, rating=True, restrict = True)
if len(kodiShowsUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1='', line2=kodiUtilities.getString(32176))
logger.debug("[Episodes Sync] Kodi show ratings are up to date.")
else:
logger.debug("[Episodes Sync] %i show(s) will have show ratings added in Kodi" % len(kodiShowsUpdate['shows']))
shows = []
for show in kodiShowsUpdate['shows']:
shows.append({'tvshowid': show['tvshowid'], 'rating': show['rating']})
# split episode list into chunks of 50
chunksize = 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "id": i, "method": "VideoLibrary.SetTVShowDetails",
"params": {"tvshowid": shows[i]['tvshowid'],
"userrating": shows[i]['rating']}} for i in range(len(shows))],
chunksize)
i = 0
x = float(len(shows))
for chunk in chunked_episodes:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent-fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y), line1='', line2=kodiUtilities.getString(32177) % ((i) * chunksize if (i) * chunksize < x else x, x))
kodiUtilities.kodiJsonRequest(chunk)
self.sync.UpdateProgress(toPercent, line2=kodiUtilities.getString(32178) % len(shows))
def add_entry():
form = EntryForm(request.form)
page_title = 'Add new blog entry'
if request.method == 'POST' and form.validate_on_submit():
entry = Entry()
entry.user_id = g.user.user_id
entry.create_time = datetime.datetime.utcnow()
form.populate_obj(entry)
db.session.add(entry)
db.session.commit()
return redirect(url_for('entries'))
tags = Tag.query.all()
tags = list(chunks(tags, COLS_IN_TAG_TABLE))
return render_template('entry_editor.html',
form_action='add_entry',
form=form,
tags=tags,
page_title=page_title)
def __traktAddEpisodes(self, shows):
if len(shows['shows']) == 0:
self.__updateProgress(98,
line1=utilities.getString(1435),
line2=utilities.getString(1490))
Debug("[Episodes Sync] trakt.tv episode collection is up to date.")
return
Debug(
"[Episodes Sync] %i show(s) have episodes (%d) to be added to your trakt.tv collection."
% (len(shows['shows']), self.__countEpisodes(shows['shows'])))
for show in shows['shows']:
Debug("[Episodes Sync] Episodes added: %s" %
self.__getShowAsString(show, short=True))
self.__updateProgress(82,
line1=utilities.getString(1435),
line2="0/%i %s" %
(len(shows['shows']), utilities.getString(1436)),
line3=" ")
#split episode list into chunks of 50
chunksize = 1
chunked_episodes = utilities.chunks(shows['shows'], chunksize)
i = 0
x = float(len(chunked_episodes))
for chunk in chunked_episodes:
if self.__isCanceled():
return
request = {'shows': chunk}
Debug("[trakt][traktAddEpisodes] Shows to add %s" % request)
result = self.traktapi.addToCollection(request)
Debug("[trakt][traktAddEpisodes] Result %s" % result)
i += 1
y = ((i / x) * 16) + 82
self.__updateProgress(
int(y),
line2="%i/%i %s" %
(i * chunksize, x, utilities.getString(1436)))
self.__updateProgress(98,
line1=utilities.getString(1435),
line2=utilities.getString(1491) %
self.__countEpisodes(shows['shows']))
def __addEpisodesToTraktCollection(self, kodiShows, traktShows, fromPercent, toPercent):
if kodiUtilities.getSettingAsBool('add_episodes_to_trakt') and not self.sync.IsCanceled():
addTraktShows = copy.deepcopy(traktShows)
addKodiShows = copy.deepcopy(kodiShows)
tmpTraktShowsAdd = self.__compareEpisodes(addKodiShows, addTraktShows)
traktShowsAdd = copy.deepcopy(tmpTraktShowsAdd)
utilities.sanitizeShows(traktShowsAdd)
# logger.debug("traktShowsAdd %s" % traktShowsAdd)
if len(traktShowsAdd['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1=kodiUtilities.getString(32068), line2=kodiUtilities.getString(32104))
logger.debug("[Episodes Sync] Trakt.tv episode collection is up to date.")
return
logger.debug("[Episodes Sync] %i show(s) have episodes (%d) to be added to your Trakt.tv collection." % (len(traktShowsAdd['shows']), self.__countEpisodes(traktShowsAdd)))
for show in traktShowsAdd['shows']:
logger.debug("[Episodes Sync] Episodes added: %s" % self.__getShowAsString(show, short=True))
self.sync.UpdateProgress(fromPercent, line1=kodiUtilities.getString(32068), line2=kodiUtilities.getString(32067) % (len(traktShowsAdd['shows'])))
# split episode list into chunks of 50
chunksize = 1
chunked_episodes = utilities.chunks(traktShowsAdd['shows'], chunksize)
errorcount = 0
i = 0
x = float(len(traktShowsAdd['shows']))
for chunk in chunked_episodes:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent-fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y), line2=kodiUtilities.getString(32069) % ((i) * chunksize if (i) * chunksize < x else x, x))
request = {'shows': chunk}
logger.debug("[traktAddEpisodes] Shows to add %s" % request)
try:
self.sync.traktapi.addToCollection(request)
except Exception as ex:
message = utilities.createError(ex)
logging.fatal(message)
errorcount += 1
logger.debug("[traktAddEpisodes] Finished with %d error(s)" % errorcount)
self.sync.UpdateProgress(toPercent, line2=kodiUtilities.getString(32105) % self.__countEpisodes(traktShowsAdd))
def __addEpisodesToKodiWatched(self, traktShows, kodiShows, kodiShowsCollected, fromPercent, toPercent):
if kodiUtilities.getSettingAsBool('kodi_episode_playcount') and not self.sync.IsCanceled():
updateKodiTraktShows = copy.deepcopy(traktShows)
updateKodiKodiShows = copy.deepcopy(kodiShows)
kodiShowsUpdate = self.__compareEpisodes(updateKodiTraktShows, updateKodiKodiShows, watched=True, restrict=True, collected=kodiShowsCollected)
if len(kodiShowsUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1=kodiUtilities.getString(32074), line2=kodiUtilities.getString(32107))
logger.debug("[Episodes Sync] Kodi episode playcounts are up to date.")
return
logger.debug("[Episodes Sync] %i show(s) shows are missing playcounts on Kodi" % len(kodiShowsUpdate['shows']))
for s in ["%s" % self.__getShowAsString(s, short=True) for s in kodiShowsUpdate['shows']]:
logger.debug("[Episodes Sync] Episodes updated: %s" % s)
# logger.debug("kodiShowsUpdate: %s" % kodiShowsUpdate)
episodes = []
for show in kodiShowsUpdate['shows']:
for season in show['seasons']:
for episode in season['episodes']:
episodes.append({'episodeid': episode['ids']['episodeid'], 'playcount': episode['plays'], "lastplayed": utilities.convertUtcToDateTime(episode['last_watched_at'])})
# split episode list into chunks of 50
chunksize = 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "method": "VideoLibrary.SetEpisodeDetails", "params": episodes[i], "id": i} for i in range(len(episodes))], chunksize)
i = 0
x = float(len(episodes))
for chunk in chunked_episodes:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent-fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y), line2=kodiUtilities.getString(32108) % ((i) * chunksize if (i) * chunksize < x else x, x))
logger.debug("[Episodes Sync] chunk %s" % str(chunk))
result = kodiUtilities.kodiJsonRequest(chunk)
logger.debug("[Episodes Sync] result %s" % str(result))
self.sync.UpdateProgress(toPercent, line2=kodiUtilities.getString(32109) % len(episodes))
def __kodiUpdateMovies(self, movies):
if len(movies) == 0:
self.__updateProgress(82, line2=utilities.getString(1471))
Debug("[Movies Sync] Kodi movie playcount is up to date.")
return
Debug("[Movies Sync] %i movie(s) playcount will be updated in Kodi" %
len(movies))
self.__updateProgress(66,
line2=utilities.getString(1430) % len(movies))
#split movie list into chunks of 50
chunksize = 50
chunked_movies = utilities.chunks([{
"jsonrpc": "2.0",
"method": "VideoLibrary.SetMovieDetails",
"params": {
"movieid": movies[i]['movieid'],
"playcount": movies[i]['plays']
},
"id": i
} for i in range(len(movies))], chunksize)
i = 0
x = float(len(chunked_movies))
for chunk in chunked_movies:
if self.__isCanceled():
return
utilities.kodiJsonRequest(chunk)
i += 1
y = ((i / x) * 16) + 66
self.__updateProgress(int(y),
line2=utilities.getString(1472) %
((i + 1) * chunksize if
(i + 1) * chunksize < x else x, x))
self.__updateProgress(82,
line2=utilities.getString(1473) % len(movies))
def createlist(caches, chunk_size, target_dir):
header_template = load_template('list_header')
footer_template = load_template('list_footer')
item_template = load_template('list_item')
chunk_count = int(math.ceil(len(caches) / chunk_size))
for chunk_index, chunk_caches in enumerate(chunks(caches, chunk_size)):
target_file_name = os.path.join(
target_dir, create_file_name(chunk_index, chunk_count))
pagination = create_pagination(chunk_index, chunk_count)
with open(target_file_name, 'w') as f:
prev = create_file_name(chunk_index - 1, chunk_count)
next = create_file_name(chunk_index + 1, chunk_count)
f.write(
header_template.replace(
'##PREVLINK##', f'<link rel="prev" href="{prev}">'
if prev else '').replace(
'##NEXTLINK##', f'<link rel="next" href="{next}">'
if next else '').replace('##PAGINATION##', pagination))
for cache in chunk_caches:
f.write(create_cache_item(item_template, cache, target_dir))
f.write(footer_template.replace('##PAGINATION##', pagination))
def __addEpisodeProgressToKodi(self, traktShows, kodiShows, fromPercent, toPercent):
if kodiUtilities.getSettingAsBool('trakt_episode_playback') and traktShows and not self.sync.IsCanceled():
updateKodiTraktShows = copy.deepcopy(traktShows)
updateKodiKodiShows = copy.deepcopy(kodiShows)
kodiShowsUpdate = self.__compareEpisodes(updateKodiTraktShows, updateKodiKodiShows, restrict=True, playback=True)
if len(kodiShowsUpdate['shows']) == 0:
self.sync.UpdateProgress(toPercent, line1=kodiUtilities.getString(1441), line2=kodiUtilities.getString(32129))
logger.debug("[Episodes Sync] Kodi episode playbacks are up to date.")
return
logger.debug("[Episodes Sync] %i show(s) shows are missing playbacks on Kodi" % len(kodiShowsUpdate['shows']))
for s in ["%s" % self.__getShowAsString(s, short=True) for s in kodiShowsUpdate['shows']]:
logger.debug("[Episodes Sync] Episodes updated: %s" % s)
episodes = []
for show in kodiShowsUpdate['shows']:
for season in show['seasons']:
for episode in season['episodes']:
episodes.append({'episodeid': episode['ids']['episodeid'], 'progress': episode['progress'], 'runtime': episode['runtime']})
# need to calculate the progress in int from progress in percent from Trakt
# split episode list into chunks of 50
chunksize = 50
chunked_episodes = utilities.chunks([{"jsonrpc": "2.0", "id": i, "method": "VideoLibrary.SetEpisodeDetails", "params": {"episodeid":episodes[i]['episodeid'], "resume": {"position": episodes[i]['runtime'] / 100.0 * episodes[i]['progress'], "total": episodes[i]['runtime']}}} for i in range(len(episodes))], chunksize)
i = 0
x = float(len(episodes))
for chunk in chunked_episodes:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent-fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y), line2=kodiUtilities.getString(32130) % ((i) * chunksize if (i) * chunksize < x else x, x))
kodiUtilities.kodiJsonRequest(chunk)
self.sync.UpdateProgress(toPercent, line2=kodiUtilities.getString(32131) % len(episodes))
def UpdatePlaysXBMC(self):
Debug('[Movies Sync] Checking if XBMC playcount is up to date')
if self.show_progress:
progress.update(85,
line1=__getstring__(1429),
line2=' ',
line3=' ')
update_playcount = []
trakt_playcounts = {}
for trakt_movie in self.trakt_movies_seen:
if 'tmdb_id' in trakt_movie:
trakt_playcounts[trakt_movie['tmdb_id']] = trakt_movie['plays']
if 'imdb_id' in trakt_movie:
trakt_playcounts[trakt_movie['imdb_id']] = trakt_movie['plays']
trakt_playcounts[trakt_movie['title']] = trakt_movie['plays']
for xbmc_movie in self.xbmc_movies:
if xbmc_movie['imdbnumber'] in trakt_playcounts:
if trakt_playcounts[
xbmc_movie['imdbnumber']] > xbmc_movie['playcount']:
xbmc_movie['playcount'] = trakt_playcounts[
xbmc_movie['imdbnumber']]
update_playcount.append(xbmc_movie)
elif xbmc_movie['title'] in trakt_playcounts:
if trakt_playcounts[
xbmc_movie['title']] > xbmc_movie['playcount']:
xbmc_movie['playcount'] = trakt_playcounts[
xbmc_movie['title']]
update_playcount.append(xbmc_movie)
if update_playcount:
Debug(
'[Movies Sync] %i movie(s) playcount will be updated on XBMC' %
len(update_playcount))
if self.show_progress:
progress.update(90,
line2='%i %s' %
(len(update_playcount), __getstring__(1430)))
#split movie list into chunks of 50
chunked_movies = chunks([{
"jsonrpc": "2.0",
"method": "VideoLibrary.SetMovieDetails",
"params": {
"movieid": update_playcount[i]['movieid'],
"playcount": update_playcount[i]['playcount']
},
"id": i
} for i in range(len(update_playcount))], 50)
for chunk in chunked_movies:
if self.Canceled():
return
xbmcJsonRequest(chunk)
else:
Debug('[Movies Sync] XBMC movie playcount is up to date')
def distributePirQueries(numServers, pirQueries, BASE_PORT):
ports = []
for i in range(numServers):
ports.append(BASE_PORT + i)
print(ports)
sock_lst = []
host = ''
backlog = 5 # Number of clients on wait.
buf_size = 20000
# buf_size = 10255
buf_size = 123000
errorFlag = 0
print('NUM SERVERS IS', numServers)
try:
for port in ports:
sock_lst.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM))
host = socket.gethostname()
sock_lst[-1].connect((host, port))
print('Connected to socket ', port)
sock_lst[-1].send(marshal.dumps(pirQueries.pop(0)))
# sock_lst[-1].bind((host, item))
# sock_lst[-1].listen(backlog)
# print "bound socket ",item
except socket.error as message:
if sock_lst[-1]:
sock_lst[-1].close()
sock_lst = sock_lst[:-1]
print('Could not connect to socket: ') # + message.args)
sys.exit(1)
# sumbit the PIR queries to each server and collect the results
result_cnt = 0
results = [[]] * numServers
while result_cnt < numServers:
# read, write, error = select.select(sock_lst,[],[])
inputready, outputready, exceptready = select.select(sock_lst, [], [])
for r in inputready:
# print('buf size is ',buf_size)
# data = r.recv(buf_size)
data = utilities.recv_msg(r) # THIS DOESN'T WORK FOR SOME REASON
# print('data size is ',len(data))
if data:
try:
# reading = []
# for i in range(0,len(data),1):
# reading += [struct.unpack('>I', data[i:i+4])[0]]
# print('REEESULTS',reading[:10])
res = marshal.loads(data)
except (EOFError, ValueError, TypeError) as m:
print('\n\nMAJOR ERROR ', len(data),
data[len(data) - 10:len(data) + 4], '\n\n')
print('Error message: ', m)
# print ('type',type(data))
# print ('The received data was ',data,len(data),'\n\n')
errorFlag = 1
return ([], errorFlag)
# print('res is ',res)
rPort = int(res.pop(0)) - BASE_PORT
results[rPort] = utilities.chunks(res, int(len(res) / nBins))
# print('res is ',res,rPort)
result_cnt = result_cnt + 1
else:
print('No data in client.py')
r.close()
sock_lst.remove(r)
if not sock_lst:
break
# close all the server sockets
for item in sock_lst:
item.close()
return (results, errorFlag)
def UpdatePlaysXBMC(self):
Debug('[Episodes Sync] Checking watched episodes on XBMC')
if self.show_progress:
progress.update(80,
line1=__getstring__(1441),
line2=' ',
line3=' ')
update_playcount = []
trakt_imdb_index = {}
trakt_tvdb_index = {}
trakt_title_index = {}
for i in range(len(self.trakt_shows['watched'])):
if 'imdb_id' in self.trakt_shows['watched'][i]:
trakt_imdb_index[self.trakt_shows['watched'][i]['imdb_id']] = i
if 'tvdb_id' in self.trakt_shows['watched'][i]:
trakt_tvdb_index[self.trakt_shows['watched'][i]['tvdb_id']] = i
trakt_title_index[self.trakt_shows['watched'][i]['title']] = i
for xbmc_show in self.xbmc_shows:
missing = []
trakt_show = None
#IMDB ID
if xbmc_show['imdbnumber'].startswith('tt') and xbmc_show[
'imdbnumber'] in trakt_imdb_index.keys():
trakt_show = self.trakt_shows['watched'][trakt_imdb_index[
xbmc_show['imdbnumber']]]
#TVDB ID
elif xbmc_show['imdbnumber'].isdigit(
) and xbmc_show['imdbnumber'] in trakt_tvdb_index.keys():
trakt_show = self.trakt_shows['watched'][trakt_tvdb_index[
xbmc_show['imdbnumber']]]
#Title
else:
if xbmc_show['title'] in trakt_title_index.keys():
trakt_show = self.trakt_shows['watched'][trakt_title_index[
xbmc_show['title']]]
if trakt_show:
missing = compare_show_watched_xbmc(xbmc_show, trakt_show)
else:
Debug('[Episodes Sync] Failed to find %s on trakt.tv' %
xbmc_show['title'])
if missing:
show = {
'title':
xbmc_show['title'],
'episodes': [{
'episodeid': x['episodeid'],
'playcount': 1
} for x in missing]
}
update_playcount.append(show)
if update_playcount:
Debug(
'[Episodes Sync] %i shows(s) shows are missing playcounts on XBMC'
% len(update_playcount))
if self.show_progress:
progress.update(85,
line1=__getstring__(1441),
line2='%i %s' %
(len(update_playcount), __getstring__(1439)))
for show in update_playcount:
if self.show_progress:
progress.update(
85,
line1=__getstring__(1441),
line2=show['title'].encode('utf-8', 'ignore'),
line3='%i %s' %
(len(show['episodes']), __getstring__(1440)))
#split episode list into chunks of 50
chunked_episodes = chunks([{
"jsonrpc": "2.0",
"method": "VideoLibrary.SetEpisodeDetails",
"params": show['episodes'][i],
"id": i
} for i in range(len(show['episodes']))], 50)
for chunk in chunked_episodes:
if self.Canceled():
return
xbmcJsonRequest(chunk)
else:
Debug('[Episodes Sync] XBMC episode playcounts are up to date')
def test_chunks():
movies = load_params_from_json('tests/fixtures/movies.json')
assert len(utilities.chunks(movies, 1)) == 3
def main(sensor, start_date, days, api_endpoint):
api = Api(api_endpoint)
logger.info('Checking consistencty for %s between %s + %s' % (sensor, start_date, days))
aoi_nw = (-180, 90)
aoi_se = (180, -90)
aoi_ne = (aoi_se[0], aoi_nw[1])
aoi_sw = (aoi_nw[0], aoi_se[1])
aoi = [aoi_nw, aoi_ne, aoi_se, aoi_sw, aoi_nw]
wrong_urls = list()
for delta_day in range(1, days):
start_time = time.time()
start_date_date = parse(start_date)+ datetime.timedelta(days=delta_day)
end_date_date = start_date_date + datetime.timedelta(days=1)
logger.info('Checking consistencty for %s between %s and %s' % (sensor, start_date_date.isoformat(), end_date_date.isoformat()))
# Object representation
results = api.search_dataset(aoi, 100, start_date_date, end_date_date, sensor, full_objects=False)
url_resources = list()
missing_urls = list()
missing_types = list()
for r in results:
if r['resources']['s3public']['zip'] != None:
url_resources.append(r['resources']['s3public']['zip'])
else:
missing_urls.append('%s:%s' % (r['tile_identifier'], r['entity_id']))
missing_types.append('zip')
if r['resources']['metadata']!= None:
url_resources.append(r['resources']['metadata'])
else:
missing_urls.append('%s:%s' % (r['tile_identifier'], r['entity_id']))
missing_types.append('metadata')
if r['resources']['quicklook'] != None:
url_resources.append(r['resources']['quicklook'])
else:
missing_urls.append('%s:%s' % (r['tile_identifier'], r['entity_id']))
missing_types.append('quicklook')
logger.info('total scans: %d' %len(url_resources))
logger.info('already missed resources: %d' %len(missing_urls))
if False:
for counter, res in enumerate(url_resources):
req = requests.head(res)
if req.status_code != requests.codes.ok:
print res, req.status_code
missing_urls.append(res)
print res
if (counter % 25) == 0:
print counter
else:
counter = 0
for url_parts in chunks(url_resources, 500):
counter+=1
rs = (grequests.head(u) for u in url_parts)
res = grequests.map(rs)
for req in res:
if req is not None:
if req.status_code != requests.codes.ok:
wrong_urls.append(res)
missing_types.append('zip_registered')
else:
print req.url, req
if len(wrong_urls) > 0:
for item in wrong_urls:
print item
for req in item:
if req.status_code != requests.codes.ok:
append_data('/tmp/wrong_urls.txt', req.url)
if len(missing_urls) > 0:
append_data('/tmp/missing_urls.txt', missing_urls)
if len(missing_types) > 0:
for type in ['zip_registered', 'quicklook', 'metadata', 'zip']:
logger.info('%d:%s' % (operator.countOf(missing_types, type), type))
logger.info('wrong resources resources: %d' % len(wrong_urls))
logger.info('Executed in %f secs.' % (time.time()-start_time))
print 'Wrong URLs:', wrong_urls
def __addMovieProgressToKodi(self, traktMovies, kodiMovies, fromPercent,
toPercent):
if kodiUtilities.getSettingAsBool(
'trakt_movie_playback'
) and traktMovies and not self.sync.IsCanceled():
updateKodiTraktMovies = copy.deepcopy(traktMovies)
updateKodiKodiMovies = copy.deepcopy(kodiMovies)
kodiMoviesToUpdate = utilities.compareMovies(
updateKodiTraktMovies['movies'],
updateKodiKodiMovies,
restrict=True,
playback=True)
if len(kodiMoviesToUpdate) == 0:
self.sync.UpdateProgress(toPercent,
line1='',
line2=kodiUtilities.getString(32125))
logger.debug(
"[Movies Sync] Kodi movie playbacks are up to date.")
return
logger.debug(
"[Movies Sync] %i movie(s) playbacks will be updated in Kodi" %
len(kodiMoviesToUpdate))
self.sync.UpdateProgress(fromPercent,
line1='',
line2=kodiUtilities.getString(32126) %
len(kodiMoviesToUpdate))
# need to calculate the progress in int from progress in percent from Trakt
# split movie list into chunks of 50
chunksize = 50
chunked_movies = utilities.chunks([{
"jsonrpc": "2.0",
"id": i,
"method": "VideoLibrary.SetMovieDetails",
"params": {
"movieid": kodiMoviesToUpdate[i]['movieid'],
"resume": {
"position":
kodiMoviesToUpdate[i]['runtime'] / 100.0 *
kodiMoviesToUpdate[i]['progress'],
"total":
kodiMoviesToUpdate[i]['runtime']
}
}
} for i in range(len(kodiMoviesToUpdate))], chunksize)
i = 0
x = float(len(kodiMoviesToUpdate))
for chunk in chunked_movies:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent - fromPercent)) + fromPercent
self.sync.UpdateProgress(int(y),
line2=kodiUtilities.getString(32127) %
((i) * chunksize if
(i) * chunksize < x else x, x))
kodiUtilities.kodiJsonRequest(chunk)
self.sync.UpdateProgress(toPercent,
line2=kodiUtilities.getString(32128) %
len(kodiMoviesToUpdate))
def __syncMovieRatings(self, traktMovies, kodiMovies, fromPercent,
toPercent):
if kodiUtilities.getSettingAsBool(
'trakt_sync_ratings'
) and traktMovies and not self.sync.IsCanceled():
updateKodiTraktMovies = copy.deepcopy(traktMovies)
updateKodiKodiMovies = copy.deepcopy(kodiMovies)
traktMoviesToUpdate = utilities.compareMovies(
updateKodiKodiMovies, updateKodiTraktMovies, rating=True)
if len(traktMoviesToUpdate) == 0:
self.sync.UpdateProgress(toPercent,
line1='',
line2=kodiUtilities.getString(32179))
logger.debug(
"[Movies Sync] Trakt movie ratings are up to date.")
else:
logger.debug(
"[Movies Sync] %i movie(s) ratings will be updated on Trakt"
% len(traktMoviesToUpdate))
self.sync.UpdateProgress(fromPercent,
line1='',
line2=kodiUtilities.getString(32180) %
len(traktMoviesToUpdate))
moviesRatings = {'movies': traktMoviesToUpdate}
self.sync.traktapi.addRating(moviesRatings)
kodiMoviesToUpdate = utilities.compareMovies(updateKodiTraktMovies,
updateKodiKodiMovies,
restrict=True,
rating=True)
if len(kodiMoviesToUpdate) == 0:
self.sync.UpdateProgress(toPercent,
line1='',
line2=kodiUtilities.getString(32169))
logger.debug(
"[Movies Sync] Kodi movie ratings are up to date.")
else:
logger.debug(
"[Movies Sync] %i movie(s) ratings will be updated in Kodi"
% len(kodiMoviesToUpdate))
self.sync.UpdateProgress(fromPercent,
line1='',
line2=kodiUtilities.getString(32170) %
len(kodiMoviesToUpdate))
# split movie list into chunks of 50
chunksize = 50
chunked_movies = utilities.chunks([{
"jsonrpc": "2.0",
"id": i,
"method": "VideoLibrary.SetMovieDetails",
"params": {
"movieid": kodiMoviesToUpdate[i]['movieid'],
"userrating": kodiMoviesToUpdate[i]['rating']
}
} for i in range(len(kodiMoviesToUpdate))], chunksize)
i = 0
x = float(len(kodiMoviesToUpdate))
for chunk in chunked_movies:
if self.sync.IsCanceled():
return
i += 1
y = ((i / x) * (toPercent - fromPercent)) + fromPercent
self.sync.UpdateProgress(
int(y),
line2=kodiUtilities.getString(32171) %
((i) * chunksize if (i) * chunksize < x else x, x))
kodiUtilities.kodiJsonRequest(chunk)
self.sync.UpdateProgress(toPercent,
line2=kodiUtilities.getString(32172) %
len(kodiMoviesToUpdate))
def remove_messages_from_queue(queue, message_list):
for x in chunks(message_list, MAX_MESSAGES):
try:
queue.delete_messages(Entries=x)
except botocore.exceptions.ClientError, e:
logger.error('Error occured during clean up queue: %s' % str(e))