def finish_subtitle_update(unique_key):
oauth_util = OAuthUtil()
query = FinishUpdateState.all()
query.filter('unique_key =', unique_key)
finish_state = query.get()
query = SubtitleData.all()
query.filter('language =', 'English')
for sdata in query:
if sdata.youtube_id in finish_state.videos_processed:
continue
try:
usub_video = json.loads(
oauth_util.access_usub_resource("/api/1.0/video/",
sdata.youtube_id, True))
except Exception, e:
logging.info(
"Unable to fetch video_id for video <%s> in playlist <%s> from universal subtitles"
% (sdata.youtube_id, sdata.playlist))
continue
# A video could be in multiple playlists, so get all of them
query2 = SubtitleData.all()
query2.filter('language =', 'English')
query2.filter('youtube_id =', sdata.youtube_id)
for sdata2 in query2:
sdata2.video_id = usub_video['video_id']
sdata2.put()
finish_state.videos_processed.append(sdata.youtube_id)
finish_state.number_of_videos_processed += 1
finish_state.put()
def populate_subtitle_data(playlist, pkey):
logging.info("Updating playlist <%s>" % playlist)
query = PlayListProcessedState.all()
query.filter('pkey =', pkey)
pstate = query.get()
oauth_util = OAuthUtil()
ka_videos = json.loads(
oauth_util.access_resource("/api/v1/playlists/%s/videos" %
urllib.quote(playlist)))
logging.info("Fetched <%d> videos from KA for playlist <%s>" %
(len(ka_videos), playlist))
query = PlayListData.all()
query.filter('playlist =', playlist)
pldata = query.get()
if pldata is None:
pldata = PlayListData()
pldata.subtitle_languages = []
pldata.playlist = playlist
pldata.subtitled_in_english_count = 0
pldata.video_count = len(ka_videos)
lang_mapped = {}
for ka_video in ka_videos:
if ka_video['youtube_id'] in pstate.videos_processed:
continue
try:
languages = json.loads(
oauth_util.access_usub_resource(
"/api/1.0/subtitles/languages/", ka_video['youtube_id'],
False))
except Exception, e:
continue
if type(languages) is not ListType:
continue
for language in languages:
lang = string.replace(language['name'], ",", "")
lang_in_native = lang
if language['name'].find('(') >= 0:
lang_in_native = language['name'].split('(')[1].split(')')[0]
lang_in_native = string.replace(lang_in_native, ",", "")
lang = language['name'].split('(')[0].strip()
lang = string.replace(lang, ",", "")
if lang not in lang_mapped:
lang_mapped[lang] = True
ldata = LanguageData.all().filter('language =', lang).get()
if ldata is None:
ldata = LanguageData()
ldata.language = lang
ldata.lang_in_native = lang_in_native
ldata.translated_videos = 0
ldata.last_week_incremental = [0, 0, 0, 0, 0, 0, 0]
ldata.put()
else:
if ldata.lang_in_native != lang_in_native:
ldata.lang_in_native = lang_in_native
ldata.language = lang
ldata.put()
pldata.subtitle_languages.append(lang)
query = SubtitleData.all()
query.filter('youtube_id =', ka_video['youtube_id'])
query.filter('language =', lang)
query.filter('playlist =', playlist)
sdata = query.get()
if sdata is None:
sdata = SubtitleData()
sdata.youtube_id = ka_video['youtube_id']
sdata.video = ka_video['title']
sdata.playlist = playlist
sdata.language = lang
sdata.percent_complete = 0
sdata.language_code = language['code']
sdata.language_id = language['id']
sdata.video_id = ""
s = language['completion']
try:
string.index(s, " Line")
except ValueError:
try:
string.index(s, "%")
except ValueError:
percent_complete = 0
else:
percent_complete = int(string.rstrip(s, " %"))
else:
i = int(string.rstrip(s, " Lines"))
if i == 0:
percent_complete = 0
elif lang == 'English':
percent_complete = 100
else:
percent_complete = 50
if percent_complete > sdata.percent_complete:
sdata.percent_complete = percent_complete
sdata.language_id = language['id']
sdata.put()
pstate.videos_processed.append(ka_video['youtube_id'])
pstate.count_of_videos_processed += 1
pstate.put()
# uniquify the list of languages interested in this playlist
s = set(pldata.subtitle_languages)
pldata.subtitle_languages = list(s)
pldata.put()