def __init__(self,command,members=None):
SubCommand.__init__(self,command)
if members is None: members=Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
sourceVideos=self._fetchSourceVideosOrderedByPublishedDesc(member)
for sourceVideo in sourceVideos:
if self._isValidSourceVideo(sourceVideo,member):
videos=self._getVideosFromSource(sourceVideo,member)
if len(videos)==0:
# the source video does not exist in our database
# this is the about video for this member!
self._updateMemberAboutVideo(sourceVideo,member)
break
else:
# got some videos that match the source video
# check if any of them are 'related' videos
relatedVideo=None
for video in videos:
if video.group=='related' and not video.sticky and not video.hide:
relatedVideo=video
break
if relatedVideo is not None:
# got a related video that is not sticky and not hidden
# hide it in the related and create it again as about
# (we could just change the video's group field
# but it's better to create it again to make sure
# it's got all the relevant data)
self._hideRelatedVideo(relatedVideo)
self._updateMemberAboutVideo(sourceVideo,member)
else:
break
def __init__(self,command,test):
SubCommand.__init__(self,command)
test.assertEquals(self._get_data_root(),'the data root')
test.assertEquals(self._get_opt('opt name'),'opt value')
ok=False
try:
self._check_timer()
except MyTimerException:
ok=True
test.assertTrue(ok)
test.assertEqual(self._timer_remaining(),92837745)
self._debug('debug')
self._info('info')
self._warn('warn')
ok=False
try:
self._error('error')
except SubCommandErrorException:
ok=True
test.assertTrue(ok)
self._error('error no ecception',no_exception=True)
test.assertEqual(self.command.logger._log,[
('debug', 'debug'),
('info', 'info'),
('warn', 'warn'),
('error', 'error'),
('error', 'error no ecception')
])
def __init__(self, command, members=None):
SubCommand.__init__(self, command)
if members is None: members = Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
sourceVideos = self._fetchSourceVideosOrderedByPublishedDesc(
member)
for sourceVideo in sourceVideos:
if self._isValidSourceVideo(sourceVideo, member):
videos = self._getVideosFromSource(sourceVideo, member)
if len(videos) == 0:
# the source video does not exist in our database
# this is the about video for this member!
self._updateMemberAboutVideo(sourceVideo, member)
break
else:
# got some videos that match the source video
# check if any of them are 'related' videos
relatedVideo = None
for video in videos:
if video.group == 'related' and not video.sticky and not video.hide:
relatedVideo = video
break
if relatedVideo is not None:
# got a related video that is not sticky and not hidden
# hide it in the related and create it again as about
# (we could just change the video's group field
# but it's better to create it again to make sure
# it's got all the relevant data)
self._hideRelatedVideo(relatedVideo)
self._updateMemberAboutVideo(sourceVideo, member)
else:
break
def __init__(self, command, committees=None):
SubCommand.__init__(self, command)
if self._get_opt("with-history"):
self._error("download of historical data is not supported yet")
if committees is None:
if self._get_opt("committee-id") is not None:
committees = [Committee.objects.get(id=self._get_opt("committee-id"))]
else:
committees = Committee.objects.all()
for comm in committees:
self._debug("UpdateCommitteesVideos - " + str(comm.id) + ": " + comm.name)
self._check_timer()
broadcasts_url = comm.portal_knesset_broadcasts_url
if len(broadcasts_url) == 0:
self._debug("committee does not have a broadcasts url, trying to find one")
broadcasts_url = self._update_committee_broadcasts_url(comm)
if len(broadcasts_url) > 0:
self._debug("got a broadcasts url - " + str(broadcasts_url))
self._debug("searching for videos in the broadcasts url")
videos = self._get_committee_videos(broadcasts_url)
self._debug("got " + str(len(videos)) + " videos")
for video in videos:
self._update_committee_mms_video(comm, video)
else:
self._warn("could not find a broadcasts url")
def __init__(self,
command,
members=None,
only_current_knesset=False,
member_ids=[]):
SubCommand.__init__(self, command)
if members is None:
if len(member_ids) > 0:
members = Member.objects.filter(id__in=member_ids)
elif only_current_knesset is True:
members = Member.current_knesset.filter(is_current=True)
self._debug('only current knesset')
else:
members = Member.objects.all()
self._debug('updating related videos for ' + str(len(members)) +
' members')
for member in members:
self._debug(member.name)
self._check_timer()
relvids = []
for name in member.names:
self._debug(name)
for video in self._getVideosForMember(name):
if self._verify_related_video(video, name):
relvids.append(video)
if len(relvids) > 0:
for video in relvids:
self._update_member_related_video(member, video)
def __init__(self, command, committees=None):
SubCommand.__init__(self, command)
if self._get_opt('with-history'):
self._error('download of historical data is not supported yet')
if committees is None:
if self._get_opt('committee-id') is not None:
committees = [
Committee.objects.get(id=self._get_opt('committee-id'))
]
else:
committees = Committee.objects.all()
for comm in committees:
self._debug('UpdateCommitteesVideos - ' + str(comm.id) + ': ' +
comm.name)
self._check_timer()
broadcasts_url = comm.portal_knesset_broadcasts_url
if len(broadcasts_url) == 0:
self._debug(
'committee does not have a broadcasts url, trying to find one'
)
broadcasts_url = self._update_committee_broadcasts_url(comm)
if len(broadcasts_url) > 0:
self._debug('got a broadcasts url - ' + str(broadcasts_url))
self._debug('searching for videos in the broadcasts url')
videos = self._get_committee_videos(broadcasts_url)
self._debug('got ' + str(len(videos)) + ' videos')
for video in videos:
self._update_committee_mms_video(comm, video)
else:
self._warn('could not find a broadcasts url')
def __init__(self,command,mms=None,mb_quota=None):
if mms is None: import video.utils.mms as mms
SubCommand.__init__(self,command)
self._verifyDataDir()
videos=self._getVideosToDownload()
self._debug('got '+str(len(videos))+' videos from db that needs to be downloaded')
total_bytes=0
for video in videos:
if mb_quota is not None and (total_bytes/1000000)>mb_quota:
self._warn('reached mb quota of '+str(mb_quota)+'mb')
break
self._check_timer()
url=video.embed_link
self._debug('downloading video - '+url)
filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: "+filename)
total_bytes=total_bytes+self._getFileSize(filename)
continue
else:
partfilename=filename+'.part'
try:
streamsize=mms.get_size(url)
except Exception as e:
self._warn('failed to get mms stream size, exception = '+str(e))
traceback.print_exc(file=sys.stdout)
else:
self._debug('got mms stream size = '+str(streamsize))
mins_remaining=round(self._timer_remaining()/60)
downloaded=False
if self._isAlreadyDownloaded(partfilename):
filesize=self._getFileSize(partfilename)
if filesize<streamsize:
self._debug('resuming download')
try:
isDownloadDone=mms.resume_download(url,partfilename,mins_remaining)
downloaded=True
except Exception as e:
self._warn('failed to resume mms download')
logger.execption('failed to resume mms download, exception')
else:
self._debug('starting new download')
try:
isDownloadDone=mms.download(url,partfilename,mins_remaining)
downloaded=True
except Exception as e:
self._warn('failed to resume mms download')
logger.execption('failed to resume mms download')
if downloaded:
self._check_timer()
filesize=self._getDownloadedFileSize(partfilename)
self._debug('downloaded file size: '+str(filesize))
if isDownloadDone:
self._renameFile(partfilename,filename)
self._debug("finished downloading: "+filename)
total_bytes=total_bytes+filesize
def __init__(self, command, committees=None):
SubCommand.__init__(self, command)
ytService = gdata.youtube.service.YouTubeService()
authSubUrl = ytService.GenerateAuthSubURL(
next="http://www.oknesset.org/", scope="http://gdata.youtube.com", secure=False, session=True
)
print "open the following url: " + str(authSubUrl)
print "authorize the application and at the end"
print "you will be redirected to anothe url"
print "paste the value of the token parameter from the query string"
token = raw_input("paste the token here: ")
ytService.SetAuthSubToken(token)
ytService.UpgradeToSessionToken()
print "thank you, this is your token:"
print ytService.GetAuthSubToken()
def __init__(self,command,members=None):
SubCommand.__init__(self,command)
if members is None: members=Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
relvids=[]
for name in member.names:
self._debug(name)
for video in self._getVideosForMember(name):
if self._verify_related_video(video,name):
relvids.append(video)
if len(relvids)>0:
for video in relvids:
self._update_member_related_video(member,video)
def __init__(self, command, members=None):
SubCommand.__init__(self, command)
if members is None: members = Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
relvids = []
for name in member.names:
self._debug(name)
for video in self._getVideosForMember(name):
if self._verify_related_video(video, name):
relvids.append(video)
if len(relvids) > 0:
for video in relvids:
self._update_member_related_video(member, video)
def __init__(self, command, committees=None):
SubCommand.__init__(self, command)
ytService = gdata.youtube.service.YouTubeService()
authSubUrl = ytService.GenerateAuthSubURL(
next='http://www.oknesset.org/',
scope='http://gdata.youtube.com',
secure=False,
session=True)
print "open the following url: " + str(authSubUrl)
print "authorize the application and at the end"
print "you will be redirected to anothe url"
print "paste the value of the token parameter from the query string"
token = raw_input("paste the token here: ")
ytService.SetAuthSubToken(token)
ytService.UpgradeToSessionToken()
print "thank you, this is your token:"
print ytService.GetAuthSubToken()
def __init__(self, command, mms=None, mb_quota=None):
if mms is None:
import video.utils.mms as mms
SubCommand.__init__(self, command)
self._verifyDataDir()
videos = self._getVideosToDownload()
self._debug("got " + str(len(videos)) + " videos from db that needs to be downloaded")
total_bytes = 0
for video in videos:
if mb_quota is not None and (total_bytes / 1000000) > mb_quota:
self._warn("reached mb quota of " + str(mb_quota) + "mb")
break
self._check_timer()
url = video.embed_link
self._debug("downloading video - " + url)
filename = self._get_data_root() + "committee_mms_videos/" + self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: " + filename)
total_bytes = total_bytes + self._getFileSize(filename)
continue
else:
partfilename = filename + ".part"
try:
streamsize = mms.get_size(url)
except Exception, e:
self._warn("failed to get mms stream size, exception = " + str(e))
traceback.print_exc(file=sys.stdout)
else:
self._debug("got mms stream size = " + str(streamsize))
mins_remaining = round(self._timer_remaining() / 60)
downloaded = False
if self._isAlreadyDownloaded(partfilename):
filesize = self._getFileSize(partfilename)
if filesize < streamsize:
self._debug("resuming download")
try:
isDownloadDone = mms.resume_download(url, partfilename, mins_remaining)
downloaded = True
except Exception, e:
self._warn("failed to resume mms download, exception = " + str(e))
traceback.print_exc(file=sys.stdout)
else:
def __init__(self, command):
SubCommand.__init__(self, command)
videos = self._getVideosToUpload()
self._debug('got ' + str(len(videos)) + ' videos to upload')
for video in videos:
self._check_timer()
url = video.embed_link
filename = self._get_data_root(
) + 'committee_mms_videos/' + self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug('file ready to be uploaded: ' + filename)
(isOk, ytId) = self._uploadVideo(filename, video)
if isOk:
self._saveVideo(self._getPreParseVideoFields(video, ytId))
self._deleteFile(filename)
self._debug('uploaded video')
else:
self._debug('failed to upload video')
else:
self._debug('file does not exist: ' + filename)
def __init__(self,command,mms=None,mb_quota=None):
if mms is None: import video.utils.mms as mms
SubCommand.__init__(self,command)
self._verifyDataDir()
videos=self._getVideosToDownload()
self._debug('got '+str(len(videos))+' videos from db that needs to be downloaded')
total_bytes=0
for video in videos:
if mb_quota is not None and (total_bytes/1000000)>mb_quota:
self._warn('reached mb quota of '+str(mb_quota)+'mb')
break
self._check_timer()
url=video.embed_link
self._debug('downloading video - '+url)
filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: "+filename)
total_bytes=total_bytes+self._getFileSize(filename)
continue
else:
partfilename=filename+'.part'
try:
streamsize=mms.get_size(url)
except Exception, e:
self._warn('failed to get mms stream size, exception = '+str(e))
traceback.print_exc(file=sys.stdout)
else:
self._debug('got mms stream size = '+str(streamsize))
mins_remaining=round(self._timer_remaining()/60)
downloaded=False
if self._isAlreadyDownloaded(partfilename):
filesize=self._getFileSize(partfilename)
if filesize<streamsize:
self._debug('resuming download')
try:
isDownloadDone=mms.resume_download(url,partfilename,mins_remaining)
downloaded=True
except Exception, e:
self._warn('failed to resume mms download, exception = '+str(e))
traceback.print_exc(file=sys.stdout)
else:
def __init__(self,command):
SubCommand.__init__(self,command)
videos=self._getVideosToUpload()
self._debug('got '+str(len(videos))+' videos to upload')
for video in videos:
self._check_timer()
url=video.embed_link
filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug('file ready to be uploaded: '+filename)
(isOk,ytId)=self._uploadVideo(filename,video)
if isOk:
self._saveVideo(
self._getPreParseVideoFields(video,ytId)
)
self._deleteFile(filename)
self._debug('uploaded video')
else:
self._debug('failed to upload video')
else:
self._debug('file does not exist: '+filename)
def __init__(self,command,members=None,only_current_knesset=False,member_ids=[]):
SubCommand.__init__(self,command)
if members is None:
if len(member_ids)>0:
members=Member.objects.filter(id__in=member_ids)
elif only_current_knesset is True:
members=Member.current_knesset.filter(is_current=True)
self._debug('only current knesset')
else:
members=Member.objects.all()
self._debug('updating related videos for '+str(len(members))+' members')
for member in members:
self._debug(member.name)
self._check_timer()
relvids=[]
for name in member.names:
self._debug(name)
for video in self._getVideosForMember(name):
if self._verify_related_video(video,name):
relvids.append(video)
if len(relvids)>0:
for video in relvids:
self._update_member_related_video(member,video)
