def __init__(self,command,members=None): SubCommand.__init__(self,command) if members is None: members=Member.objects.all() for member in members: self._debug(member.name) self._check_timer() sourceVideos=self._fetchSourceVideosOrderedByPublishedDesc(member) for sourceVideo in sourceVideos: if self._isValidSourceVideo(sourceVideo,member): videos=self._getVideosFromSource(sourceVideo,member) if len(videos)==0: # the source video does not exist in our database # this is the about video for this member! self._updateMemberAboutVideo(sourceVideo,member) break else: # got some videos that match the source video # check if any of them are 'related' videos relatedVideo=None for video in videos: if video.group=='related' and not video.sticky and not video.hide: relatedVideo=video break if relatedVideo is not None: # got a related video that is not sticky and not hidden # hide it in the related and create it again as about # (we could just change the video's group field # but it's better to create it again to make sure # it's got all the relevant data) self._hideRelatedVideo(relatedVideo) self._updateMemberAboutVideo(sourceVideo,member) else: break
def __init__(self,command,test): SubCommand.__init__(self,command) test.assertEquals(self._get_data_root(),'the data root') test.assertEquals(self._get_opt('opt name'),'opt value') ok=False try: self._check_timer() except MyTimerException: ok=True test.assertTrue(ok) test.assertEqual(self._timer_remaining(),92837745) self._debug('debug') self._info('info') self._warn('warn') ok=False try: self._error('error') except SubCommandErrorException: ok=True test.assertTrue(ok) self._error('error no ecception',no_exception=True) test.assertEqual(self.command.logger._log,[ ('debug', 'debug'), ('info', 'info'), ('warn', 'warn'), ('error', 'error'), ('error', 'error no ecception') ])
def __init__(self, command, members=None): SubCommand.__init__(self, command) if members is None: members = Member.objects.all() for member in members: self._debug(member.name) self._check_timer() sourceVideos = self._fetchSourceVideosOrderedByPublishedDesc( member) for sourceVideo in sourceVideos: if self._isValidSourceVideo(sourceVideo, member): videos = self._getVideosFromSource(sourceVideo, member) if len(videos) == 0: # the source video does not exist in our database # this is the about video for this member! self._updateMemberAboutVideo(sourceVideo, member) break else: # got some videos that match the source video # check if any of them are 'related' videos relatedVideo = None for video in videos: if video.group == 'related' and not video.sticky and not video.hide: relatedVideo = video break if relatedVideo is not None: # got a related video that is not sticky and not hidden # hide it in the related and create it again as about # (we could just change the video's group field # but it's better to create it again to make sure # it's got all the relevant data) self._hideRelatedVideo(relatedVideo) self._updateMemberAboutVideo(sourceVideo, member) else: break
def __init__(self, command, committees=None): SubCommand.__init__(self, command) if self._get_opt("with-history"): self._error("download of historical data is not supported yet") if committees is None: if self._get_opt("committee-id") is not None: committees = [Committee.objects.get(id=self._get_opt("committee-id"))] else: committees = Committee.objects.all() for comm in committees: self._debug("UpdateCommitteesVideos - " + str(comm.id) + ": " + comm.name) self._check_timer() broadcasts_url = comm.portal_knesset_broadcasts_url if len(broadcasts_url) == 0: self._debug("committee does not have a broadcasts url, trying to find one") broadcasts_url = self._update_committee_broadcasts_url(comm) if len(broadcasts_url) > 0: self._debug("got a broadcasts url - " + str(broadcasts_url)) self._debug("searching for videos in the broadcasts url") videos = self._get_committee_videos(broadcasts_url) self._debug("got " + str(len(videos)) + " videos") for video in videos: self._update_committee_mms_video(comm, video) else: self._warn("could not find a broadcasts url")
def __init__(self, command, members=None, only_current_knesset=False, member_ids=[]): SubCommand.__init__(self, command) if members is None: if len(member_ids) > 0: members = Member.objects.filter(id__in=member_ids) elif only_current_knesset is True: members = Member.current_knesset.filter(is_current=True) self._debug('only current knesset') else: members = Member.objects.all() self._debug('updating related videos for ' + str(len(members)) + ' members') for member in members: self._debug(member.name) self._check_timer() relvids = [] for name in member.names: self._debug(name) for video in self._getVideosForMember(name): if self._verify_related_video(video, name): relvids.append(video) if len(relvids) > 0: for video in relvids: self._update_member_related_video(member, video)
def __init__(self, command, committees=None): SubCommand.__init__(self, command) if self._get_opt('with-history'): self._error('download of historical data is not supported yet') if committees is None: if self._get_opt('committee-id') is not None: committees = [ Committee.objects.get(id=self._get_opt('committee-id')) ] else: committees = Committee.objects.all() for comm in committees: self._debug('UpdateCommitteesVideos - ' + str(comm.id) + ': ' + comm.name) self._check_timer() broadcasts_url = comm.portal_knesset_broadcasts_url if len(broadcasts_url) == 0: self._debug( 'committee does not have a broadcasts url, trying to find one' ) broadcasts_url = self._update_committee_broadcasts_url(comm) if len(broadcasts_url) > 0: self._debug('got a broadcasts url - ' + str(broadcasts_url)) self._debug('searching for videos in the broadcasts url') videos = self._get_committee_videos(broadcasts_url) self._debug('got ' + str(len(videos)) + ' videos') for video in videos: self._update_committee_mms_video(comm, video) else: self._warn('could not find a broadcasts url')
def __init__(self,command,mms=None,mb_quota=None): if mms is None: import video.utils.mms as mms SubCommand.__init__(self,command) self._verifyDataDir() videos=self._getVideosToDownload() self._debug('got '+str(len(videos))+' videos from db that needs to be downloaded') total_bytes=0 for video in videos: if mb_quota is not None and (total_bytes/1000000)>mb_quota: self._warn('reached mb quota of '+str(mb_quota)+'mb') break self._check_timer() url=video.embed_link self._debug('downloading video - '+url) filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url) if self._isAlreadyDownloaded(filename): self._debug("file already downloaded: "+filename) total_bytes=total_bytes+self._getFileSize(filename) continue else: partfilename=filename+'.part' try: streamsize=mms.get_size(url) except Exception as e: self._warn('failed to get mms stream size, exception = '+str(e)) traceback.print_exc(file=sys.stdout) else: self._debug('got mms stream size = '+str(streamsize)) mins_remaining=round(self._timer_remaining()/60) downloaded=False if self._isAlreadyDownloaded(partfilename): filesize=self._getFileSize(partfilename) if filesize<streamsize: self._debug('resuming download') try: isDownloadDone=mms.resume_download(url,partfilename,mins_remaining) downloaded=True except Exception as e: self._warn('failed to resume mms download') logger.execption('failed to resume mms download, exception') else: self._debug('starting new download') try: isDownloadDone=mms.download(url,partfilename,mins_remaining) downloaded=True except Exception as e: self._warn('failed to resume mms download') logger.execption('failed to resume mms download') if downloaded: self._check_timer() filesize=self._getDownloadedFileSize(partfilename) self._debug('downloaded file size: '+str(filesize)) if isDownloadDone: self._renameFile(partfilename,filename) self._debug("finished downloading: "+filename) total_bytes=total_bytes+filesize
def __init__(self, command, committees=None): SubCommand.__init__(self, command) ytService = gdata.youtube.service.YouTubeService() authSubUrl = ytService.GenerateAuthSubURL( next="http://www.oknesset.org/", scope="http://gdata.youtube.com", secure=False, session=True ) print "open the following url: " + str(authSubUrl) print "authorize the application and at the end" print "you will be redirected to anothe url" print "paste the value of the token parameter from the query string" token = raw_input("paste the token here: ") ytService.SetAuthSubToken(token) ytService.UpgradeToSessionToken() print "thank you, this is your token:" print ytService.GetAuthSubToken()
def __init__(self,command,members=None): SubCommand.__init__(self,command) if members is None: members=Member.objects.all() for member in members: self._debug(member.name) self._check_timer() relvids=[] for name in member.names: self._debug(name) for video in self._getVideosForMember(name): if self._verify_related_video(video,name): relvids.append(video) if len(relvids)>0: for video in relvids: self._update_member_related_video(member,video)
def __init__(self, command, members=None): SubCommand.__init__(self, command) if members is None: members = Member.objects.all() for member in members: self._debug(member.name) self._check_timer() relvids = [] for name in member.names: self._debug(name) for video in self._getVideosForMember(name): if self._verify_related_video(video, name): relvids.append(video) if len(relvids) > 0: for video in relvids: self._update_member_related_video(member, video)
def __init__(self, command, committees=None): SubCommand.__init__(self, command) ytService = gdata.youtube.service.YouTubeService() authSubUrl = ytService.GenerateAuthSubURL( next='http://www.oknesset.org/', scope='http://gdata.youtube.com', secure=False, session=True) print "open the following url: " + str(authSubUrl) print "authorize the application and at the end" print "you will be redirected to anothe url" print "paste the value of the token parameter from the query string" token = raw_input("paste the token here: ") ytService.SetAuthSubToken(token) ytService.UpgradeToSessionToken() print "thank you, this is your token:" print ytService.GetAuthSubToken()
def __init__(self, command, mms=None, mb_quota=None): if mms is None: import video.utils.mms as mms SubCommand.__init__(self, command) self._verifyDataDir() videos = self._getVideosToDownload() self._debug("got " + str(len(videos)) + " videos from db that needs to be downloaded") total_bytes = 0 for video in videos: if mb_quota is not None and (total_bytes / 1000000) > mb_quota: self._warn("reached mb quota of " + str(mb_quota) + "mb") break self._check_timer() url = video.embed_link self._debug("downloading video - " + url) filename = self._get_data_root() + "committee_mms_videos/" + self._getFilenameFromUrl(url) if self._isAlreadyDownloaded(filename): self._debug("file already downloaded: " + filename) total_bytes = total_bytes + self._getFileSize(filename) continue else: partfilename = filename + ".part" try: streamsize = mms.get_size(url) except Exception, e: self._warn("failed to get mms stream size, exception = " + str(e)) traceback.print_exc(file=sys.stdout) else: self._debug("got mms stream size = " + str(streamsize)) mins_remaining = round(self._timer_remaining() / 60) downloaded = False if self._isAlreadyDownloaded(partfilename): filesize = self._getFileSize(partfilename) if filesize < streamsize: self._debug("resuming download") try: isDownloadDone = mms.resume_download(url, partfilename, mins_remaining) downloaded = True except Exception, e: self._warn("failed to resume mms download, exception = " + str(e)) traceback.print_exc(file=sys.stdout) else:
def __init__(self, command): SubCommand.__init__(self, command) videos = self._getVideosToUpload() self._debug('got ' + str(len(videos)) + ' videos to upload') for video in videos: self._check_timer() url = video.embed_link filename = self._get_data_root( ) + 'committee_mms_videos/' + self._getFilenameFromUrl(url) if self._isAlreadyDownloaded(filename): self._debug('file ready to be uploaded: ' + filename) (isOk, ytId) = self._uploadVideo(filename, video) if isOk: self._saveVideo(self._getPreParseVideoFields(video, ytId)) self._deleteFile(filename) self._debug('uploaded video') else: self._debug('failed to upload video') else: self._debug('file does not exist: ' + filename)
def __init__(self,command,mms=None,mb_quota=None): if mms is None: import video.utils.mms as mms SubCommand.__init__(self,command) self._verifyDataDir() videos=self._getVideosToDownload() self._debug('got '+str(len(videos))+' videos from db that needs to be downloaded') total_bytes=0 for video in videos: if mb_quota is not None and (total_bytes/1000000)>mb_quota: self._warn('reached mb quota of '+str(mb_quota)+'mb') break self._check_timer() url=video.embed_link self._debug('downloading video - '+url) filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url) if self._isAlreadyDownloaded(filename): self._debug("file already downloaded: "+filename) total_bytes=total_bytes+self._getFileSize(filename) continue else: partfilename=filename+'.part' try: streamsize=mms.get_size(url) except Exception, e: self._warn('failed to get mms stream size, exception = '+str(e)) traceback.print_exc(file=sys.stdout) else: self._debug('got mms stream size = '+str(streamsize)) mins_remaining=round(self._timer_remaining()/60) downloaded=False if self._isAlreadyDownloaded(partfilename): filesize=self._getFileSize(partfilename) if filesize<streamsize: self._debug('resuming download') try: isDownloadDone=mms.resume_download(url,partfilename,mins_remaining) downloaded=True except Exception, e: self._warn('failed to resume mms download, exception = '+str(e)) traceback.print_exc(file=sys.stdout) else:
def __init__(self,command): SubCommand.__init__(self,command) videos=self._getVideosToUpload() self._debug('got '+str(len(videos))+' videos to upload') for video in videos: self._check_timer() url=video.embed_link filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url) if self._isAlreadyDownloaded(filename): self._debug('file ready to be uploaded: '+filename) (isOk,ytId)=self._uploadVideo(filename,video) if isOk: self._saveVideo( self._getPreParseVideoFields(video,ytId) ) self._deleteFile(filename) self._debug('uploaded video') else: self._debug('failed to upload video') else: self._debug('file does not exist: '+filename)
def __init__(self,command,members=None,only_current_knesset=False,member_ids=[]): SubCommand.__init__(self,command) if members is None: if len(member_ids)>0: members=Member.objects.filter(id__in=member_ids) elif only_current_knesset is True: members=Member.current_knesset.filter(is_current=True) self._debug('only current knesset') else: members=Member.objects.all() self._debug('updating related videos for '+str(len(members))+' members') for member in members: self._debug(member.name) self._check_timer() relvids=[] for name in member.names: self._debug(name) for video in self._getVideosForMember(name): if self._verify_related_video(video,name): relvids.append(video) if len(relvids)>0: for video in relvids: self._update_member_related_video(member,video)