def __init__(self, command, test):
SubCommand.__init__(self, command)
test.assertEquals(self._get_data_root(), 'the data root')
test.assertEquals(self._get_opt('opt name'), 'opt value')
ok = False
try:
self._check_timer()
except MyTimerException:
ok = True
test.assertTrue(ok)
test.assertEqual(self._timer_remaining(), 92837745)
self._debug('debug')
self._info('info')
self._warn('warn')
ok = False
try:
self._error('error')
except SubCommandErrorException:
ok = True
test.assertTrue(ok)
self._error('error no ecception', no_exception=True)
test.assertEqual(self.command.logger._log,
[('debug', 'debug'), ('info', 'info'),
('warn', 'warn'), ('error', 'error'),
('error', 'error no ecception')])
def __init__(self,command,members=None):
SubCommand.__init__(self,command)
if members is None: members=Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
sourceVideos=self._fetchSourceVideosOrderedByPublishedDesc(member)
for sourceVideo in sourceVideos:
if self._isValidSourceVideo(sourceVideo,member):
videos=self._getVideosFromSource(sourceVideo,member)
if len(videos)==0:
# the source video does not exist in our database
# this is the about video for this member!
self._updateMemberAboutVideo(sourceVideo,member)
break
else:
# got some videos that match the source video
# check if any of them are 'related' videos
relatedVideo=None
for video in videos:
if video.group=='related' and not video.sticky and not video.hide:
relatedVideo=video
break
if relatedVideo is not None:
# got a related video that is not sticky and not hidden
# hide it in the related and create it again as about
# (we could just change the video's group field
# but it's better to create it again to make sure
# it's got all the relevant data)
self._hideRelatedVideo(relatedVideo)
self._updateMemberAboutVideo(sourceVideo,member)
else:
break
def __init__(self,command,committees=None):
SubCommand.__init__(self,command)
if self._get_opt('with-history'):
self._error('download of historical data is not supported yet')
if committees is None:
if self._get_opt('committee-id') is not None:
committees=[Committee.objects.get(id=self._get_opt('committee-id'))]
else:
committees=Committee.objects.all()
for comm in committees:
self._debug('UpdateCommitteesVideos - '+str(comm.id)+': '+comm.name)
self._check_timer()
broadcasts_url=comm.portal_knesset_broadcasts_url
if len(broadcasts_url)==0:
self._debug('committee does not have a broadcasts url, trying to find one')
broadcasts_url=self._update_committee_broadcasts_url(comm)
if len(broadcasts_url)>0:
self._debug('got a broadcasts url - '+str(broadcasts_url))
self._debug('searching for videos in the broadcasts url')
videos=self._get_committee_videos(broadcasts_url)
self._debug('got '+str(len(videos))+' videos')
for video in videos:
self._update_committee_mms_video(comm,video)
else:
self._warn('could not find a broadcasts url')
def __init__(self,command):
SubCommand.__init__(self,command)
for video in self._getVideosToDownload():
self._check_timer()
url=video.embed_link
self._debug(url)
filename=self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: "+filename)
continue
else:
self._debug('downloading: '+filename)
partfilename=self._get_data_root()+filename+'.part'
streamsize=mms.get_size(url)
self._debug('streamsize = '+str(streamsize))
mins_remaining=round(self._timer_remaining()/60)
if os.path.exists(partfilename):
filesize=os.path.getsize(partfilename)
if filesize<streamsize:
self._debug('mms resume download')
mms.resume_download(url,partfilename,mins_remaining)
else:
self._debug('mms download')
mms.download(url,partfilename,mins_remaining)
self._debug('done')
self._check_timer()
filesize=os.path.getsize(partfilename)
if filesize==streamsize:
os.rename(partfilename,filename)
self._debug("finished downloading: "+filename)
def __init__(self, command, committees=None):
SubCommand.__init__(self, command)
if self._get_opt('with-history'):
self._error('download of historical data is not supported yet')
if committees is None:
if self._get_opt('committee-id') is not None:
committees = [
Committee.objects.get(id=self._get_opt('committee-id'))
]
else:
committees = Committee.objects.all()
for comm in committees:
self._debug('UpdateCommitteesVideos - ' + str(comm.id) + ': ' +
comm.name)
self._check_timer()
broadcasts_url = comm.portal_knesset_broadcasts_url
if len(broadcasts_url) == 0:
self._debug(
'committee does not have a broadcasts url, trying to find one'
)
broadcasts_url = self._update_committee_broadcasts_url(comm)
if len(broadcasts_url) > 0:
self._debug('got a broadcasts url - ' + str(broadcasts_url))
self._debug('searching for videos in the broadcasts url')
videos = self._get_committee_videos(broadcasts_url)
self._debug('got ' + str(len(videos)) + ' videos')
for video in videos:
self._update_committee_mms_video(comm, video)
else:
self._warn('could not find a broadcasts url')
def __init__(self, command, members=None):
SubCommand.__init__(self, command)
if members is None: members = Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
sourceVideos = self._fetchSourceVideosOrderedByPublishedDesc(
member)
for sourceVideo in sourceVideos:
if self._isValidSourceVideo(sourceVideo, member):
videos = self._getVideosFromSource(sourceVideo, member)
if len(videos) == 0:
# the source video does not exist in our database
# this is the about video for this member!
self._updateMemberAboutVideo(sourceVideo, member)
break
else:
# got some videos that match the source video
# check if any of them are 'related' videos
relatedVideo = None
for video in videos:
if video.group == 'related' and not video.sticky and not video.hide:
relatedVideo = video
break
if relatedVideo is not None:
# got a related video that is not sticky and not hidden
# hide it in the related and create it again as about
# (we could just change the video's group field
# but it's better to create it again to make sure
# it's got all the relevant data)
self._hideRelatedVideo(relatedVideo)
self._updateMemberAboutVideo(sourceVideo, member)
else:
break
def __init__(self,command,test):
SubCommand.__init__(self,command)
test.assertEquals(self._get_data_root(),'the data root')
test.assertEquals(self._get_opt('opt name'),'opt value')
ok=False
try:
self._check_timer()
except MyTimerException:
ok=True
test.assertTrue(ok)
test.assertEqual(self._timer_remaining(),92837745)
self._debug('debug')
self._info('info')
self._warn('warn')
ok=False
try:
self._error('error')
except SubCommandErrorException:
ok=True
test.assertTrue(ok)
self._error('error no ecception',no_exception=True)
test.assertEqual(self.command.logger._log,[
('debug', 'debug'),
('info', 'info'),
('warn', 'warn'),
('error', 'error'),
('error', 'error no ecception')
])
def __init__(self, command):
SubCommand.__init__(self, command)
for video in self._getVideosToDownload():
self._check_timer()
url = video.embed_link
self._debug(url)
filename = self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: " + filename)
continue
else:
self._debug('downloading: ' + filename)
partfilename = self._get_data_root() + filename + '.part'
streamsize = mms.get_size(url)
self._debug('streamsize = ' + str(streamsize))
mins_remaining = round(self._timer_remaining() / 60)
if os.path.exists(partfilename):
filesize = os.path.getsize(partfilename)
if filesize < streamsize:
self._debug('mms resume download')
mms.resume_download(url, partfilename, mins_remaining)
else:
self._debug('mms download')
mms.download(url, partfilename, mins_remaining)
self._debug('done')
self._check_timer()
filesize = os.path.getsize(partfilename)
if filesize == streamsize:
os.rename(partfilename, filename)
self._debug("finished downloading: " + filename)
def __init__(self,command,members=None):
SubCommand.__init__(self,command)
if members is None: members=Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
relvids=[]
for name in member.names:
self._debug(name)
for video in self._getVideosForMember(name):
if self._verify_related_video(video,name):
relvids.append(video)
if len(relvids)>0:
for video in relvids:
self._update_member_related_video(member,video)
def __init__(self, command, members=None):
SubCommand.__init__(self, command)
if members is None: members = Member.objects.all()
for member in members:
self._debug(member.name)
self._check_timer()
relvids = []
for name in member.names:
self._debug(name)
for video in self._getVideosForMember(name):
if self._verify_related_video(video, name):
relvids.append(video)
if len(relvids) > 0:
for video in relvids:
self._update_member_related_video(member, video)
def __init__(self, command, mms=None, mb_quota=None):
if mms is None: mms = utils_mms
SubCommand.__init__(self, command)
self._verifyDataDir()
videos = self._getVideosToDownload()
self._debug('got ' + str(len(videos)) +
' videos from db that needs to be downloaded')
total_bytes = 0
for video in videos:
if mb_quota is not None and (total_bytes / 1000000) > mb_quota:
self._warn('reached mb quota of ' + str(mb_quota) + 'mb')
break
self._check_timer()
url = video.embed_link
self._debug('downloading video - ' + url)
filename = self._get_data_root(
) + 'committee_mms_videos/' + self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: " + filename)
total_bytes = total_bytes + self._getFileSize(filename)
continue
else:
partfilename = filename + '.part'
try:
streamsize = mms.get_size(url)
except Exception, e:
self._warn('failed to get mms stream size, exception = ' +
str(e))
traceback.print_exc(file=sys.stdout)
else:
self._debug('got mms stream size = ' + str(streamsize))
mins_remaining = round(self._timer_remaining() / 60)
downloaded = False
if self._isAlreadyDownloaded(partfilename):
filesize = self._getFileSize(partfilename)
if filesize < streamsize:
self._debug('resuming download')
try:
isDownloadDone = mms.resume_download(
url, partfilename, mins_remaining)
downloaded = True
except Exception, e:
self._warn(
'failed to resume mms download, exception = '
+ str(e))
traceback.print_exc(file=sys.stdout)
else:
def __init__(self, command, committees=None):
SubCommand.__init__(self, command)
if self._get_opt('with-history'):
self._error('download of historical data is not supported yet')
if committees is None: committees = Committee.objects.all()
for comm in committees:
self._debug(comm.name)
self._check_timer()
broadcasts_url = comm.portal_knesset_broadcasts_url
if len(broadcasts_url) == 0:
broadcasts_url = self._update_committee_broadcasts_url(comm)
if len(broadcasts_url) > 0:
videos = self._get_committee_videos(broadcasts_url)
for video in videos:
self._update_committee_mms_video(comm, video)
else:
self._warn('no broadcasts url :' + comm.name)
def __init__(self,command,committees=None):
SubCommand.__init__(self,command)
if self._get_opt('with-history'):
self._error('download of historical data is not supported yet')
if committees is None: committees=Committee.objects.all()
for comm in committees:
self._debug(comm.name)
self._check_timer()
broadcasts_url=comm.portal_knesset_broadcasts_url
if len(broadcasts_url)==0:
broadcasts_url=self._update_committee_broadcasts_url(comm)
if len(broadcasts_url)>0:
videos=self._get_committee_videos(broadcasts_url)
for video in videos:
self._update_committee_mms_video(comm,video)
else:
self._warn('no broadcasts url :'+comm.name)
def __init__(self,command,committees=None):
SubCommand.__init__(self,command)
ytService=gdata.youtube.service.YouTubeService()
authSubUrl=ytService.GenerateAuthSubURL(
next='http://www.oknesset.org/',
scope='http://gdata.youtube.com',
secure=False,
session=True
)
print "open the following url: "+str(authSubUrl)
print "authorize the application and at the end"
print "you will be redirected to anothe url"
print "paste the value of the token parameter from the query string"
token=raw_input("paste the token here: ")
ytService.SetAuthSubToken(token)
ytService.UpgradeToSessionToken()
print "thank you, this is your token:"
print ytService.GetAuthSubToken()
def __init__(self, command):
SubCommand.__init__(self, command)
videos = self._getVideosToUpload()
self._debug('got ' + str(len(videos)) + ' videos to upload')
for video in videos:
self._check_timer()
url = video.embed_link
filename = self._get_data_root(
) + 'committee_mms_videos/' + self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug('file ready to be uploaded: ' + filename)
(isOk, ytId) = self._uploadVideo(filename, video)
if isOk:
self._saveVideo(self._getPreParseVideoFields(video, ytId))
self._deleteFile(filename)
self._debug('uploaded video')
else:
self._debug('failed to upload video')
else:
self._debug('file does not exist: ' + filename)
def __init__(self,command,mms=None,mb_quota=None):
if mms is None: import knesset.video.utils.mms as mms
SubCommand.__init__(self,command)
self._verifyDataDir()
videos=self._getVideosToDownload()
self._debug('got '+str(len(videos))+' videos from db that needs to be downloaded')
total_bytes=0
for video in videos:
if mb_quota is not None and (total_bytes/1000000)>mb_quota:
self._warn('reached mb quota of '+str(mb_quota)+'mb')
break
self._check_timer()
url=video.embed_link
self._debug('downloading video - '+url)
filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug("file already downloaded: "+filename)
total_bytes=total_bytes+self._getFileSize(filename)
continue
else:
partfilename=filename+'.part'
try:
streamsize=mms.get_size(url)
except Exception, e:
self._warn('failed to get mms stream size, exception = '+str(e))
traceback.print_exc(file=sys.stdout)
else:
self._debug('got mms stream size = '+str(streamsize))
mins_remaining=round(self._timer_remaining()/60)
downloaded=False
if self._isAlreadyDownloaded(partfilename):
filesize=self._getFileSize(partfilename)
if filesize<streamsize:
self._debug('resuming download')
try:
isDownloadDone=mms.resume_download(url,partfilename,mins_remaining)
downloaded=True
except Exception, e:
self._warn('failed to resume mms download, exception = '+str(e))
traceback.print_exc(file=sys.stdout)
else:
def __init__(self,command):
SubCommand.__init__(self,command)
videos=self._getVideosToUpload()
self._debug('got '+str(len(videos))+' videos to upload')
for video in videos:
self._check_timer()
url=video.embed_link
filename=self._get_data_root()+'committee_mms_videos/'+self._getFilenameFromUrl(url)
if self._isAlreadyDownloaded(filename):
self._debug('file ready to be uploaded: '+filename)
(isOk,ytId)=self._uploadVideo(filename,video)
if isOk:
self._saveVideo(
self._getPreParseVideoFields(video,ytId)
)
self._deleteFile(filename)
self._debug('uploaded video')
else:
self._debug('failed to upload video')
else:
self._debug('file does not exist: '+filename)