def test_Dataheap_Job_002_01(self): """Test classmethod Job.fromProgram() from 'dataheap'. This test triggers a job to rebuild the the seek table of program. Note: I did not realized that this is possible, it is not mentioned in the wiki: You have to grep the source code for the flag 'JOB_REBUILD'. Additional note: The log does not say anything about rebuilding the seektable. """ chanid = self.testenv['DOWNCHANID'] starttimemyth = self.testenv['DOWNSTARTTIME'] hostname = self.mydb.getMasterBackend() rec = Recorded((chanid, starttimemyth), db = self.mydb) prgrm = rec.getProgram() self.assertEqual(RECSTATUS.rsRecorded, prgrm.recstatus) myjob = Job.fromProgram(prgrm, JOBTYPE.COMMFLAG, hostname=hostname, flags=JOBFLAG.REBUILD) loopnr = 0 while (myjob.status < JOBSTATUS.FINISHED): time.sleep(10) myjob._pull() # this re-reads the jobqueue table loopnr += 1 if (loopnr > 60): break self.assertEqual(myjob.status, JOBSTATUS.FINISHED) # test '__repr__' and '__str__' print() print(repr(myjob)) print(str(myjob))
def test_Dataheap_Recorded_001_02(self): """Test method 'getProgram()' in class 'Recorded' from 'dataheap'. """ chanid = self.testenv['RECCHANID'] starttimeutc = self.testenv['RECSTARTTIMEUTC'] starttimemyth = self.testenv['RECSTARTTIMEMYTH'] title = self.testenv['RECTITLE'] basename = self.testenv['RECBASENAME'] recordedid = self.testenv['RECRECORDID'] inetref = self.testenv['RECINETREF'] rec = Recorded((chanid, starttimemyth), db = self.mydb) prgrm = rec.getProgram() self.assertTrue(isinstance(prgrm, Program)) self.assertEqual(RECSTATUS.rsRecorded, prgrm.rsRecorded)
def test_Dataheap_Job_002_02(self): """Test exception of classmethod Job.fromProgram() from 'dataheap'. """ chanid = self.testenv['DOWNCHANID'] starttimemyth = self.testenv['DOWNSTARTTIME'] hostname = self.mydb.getMasterBackend() rec = Recorded((chanid, starttimemyth), db = self.mydb) prgrm = rec.getProgram() self.assertEqual(RECSTATUS.rsRecorded, prgrm.recstatus) # set recstatus to 'rsUnknown' and capture the error: prgrm.recstatus = RECSTATUS.rsUnknown with self.assertRaises(MythError): myjob = Job.fromProgram(prgrm, JOBTYPE.COMMFLAG, hostname=hostname, flags=JOBFLAG.REBUILD)
class VIDEO: def __init__(self, opts, jobid=None): # Setup for the job to run if jobid: self.thisJob = Job(jobid) self.chanID = self.thisJob.chanid self.startTime = self.thisJob.starttime self.thisJob.update(status=Job.STARTING) # If no job ID given, must be a command line run else: self.thisJob = jobid self.chanID = opts.chanid self.startTime = opts.startdate + " " + opts.starttime + opts.offset self.opts = opts self.type = "none" self.db = MythDB() self.log = MythLog(module='Myth-Rec-to-Vid.py', db=self.db) # Capture the backend host name self.host = self.db.gethostname() # prep objects self.rec = Recorded((self.chanID, self.startTime), db=self.db) self.log( MythLog.GENERAL, MythLog.INFO, 'Using recording', '%s - %s' % (self.rec.title.encode('utf-8'), self.rec.subtitle.encode('utf-8'))) self.vid = Video(db=self.db).create({ 'title': '', 'filename': '', 'host': self.host }) self.bend = MythBE(db=self.db) def check_hash(self): self.log(self.log.GENERAL, self.log.INFO, 'Performing copy validation.') srchash = self.bend.getHash(self.rec.basename, self.rec.storagegroup) dsthash = self.bend.getHash(self.vid.filename, 'Videos') if srchash != dsthash: return False else: return True def copy(self): stime = time.time() srcsize = self.rec.filesize htime = [stime, stime, stime, stime] self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Copying myth://%s@%s/%s"\ % (self.rec.storagegroup, self.rec.hostname, self.rec.basename)\ +" to myth://Videos@%s/%s"\ % (self.host, self.vid.filename)) srcfp = self.rec.open('r') dstfp = self.vid.open('w') if self.thisJob: self.set_job_status(Job.RUNNING) tsize = 2**24 while tsize == 2**24: tsize = min(tsize, srcsize - dstfp.tell()) dstfp.write(srcfp.read(tsize)) htime.append(time.time()) rate = float(tsize * 4) / (time.time() - htime.pop(0)) remt = (srcsize - dstfp.tell()) / rate if self.thisJob: self.thisJob.setComment("%02d%% complete - %d seconds remaining" %\ (dstfp.tell()*100/srcsize, remt)) srcfp.close() dstfp.close() self.vid.hash = self.vid.getHash() self.log(MythLog.GENERAL | MythLog.FILE, MythLog.INFO, "Transfer Complete", "%d seconds elapsed" % int(time.time() - stime)) if self.thisJob: self.thisJob.setComment("Complete - %d seconds elapsed" % \ (int(time.time()-stime))) def copy_markup(self, start, stop): for mark in self.rec.markup: if mark.type in (start, stop): self.vid.markup.add(mark.mark, 0, mark.type) def copy_seek(self): for seek in self.rec.seek: self.vid.markup.add(seek.mark, seek.offset, seek.type) def delete_vid(self): self.vid.delete() def delete_rec(self): self.rec.delete() def dup_check(self): self.log(MythLog.GENERAL, MythLog.INFO, 'Processing new file name ', '%s' % (self.vid.filename)) self.log( MythLog.GENERAL, MythLog.INFO, 'Checking for duplication of ', '%s - %s' % (self.rec.title.encode('utf-8'), self.rec.subtitle.encode('utf-8'))) if self.bend.fileExists(self.vid.filename, 'Videos'): self.log(MythLog.GENERAL, MythLog.INFO, 'Recording already exists in Myth Videos') if self.thisJob: self.thisJob.setComment( "Action would result in duplicate entry") return True else: self.log( MythLog.GENERAL, MythLog.INFO, 'No duplication found for ', '%s - %s' % (self.rec.title.encode('utf-8'), self.rec.subtitle.encode('utf-8'))) return False def get_dest(self): if self.type == 'TV': self.vid.filename = self.process_fmt(TVFMT) elif self.type == 'MOVIE': self.vid.filename = self.process_fmt(MVFMT) self.vid.markup._refdat = (self.vid.filename, ) def get_meta(self): import_info = 'Listing only MetaData import complete' metadata = self.rec.exportMetadata() yrInfo = self.rec.getProgram() metadata['year'] = yrInfo.get('year') self.vid.importMetadata(metadata) if self.type == 'MOVIE': grab = VideoGrabber('Movie') results = grab.sortedSearch(self.rec.title) if len(results) > 0: for i in results: if i.year == yrInfo.get( 'year') and i.title == self.rec.get('title'): self.vid.importMetadata(i) match = grab.grabInetref(i.get('inetref')) length = len(match.people) for p in range(length - 2): self.vid.cast.add(match.people[p].get('name')) self.vid.director = match.people[length - 1].get('name') import_info = 'Full MetaData Import complete' else: grab = VideoGrabber('TV') results = grab.sortedSearch(self.rec.title, self.rec.subtitle) if len(results) > 0: for i in results: if i.title == self.rec.get( 'title') and i.subtitle == self.rec.get( 'subtitle'): self.vid.importMetadata(i) match = grab.grabInetref(grab.grabInetref(i.get('inetref'), \ season=i.get('season'),episode=i.get('episode'))) length = len(match.people) for p in range(length - 2): self.vid.cast.add(match.people[p].get('name')) self.vid.director = match.people[length - 1].get('name') import_info = 'Full MetaData Import complete' self.vid.category = self.rec.get('category') self.log(self.log.GENERAL, self.log.INFO, import_info) def get_type(self): if self.rec.seriesid != None and self.rec.programid[:2] != 'MV': self.type = 'TV' self.log(self.log.GENERAL, self.log.INFO, 'Performing TV type migration.') else: self.type = 'MOVIE' self.log(self.log.GENERAL, self.log.INFO, 'Performing Movie type migration.') def process_fmt(self, fmt): # replace fields from viddata ext = '.' + self.rec.basename.rsplit('.', 1)[1] rep = (('%TITLE%', 'title', '%s'), ('%SUBTITLE%', 'subtitle', '%s'), ('%SEASON%', 'season', '%d'), ('%SEASONPAD%', 'season', '%02d'), ('%EPISODE%', 'episode', '%d'), ('%EPISODEPAD%', 'episode', '%02d'), ('%YEAR%', 'year', '%s'), ('%DIRECTOR%', 'director', '%s')) for tag, data, format in rep: if self.vid[data]: fmt = fmt.replace(tag, format % self.vid[data]) else: fmt = fmt.replace(tag, '') # replace fields from program data rep = (('%HOSTNAME%', 'hostname', '%s'), ('%STORAGEGROUP%', 'storagegroup', '%s')) for tag, data, format in rep: data = getattr(self.rec, data) fmt = fmt.replace(tag, format % data) if len(self.vid.genre): fmt = fmt.replace('%GENRE%', self.vid.genre[0].genre) else: fmt = fmt.replace('%GENRE%', '') return fmt + ext def set_job_status(self, status): self.thisJob.setStatus(status) def set_vid_hash(self): self.vid.hash = self.vid.getHash() def update_vid(self): self.vid.update()
def runjob(jobid=None, chanid=None, starttime=None, tzoffset=None, maxWidth=maxWidth, maxHeight=maxHeight, sdonly=0, burncc=0, usemkv=0, overwrite=1): global estimateBitrate db = MythDB() try: if jobid: job = Job(jobid, db=db) chanid = job.chanid utcstarttime = job.starttime else: job = None utcstarttime = datetime.strptime(starttime, "%Y%m%d%H%M%S") utcstarttime = utcstarttime + timedelta(hours=tzoffset) if debug: print 'chanid "%s"' % chanid print 'utcstarttime "%s"' % utcstarttime rec = Recorded((chanid, utcstarttime), db=db) utcstarttime = rec.starttime starttime_datetime = utcstarttime # reformat 'starttime' for use with mythtranscode/HandBrakeCLI/mythcommflag starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace( u' ', '').replace(u'T', '').replace('-', '')) if debug: print 'mythtv format starttime "%s"' % starttime input_filesize = rec.filesize if rec.commflagged: if debug: print 'Recording has been scanned to detect commerical breaks.' waititer = 1 keepWaiting = True while keepWaiting == True: keepWaiting = False for index, jobitem in reversed( list( enumerate( db.searchJobs(chanid=chanid, starttime=starttime_datetime)))): if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if debug: print 'Commercial flagging job detected with status %s' % jobitem.status if jobitem.status == jobitem.RUNNING: # status = RUNNING? job.update({'status':job.PAUSED, 'comment':'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.'}) if debug: print 'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.' time.sleep(POLL_INTERVAL) keepWaiting = True waititer = waititer + 1 break else: if debug: print 'Recording has not been scanned to detect/remove commercial breaks.' if require_commflagged: if jobid: job.update({ 'status': job.RUNNING, 'comment': 'Required commercial flagging for this file is not found.' + 'Flagging commercials and cancelling any queued commercial flagging.' }) # cancel any queued job to flag commercials for this recording and run commercial flagging in this script for index, jobitem in reversed( list( enumerate( db.searchJobs(chanid=chanid, starttime=starttime_datetime)))): if debug: if index == 0: print jobitem.keys() print index, jobitem.id, jobitem.chanid if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if jobitem.status == jobitem.RUNNING: # status = RUNNING? jobitem.cmds = jobitem.STOP # stop command from the frontend to stop the commercial flagging job #jobitem.setStatus(jobitem.CANCELLED) #jobitem.setComment('Cancelled: Transcode command ran commercial flagging for this recording.') jobitem.update({ 'status': jobitem.CANCELLED, 'comment': 'A user transcode job ran commercial flagging for' + ' this recording and cancelled this job.' }) if debug: print 'Flagging Commercials...' # Call "mythcommflag --chanid $CHANID --starttime $STARTTIME" task = System(path='mythcommflag', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '2> /dev/null') except MythError, e: # it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)" pass #print 'Command failed with output:\n%s' % e.stderr #if jobid: # job.update({'status':304, 'comment':'Flagging commercials failed'}) #sys.exit(e.retcode) sg = findfile('/' + rec.basename, rec.storagegroup, db=db) if sg is None: print 'Local access to recording not found.' sys.exit(1) infile = os.path.join(sg.dirname, rec.basename) #TODO: set overWrite to 0 if infile is m4v or mkv (already converted) #tmpfile = '%s.tmp' % infile.rsplit('.',1)[0] outtitle = rec.title.replace("&", "and") outtitle = re.sub('[^A-Za-z0-9 ]+', '', outtitle) filetype = 'm4v' #DEBUG CODE TO FIND OBJECT STRUCT: #print '{}'.format(dir(rec.getProgram())) #print '{}'.format(rec.getProgram().year) if usemkv == 1: filetype = 'mkv' #print '!{}!'.format(rec.programid[0:2]) if rec.season > 0 and rec.episode > 0: #if there are seasons and episode numbers in the recording data outtitle = '{0:s} S{1:d} E{2:02d}'.format(outtitle, rec.season, rec.episode) elif rec.programid[0:2] == 'MV' and str(rec.getProgram().year).isdigit( ): #if it's a movie and has an original air date for when it came out outtitle = '{} ({})'.format(outtitle, rec.getProgram().year) elif rec.programid[ 0: 2] == 'MV' and rec.originalairdate != None and rec.originalairdate > datetime.date( datetime(1, 1, 1, 0, 0) ): #if it's a movie and has an original air date for when it came out outtitle = '{} ({})'.format(outtitle, rec.originalairdate.year) elif 'Sports' in rec.category: #if it's sports outtitle = '{}-{}-{}'.format( outtitle, re.sub('[^A-Za-z0-9 ]+', '', rec.subtitle), str(rec.starttime.strftime("%Y%m%d"))) elif rec.programid[0:2] == 'SH' and (' News ' in rec.title or rec.category == 'News'): #if it's a news show outtitle = '{}-{}'.format(outtitle, str(rec.starttime.strftime("%Y%m%d"))) elif rec.originalairdate != None and rec.originalairdate > datetime.date( datetime(1, 1, 1, 0, 0)): #if it has an original air date outtitle = '{} {}'.format( outtitle, str(rec.originalairdate.strftime("%Y%m%d"))) else: outtitle = '{} {}'.format(outtitle, str(rec.starttime.strftime("%Y%m%d"))) outtitle = '{}.{}'.format(outtitle, filetype) outfile = os.path.join(sg.dirname, outtitle) tmpfile = '{}.{}'.format( outfile.rsplit('.', 1)[0], infile.rsplit('.', 1)[1]) if tmpfile == infile: tmpfile = '{}.tmp'.format(infile.rsplit('.', 1)[0]) if (overwrite == 0): # If not overwritting the file, use the export folder outfile = os.path.join(exportFolder, outtitle) if debug: print 'overwrite is 0. outfile "{}"'.format(outfile) if os.path.isfile(outfile) or infile == outfile: # If outfile exists already, create a new name for the file. outfile = '{}-{}.{}'.format( outfile.rsplit('.', 1)[0], str(rec.starttime.strftime("%Y%m%d")), filetype) if os.path.isfile(tmpfile): # If the infile and tmpfile are the same, create a new name for the tmpfile tmpfile = '{}-{}.tmp'.format( outfile.rsplit('.', 1)[0], str(rec.starttime.strftime("%Y%m%d"))) if os.path.isfile(tmpfile): # If tmp exists already, create a new name for the file. outfile = '{}-{}.tmp'.format( tmpfile.rsplit('.', 1)[0], str(rec.starttime.strftime("%Y%m%d"))) if debug: print 'tmp exists. outfile "{}"'.format(outfile) if debug: print 'infile "{}"'.format(infile) print 'tmpfile "{}"'.format(tmpfile) print 'outfile "{}"'.format(outfile) #add_metadata(db, jobid, debug, job, rec, filetype, tmpfile) clipped_bytes = 0 # If selected, create a cutlist to remove commercials via mythtranscode by running: # mythutil --gencutlist --chanid $CHANID --starttime $STARTTIME if generate_commcutlist: if jobid: job.update({ 'status': job.RUNNING, 'comment': 'Generating Cutlist for commercial removal' }) task = System(path='mythutil', db=db) try: output = task('--gencutlist', '--chanid "%s"' % chanid, '--starttime "%s"' % starttime) except MythError, e: print 'Command "mythutil --gencutlist" failed with output:\n%s' % e.stderr if jobid: job.update({ 'status': job.ERRORED, 'comment': 'Generation of commercial Cutlist failed' }) sys.exit(e.retcode)
class VIDEO: def __init__(self, opts, jobid=None): # Setup for the job to run if jobid: self.thisJob = Job(jobid) self.chanID = self.thisJob.chanid self.startTime = self.thisJob.starttime self.thisJob.update(status=Job.STARTING) # If no job ID given, must be a command line run else: self.thisJob = jobid self.chanID = opts.chanid self.startTime = opts.startdate + " " + opts.starttime + opts.offset self.opts = opts self.type = "none" self.db = MythDB() self.log = MythLog(module='Myth-Rec-to-Vid.py', db=self.db) # Capture the backend host name self.host = self.db.gethostname() # prep objects self.rec = Recorded((self.chanID,self.startTime), db=self.db) self.log(MythLog.GENERAL, MythLog.INFO, 'Using recording', '%s - %s' % (self.rec.title.encode('utf-8'), self.rec.subtitle.encode('utf-8'))) self.vid = Video(db=self.db).create({'title':'', 'filename':'', 'host':self.host}) self.bend = MythBE(db=self.db) def check_hash(self): self.log(self.log.GENERAL, self.log.INFO, 'Performing copy validation.') srchash = self.bend.getHash(self.rec.basename, self.rec.storagegroup) dsthash = self.bend.getHash(self.vid.filename, 'Videos') if srchash != dsthash: return False else: return True def copy(self): stime = time.time() srcsize = self.rec.filesize htime = [stime,stime,stime,stime] self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Copying myth://%s@%s/%s"\ % (self.rec.storagegroup, self.rec.hostname, self.rec.basename)\ +" to myth://Videos@%s/%s"\ % (self.host, self.vid.filename)) srcfp = self.rec.open('r') dstfp = self.vid.open('w') if self.thisJob: self.set_job_status(Job.RUNNING) tsize = 2**24 while tsize == 2**24: tsize = min(tsize, srcsize - dstfp.tell()) dstfp.write(srcfp.read(tsize)) htime.append(time.time()) rate = float(tsize*4)/(time.time()-htime.pop(0)) remt = (srcsize-dstfp.tell())/rate if self.thisJob: self.thisJob.setComment("%02d%% complete - %d seconds remaining" %\ (dstfp.tell()*100/srcsize, remt)) srcfp.close() dstfp.close() self.vid.hash = self.vid.getHash() self.log(MythLog.GENERAL|MythLog.FILE, MythLog.INFO, "Transfer Complete", "%d seconds elapsed" % int(time.time()-stime)) if self.thisJob: self.thisJob.setComment("Complete - %d seconds elapsed" % \ (int(time.time()-stime))) def copy_markup(self, start, stop): for mark in self.rec.markup: if mark.type in (start, stop): self.vid.markup.add(mark.mark, 0, mark.type) def copy_seek(self): for seek in self.rec.seek: self.vid.markup.add(seek.mark, seek.offset, seek.type) def delete_vid(self): self.vid.delete() def delete_rec(self): self.rec.delete() def dup_check(self): self.log(MythLog.GENERAL, MythLog.INFO, 'Processing new file name ', '%s' % (self.vid.filename)) self.log(MythLog.GENERAL, MythLog.INFO, 'Checking for duplication of ', '%s - %s' % (self.rec.title.encode('utf-8'), self.rec.subtitle.encode('utf-8'))) if self.bend.fileExists(self.vid.filename, 'Videos'): self.log(MythLog.GENERAL, MythLog.INFO, 'Recording already exists in Myth Videos') if self.thisJob: self.thisJob.setComment("Action would result in duplicate entry" ) return True else: self.log(MythLog.GENERAL, MythLog.INFO, 'No duplication found for ', '%s - %s' % (self.rec.title.encode('utf-8'), self.rec.subtitle.encode('utf-8'))) return False def get_dest(self): if self.type == 'TV': self.vid.filename = self.process_fmt(TVFMT) elif self.type == 'MOVIE': self.vid.filename = self.process_fmt(MVFMT) self.vid.markup._refdat = (self.vid.filename,) def get_meta(self): import_info = 'Listing only MetaData import complete' metadata = self.rec.exportMetadata() yrInfo = self.rec.getProgram() metadata['year'] = yrInfo.get('year') self.vid.importMetadata(metadata) if self.type == 'MOVIE': grab = VideoGrabber('Movie') results = grab.sortedSearch(self.rec.title) if len(results) > 0: for i in results: if i.year == yrInfo.get('year') and i.title == self.rec.get('title'): self.vid.importMetadata(i) match = grab.grabInetref(i.get('inetref')) length = len(match.people) for p in range(length-2): self.vid.cast.add(match.people[p].get('name')) self.vid.director = match.people[length - 1].get('name') import_info = 'Full MetaData Import complete' else: grab = VideoGrabber('TV') results = grab.sortedSearch(self.rec.title, self.rec.subtitle) if len(results) > 0: for i in results: if i.title == self.rec.get('title') and i.subtitle == self.rec.get('subtitle'): self.vid.importMetadata(i) match = grab.grabInetref(grab.grabInetref(i.get('inetref'), \ season=i.get('season'),episode=i.get('episode'))) length = len(match.people) for p in range(length-2): self.vid.cast.add(match.people[p].get('name')) self.vid.director = match.people[length - 1].get('name') import_info = 'Full MetaData Import complete' self.vid.category = self.rec.get('category') self.log(self.log.GENERAL, self.log.INFO, import_info) def get_type(self): if self.rec.seriesid != None and self.rec.programid[:2] != 'MV': self.type = 'TV' self.log(self.log.GENERAL, self.log.INFO, 'Performing TV type migration.') else: self.type = 'MOVIE' self.log(self.log.GENERAL, self.log.INFO, 'Performing Movie type migration.') def process_fmt(self, fmt): # replace fields from viddata ext = '.'+self.rec.basename.rsplit('.',1)[1] rep = ( ('%TITLE%','title','%s'), ('%SUBTITLE%','subtitle','%s'), ('%SEASON%','season','%d'), ('%SEASONPAD%','season','%02d'), ('%EPISODE%','episode','%d'), ('%EPISODEPAD%','episode','%02d'), ('%YEAR%','year','%s'), ('%DIRECTOR%','director','%s')) for tag, data, format in rep: if self.vid[data]: fmt = fmt.replace(tag,format % self.vid[data]) else: fmt = fmt.replace(tag,'') # replace fields from program data rep = ( ('%HOSTNAME%', 'hostname', '%s'), ('%STORAGEGROUP%','storagegroup','%s')) for tag, data, format in rep: data = getattr(self.rec, data) fmt = fmt.replace(tag,format % data) if len(self.vid.genre): fmt = fmt.replace('%GENRE%',self.vid.genre[0].genre) else: fmt = fmt.replace('%GENRE%','') return fmt+ext def set_job_status(self, status): self.thisJob.setStatus(status) def set_vid_hash(self): self.vid.hash = self.vid.getHash() def update_vid(self): self.vid.update()