def get(self):
""" list top recordings with filters """
""" retrieve Boards with filters """
toplist = StorageTableCollection('recordings', "PartitionKey eq 'top'")
toplist = db.query(toplist)
toplist.sort(key=lambda item: item.beginn, reverse=True)
""" apply filters """
for key, value in request.args.items():
if key == 'Genre':
toplist.filter('genre', value)
elif key == 'Channel':
toplist.filter('sender', value)
elif key == 'Sort':
reverse = safe_cast(request.args.get('Descending', False),
bool)
field = recording[value].attribute
log.debug('Sort field = {} with reverse {!s}'.format(
field, reverse))
toplist = sorted(toplist,
key=lambda k: k[field],
reverse=reverse)
elif key == 'Descending':
if not request.args.get('Sort', False):
api.abort(403, __class__._responses['get'][403])
else:
api.abort(403, __class__._responses['get'][403])
""" abort if no toplist filtered """
if toplist == 0:
api.abort(404, __class__._responses['get'][404])
""" return list, httpstatus """
return toplist, 200
def get(self, id):
""" request top recording detail data """
""" logging """
log.info('select all details for recording: {!s}'.format(id))
""" retrieve board """
recording = db.get(Recording(PartitionKey='top', RowKey=str(id)))
if not db.exists(recording):
api.abort(404, __class__._responses['get'][404])
recording.Torrents = db.query(recording.Torrents)
""" return recording """
return recording, 200
def import_otrgenres(config, log) -> StorageTableCollection:
""" import genres csv into azure storage table """
log.debug('try to import genres...')
""" load Genres """
db.register_model(Genre())
Genres = StorageTableCollection('genres', "PartitionKey eq 'all'")
Genres = db.query(Genres)
if len(Genres) == 0:
""" if genres are empty """
if not os.path.exists('genre.csv'):
with urllib.request.urlopen(
'https://www.onlinetvrecorder.com/epg/genres.csv'
) as response:
genrecsv = response.read()
with open('genre.csv', 'wb') as ofile:
ofile.write(genrecsv)
with open('genre.csv', 'r') as csvfile:
reader = csv.DictReader(csvfile, dialect='excel', delimiter=';')
fieldnames = reader.fieldnames
rows = [row for row in reader]
for row in rows:
db.insert(Genre(Genre_Id=row['Nummer'],
Genre=row['Kategorie']))
os.remove('genre.csv')
log.info('genres successfully imported')
Genres = db.query(Genres)
else:
""" genres are not empty """
log.info('genres already imported')
return Genres
pass
def ExistsHistory(fingerprint, epgid) -> bool:
""" get job history for users fingerprint and epg id """
historylist = StorageTableCollection(
'history', "PartitionKey eq '" + fingerprint + "'")
historylist = db.query(historylist)
identicalepgs = [
item for item in historylist if item['epgid'] == int(epgid)
]
if len(identicalepgs) > 0:
return True
else:
return False
def housekeeping(date, config, log):
""" housekeeping epg for date into database:
date:Str = Date a in dd.mm.yyyy to import
"""
PartitionKey = date.strftime('%Y_%m_%d')
log.debug(
'housekeeping: try to delete all epg entries with Partitionkey: {!s} ...'
.format(PartitionKey))
""" load Genres """
epgs = StorageTableCollection('recordings',
"PartitionKey eq '" + PartitionKey + "'")
epgs = db.query(epgs)
for epg in epgs:
db.delete(epg)
log.info('housekeeping finished for Partitionkey: {!s} ...'.format(
PartitionKey))
pass
def history():
""" retrieve top recordings with filters """
historylist = StorageTableCollection(
'history', "PartitionKey eq '" + g.user.RowKey + "'")
historylist = db.query(historylist)
historylist.sort(key=lambda item: item.created, reverse=True)
for item in historylist:
item['startdate'] = item.beginn.strftime('%d.%m.%Y')
item['starttime'] = item.beginn.strftime('%H:%M')
item['createdate'] = item.created.strftime('%d.%m.%Y')
item['updatedate'] = item.updated.strftime('%d.%m.%Y %H:%M')
item['previewimagelink'] = item['previewimagelink'].replace(
'http://', 'https://', 1)
""" render platform template """
pathtemplate = g.platform + '/' + 'history.html'
return render_template(pathtemplate,
title='Verlauf',
pagetitle='history',
items=historylist,
message=g.message)
def index():
""" retrieve top recordings with filters """
#get toplist
toplist = StorageTableCollection('recordings', "PartitionKey eq 'top'")
toplist = db.query(toplist)
toplist.sort(key=lambda item: item.beginn, reverse=True)
for item in toplist:
item['startdate'] = item.beginn.strftime('%d.%m.%Y')
item['starttime'] = item.beginn.strftime('%H:%M')
item['previewimagelink'] = item['previewimagelink'].replace(
'http://', 'https://', 1)
if not 'torrentCount' in item:
item['torrentCount'] = 0
""" render platform template """
pathtemplate = g.platform + '/' + 'index.html'
return render_template(pathtemplate,
title='OTR Top Aufnahmen',
pagetitle='index',
items=toplist,
message=g.message)
def update_torrents(startdate: date, config, log):
"""
rufe alle torrents der letzten 8 Tage ab und ordne diese einem top recording zu
https://www.onlinetvrecorder.com/v2/?go=tracker&search=&order=ctime%20DESC&start=0
"""
log.debug('try to update torrents webcontent...')
stopflag = False
start = 0
torrentlist = []
db.register_model(Torrent())
while not stopflag:
""" download webcontent into content"""
with urllib.request.urlopen(
'https://www.onlinetvrecorder.com/v2/?go=tracker&search=&order=ctime%20DESC&start='
+ str(start)) as response:
content = response.read()
""" für jeden Eintrag in ID= searchrow """
content = str(content.decode('utf-8', 'ignore')).split(
' class="bordertable">')[1].split('</table>')[0].split('</tr>')
for index in range(1, len(content) - 1):
lines = content[index].split('</td>')
""" parse data from entry """
torrentlink = lines[1].split("href='")[1].split("'")[0]
torrentfile = lines[1].split(torrentlink +
"'>")[1].split('</a>')[0]
finished = safe_cast(lines[2].split('>')[1].split('</td>')[0], int,
0)
loading = safe_cast(lines[3].split('>')[1].split('</td>')[0], int,
0)
loaded = safe_cast(lines[4].split('>')[1].split('</td>')[0], int,
0)
fileparts = torrentfile.split(' ')
beginn = safe_cast(
fileparts[len(fileparts) - 4] + ' ' +
fileparts[len(fileparts) - 3] + '-00', datetime, None,
'%y.%m.%d %H-%M-%S')
sender = fileparts[len(fileparts) - 2]
if beginn.date() >= startdate:
""" update list """
torrent = {}
torrent['TorrentLink'] = torrentlink
torrent['TorrentFile'] = torrentfile
torrent['finished'] = finished
torrent['loading'] = loading
torrent['loaded'] = loaded
torrent['beginn'] = beginn
torrent['sender'] = sender.replace(' ', '').lower()
resolution = ''
resolution = torrentlink.split('TVOON_DE')[1].split(
'otrkey.torrent')[0]
if resolution == ('.mpg.HD.avi.'):
""" TVOON_DE.mpg.HD.avi.otrkey.torrent"""
resolution = 'HD'
elif resolution == ('.mpg.HQ.avi.'):
""" _TVOON_DE.mpg.HQ.avi.otrkey.torrent"""
resolution = 'HQ'
elif resolution == ('.mpg.avi.'):
""" DIVX _TVOON_DE.mpg.avi.otrkey.torrent """
resolution = 'DIVX'
elif resolution == ('.mpg.mp4.'):
""" MP4 0_TVOON_DE.mpg.mp4.otrkey.torrent """
resolution = 'MP4'
elif resolution == ('.mpg.HD.ac3.'):
""" f1_130_TVOON_DE.mpg.HD.ac3.otrkey.torrent """
resolution = 'HD.AC3'
else:
resolution = 'AVI'
torrent['Resolution'] = resolution
torrentlist.append(torrent)
#log.debug('parsed torrent: {} in {} recorded at {!s} on {}'.format(torrentfile, resolution, beginn, sender))
else:
stopflag = True
break
start = start + 50
log.info('{!s} torrents successfully retrieved...'.format(
len(torrentlist)))
""" retrieve epg id from top recordings """
tops = StorageTableCollection('recordings', "PartitionKey eq 'top'")
tops = db.query(tops)
for top in tops:
torrents = [
item for item in torrentlist
if item['beginn'].strftime('%y.%m.%d %H-%M-%S') ==
top.beginn.strftime('%y.%m.%d %H-%M-%S')
and item['sender'] == top.sender.replace(' ', '').lower()
]
log.debug('filterded {!s} torrents for top recording {}'.format(
len(torrents), top.titel))
if len(torrents) >= 1:
""" Torrent Count """
topItem = Recording(**top)
topItem.torrentCount = len(torrents)
db.insert(topItem)
""" Insert Torrent """
for torrent in torrents:
db.insert(Torrent(Id=top.Id, **torrent))
else:
db.delete(Torrent(Id=top.Id, **torrent))
db.delete(Recording(**top))
def details(epgid):
""" request top recording detail data """
""" logging """
log.info('select all details for recording: {!s} with method {!s}'.format(
epgid, request.method))
""" determine template & messages """
pathtemplate = g.platform + '/' + 'details.html'
#get message
message = g.message
g.pop('message')
""" retrieve recording from storage """
recording = db.get(Recording(PartitionKey='top', RowKey=str(epgid)))
""" show details including all torrents """
if not db.exists(recording):
return index()
recording.Torrents = db.query(recording.Torrents)
recording.startdate = recording.beginn.strftime('%d.%m.%Y')
recording.starttime = recording.beginn.strftime('%H:%M')
recording.previewimagelink = recording.previewimagelink.replace(
'http://', 'https://', 1)
if request.method == 'POST':
""" retrieve Form Data """
data = request.form.to_dict()
log.debug(data)
""" post method = push only available for logged in users """
if not g.user:
message.show = True
message.error = True
message.header = 'Fehler'
message.text = 'Bitte loggen Sie sich ein!'
elif (not g.user.ProUser) and (g.user.PushVideo):
message.show = True
message.error = True
message.header = 'Fehler'
message.text = 'Um Videos zu decodieren müssen Sie Pro Status haben!'
elif ExistsHistory(g.user.RowKey, epgid):
""" already in History ? """
message.show = True
message.error = True
message.header = 'Fehler'
message.text = 'Sie haben diese Aufnahme bereits erfolgreich gepushed!'
elif (g.user.PushVideo):
""" successfully configured ? """
if not g.user.FtpConnectionChecked:
return redirect(url_for('ui.settings', messageid=5))
""" push video """
job, errormessage = PushVideo(epgid, data['Resolution'],
data['TorrentFile'],
data['TorrentLink'], g.user)
if not job:
message.show = True
message.error = True
message.header = 'Fehler'
message.text = errormessage
else:
""" success """
message.show = True
message.header = 'Erfolg'
message.text = '{!s} wird heruntergeladen, decodiert und danach an Ihren Endpoint gepushed. Die Aufgabe {!s} wurde dazu erfolgreich angelegt'.format(
job.sourcefile, job.id)
""" add history """
AddHistory(g.user, job.id, 'video', epgid, recording.beginn,
recording.sender, recording.titel, recording.genre,
recording.previewimagelink, data['Resolution'],
job.sourcefile, request.remote_addr,
request.user_agent.platform,
request.user_agent.browser,
request.user_agent.version,
request.user_agent.language)
else:
""" successfully configured ? """
if not g.user.FtpConnectionChecked:
return redirect(url_for('ui.settings', messageid=5))
""" push torrent """
job, errormessage = PushTorrent(epgid, data['Resolution'],
data['TorrentFile'],
data['TorrentLink'], g.user)
if not job:
message.show = True
message.error = True
message.header = 'Fehler'
message.text = errormessage
else:
""" success """
message.show = True
message.header = 'Erfolg'
message.text = '{!s} wird heruntergeladen und an Ihren Endpoint gepushed. Die Aufgabe {!s} wurde dazu erfolgreich angelegt'.format(
job.sourcefile, job.id)
""" add history """
AddHistory(g.user, job.id, 'torrent', epgid, recording.beginn,
recording.sender, recording.titel, recording.genre,
recording.previewimagelink, data['Resolution'],
job.sourcefile, request.remote_addr,
request.user_agent.platform,
request.user_agent.browser,
request.user_agent.version,
request.user_agent.language)
""" render platform template """
return render_template(pathtemplate,
title='OTR Aufnahme Details',
pagetitle='details',
item=recording,
message=message)
def do_pushtorrent_queue_message(config, log):
""" retrieve and process all visible push queue messages
- if link is local check if file exist then push to ftp endpoint
- if link is url then download torrentfile and push to endpoint
"""
queue.register_model(PushMessage())
db.register_model(History())
if config['APPLICATION_ENVIRONMENT'] in ['Development', 'Test']:
queuehide = 1
else:
queuehide = 5 * 60
""" loop all push queue messages """
message = queue.get(PushMessage(), queuehide)
while not message is None:
""" get history entry for message for an status update """
historylist = StorageTableCollection('history',
"RowKey eq '" + message.id + "'")
historylist = db.query(historylist)
for item in historylist:
history = db.get(
History(PartitionKey=item.PartitionKey, RowKey=message.id))
if not history:
history = History(PartitionKey='torrent', RowKey=message.id)
history.created = datetime.now()
history.epgid = message.epgid
history.sourcefile = message.sourcefile
if message.sourcelink in ['', 'string']:
""" no sourcelink ? """
""" delete queue message and tmp file """
queue.delete(message)
else:
""" push torrentfile
---------------------------------------------------------------------
1) download torrent to local tmp folder
2) pushfile to ftp
3) delete torrent from local tmp folder
4) delete queue message
"""
""" 1) download torrent to local tmp folder """
filename, localfile = get_torrentfile(
message.sourcelink, config['APPLICATION_PATH_TMP'])
if (not filename is None) and (not localfile is None):
downloaded, errormessage = download_fromurl(
message.sourcelink, localfile)
if downloaded:
""" 2) pushfile to ftp """
uploaded, errormessage = ftp_upload_file2(
log, message.server, message.port, message.user,
message.password, message.destpath, filename,
localfile)
if uploaded:
""" 3) delete torrent from local tmp folder,
4) delete queue message
"""
queue.delete(message)
if os.path.exists(localfile):
os.remove(localfile)
history.status = 'finished'
if not errormessage is None:
""" delete message after 3 tries """
log.error('push failed because {}'.format(errormessage))
history.status = 'error'
if (config['APPLICATION_ENVIRONMENT']
== 'Production') and (message.dequeue_count >= 3):
queue.delete(message)
history.status = 'deleted'
if os.path.exists(localfile):
os.remove(localfile)
""" update history entry """
history.updated = datetime.now()
db.insert(history)
""" next message """
message = queue.get(PushMessage(), queuehide)
pass
def do_pushvideo_queue_message(config, log):
""" retrieve and process all visible download queue mess ages
- if link is url then download torrentfile
---------------------------------------------------------------------
1) if videofile is in place:
1a) push video to ftp
1b) delete videofile, otrkeyfile, torrentfile
1c) delete queue message
2) OR if otrkeyfile is in place
2a) init decodingprocess to videofile
3) ELSE if transmission job is not running
3a) add transmission torrent
"""
queue.register_model(PushVideoMessage())
db.register_model(History())
if config['APPLICATION_ENVIRONMENT'] in ['Development', 'Test']:
queuehide = 60
else:
queuehide = 5 * 60
""" housekeeping array of files and transmission-queues to be deleted """
houskeeping = []
housekeepingTransmission = []
""" get transmission status """
transmissionstatus = get_transmissionstatus(log)
""" loop all push queue messages """
message = queue.get(PushVideoMessage(), queuehide)
while not message is None:
""" get history entry for message for an status update """
historylist = StorageTableCollection('history',
"RowKey eq '" + message.id + "'")
historylist = db.query(historylist)
for item in historylist:
history = db.get(
History(PartitionKey=item.PartitionKey, RowKey=message.id))
if not history:
history = History(PartitionKey='video', RowKey=message.id)
history.created = datetime.now()
history.epgid = message.epgid
history.sourcefile = message.videofile
user = None
else:
""" get user """
user = None
userlist = db.query(
StorageTableCollection(
'userprofile', "RowKey eq '" + history.PartitionKey + "'"))
for item in userlist:
user = db.get(
User(PartitionKey=item.PartitionKey,
RowKey=history.PartitionKey))
#log.debug('{!s}'.format(history.dict()))
""" get single transmission download status """
downloadstatus = [
element for element in transmissionstatus
if element['Name'] == message.otrkeyfile
]
if downloadstatus != []:
downloadstatus = downloadstatus[0]
else:
downloadstatus = None
#log.debug('{!s}'.format(downloadstatus))
if message.sourcelink in ['', 'string']:
""" no sourcelink ? """
""" delete queue message and tmp file """
queue.delete(message)
history.status = 'deleted'
else:
""" process push video """
try:
localvideofile = os.path.join(
config['APPLICATION_PATH_VIDEOS'], message.videofile)
localotrkeyfile = os.path.join(
config['APPLICATION_PATH_OTRKEYS'], message.otrkeyfile)
message.sourcefile, localtorrentfile = get_torrentfile(
message.sourcelink, config['APPLICATION_PATH_TORRENTS'])
if os.path.exists(localvideofile):
""" 1) videofile is in place:
1a) push video to ftp
1b) delete videofile, otrkeyfile, torrentfile
1c) delete queue message """
""" 1a) push video to ftp """
uploaded, errormessage = ftp_upload_file2(
log, message.server, message.port, message.user,
message.password, message.destpath, message.videofile,
localvideofile)
if uploaded:
""" 1b) delete videofile, otrkeyfile, torrentfile """
houskeeping.append(localvideofile)
houskeeping.append(localotrkeyfile)
houskeeping.append(localtorrentfile)
""" 1c) delete queue message """
queue.delete(message)
log.info(
'push video queue message {!s} for {!s} successfully processed!'
.format(message.id, message.videofile))
history.status = 'finished'
else:
raise Exception(
'push failed because {}'.format(errormessage))
elif os.path.exists(localotrkeyfile):
""" 2) OR if otrkeyfile is in place
2a) init decodingprocess to videofile
2b) delete transmission queue """
""" 2a) init decodingprocess to videofile """
if message.usecutlist:
localcutlistfile = get_cutlist(
message.otrkeyfile, message.videofile,
config['APPLICATION_PATH_TMP'], log)
else:
localcutlistfile = None
decoded, errormessage = decode(
log, message.otruser, message.otrpassword,
message.usecutlist, localotrkeyfile,
config['APPLICATION_PATH_VIDEOS'], localcutlistfile)
if decoded == 0:
""" successfully decoded """
houskeeping.append(localcutlistfile)
if not downloadstatus is None:
housekeepingTransmission.append(downloadstatus)
log.info(
'decoding otrkeyfile {!s} successfully processed!'.
format(message.otrkeyfile))
history.status = 'decoded'
if not user is None:
user.FtpConnectionChecked = False
db.insert(user)
elif decoded == 255:
""" otr credentials not worked """
if not user is None:
user.OtrCredentialsChecked = False
db.insert(user)
raise Exception(errormessage)
else:
""" other error """
raise Exception(errormessage)
else:
""" 3) ELSE if transmission job is not running
3a) add transmission torrent """
if not downloadstatus is None:
history.status = downloadstatus[
'Status'] + ' ' + downloadstatus[
'Done'] + ' (ETA ' + downloadstatus['ETA'] + ')'
log.info('otrkeyfile {!s} {}'.format(
message.otrkeyfile, history.status))
else:
""" 3a) add transmission torrent """
if os.path.exists(localtorrentfile):
downloaded = True
else:
downloaded, errormessage = download_fromurl(
message.sourcelink, localtorrentfile)
if downloaded:
log.info(
'downloading torrentfile {!s} successfully initiated!'
.format(message.sourcefile))
history.status = 'download started'
else:
raise Exception(errormessage)
except Exception as e:
if isinstance(e, subprocess.CalledProcessError):
errormessage = 'cmd {!s} failed because {!s}, {!s}'.format(
e.cmd, e.stderr, e.stdout)
else:
errormessage = e
log.exception(
'push video failed because {!s}'.format(errormessage))
history.status = 'error'
""" delete message after 3 tries """
if (config['APPLICATION_ENVIRONMENT']
== 'Production') and (message.dequeue_count >= 3):
queue.delete(message)
history.status = 'deleted'
""" update history entry """
history.updated = datetime.now()
db.insert(history)
""" next message """
message = queue.get(PushVideoMessage(), queuehide)
""" housekeeping temporary files """
for file in houskeeping:
if not file is None:
if os.path.exists(file):
os.remove(file)
""" houskeeping torrent queue """
for torrentsinglestate in housekeepingTransmission:
call = 'transmission-remote -t ' + torrentsinglestate['ID'] + ' -r'
process = subprocess.run(call,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.debug('{} finished with {}'.format(
call, process.stdout.decode(encoding='utf-8')))
for torrentsinglestate in transmissionstatus:
""" restart queue entries """
if torrentsinglestate['Status'] == 'Stopped':
call = 'transmission-remote -t ' + torrentsinglestate['ID'] + ' -s'
process = subprocess.run(call,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.debug('{} finished with {}'.format(
call, process.stdout.decode(encoding='utf-8')))
pass