def post(self):
""" parse request data and use model attribute info"""
data = request.json
for key, value in login.items():
if key in data:
data[value.attribute] = data.pop(key)
#log.debug('{!s}: {!s}'.format(value.attribute, data[value.attribute]))
""" retrieve user info """
if 'PartitionKey' in data:
if data['PartitionKey'] == "":
data['PartitionKey'] = config['APPLICATION_CLIENT_ID']
else:
data['PartitionKey'] = config['APPLICATION_CLIENT_ID']
loginuser = db.get(User(**data))
""" user exists ? Create a new and """
if not db.exists(loginuser):
loginuser.created = datetime.now()
db.insert(loginuser)
""" login user """
#log.debug(loginuser.dict())
g.user = loginuser
token = generate_auth_token(loginuser)
session['authtoken'] = token
""" prepare return dict """
data['loggedin'] = True
data['timeout'] = 600
log.debug(session)
return data, 200
def post(self):
""" create a new push task for a decoded video in queue to be worked off (only available for PRO clients) """
""" retrieve client role """
AuthUser = g.get('clientrole')
if AuthUser is None:
api.abort(401, __class__._responses['post'][401])
if AuthUser != 'PRO':
api.abort(403, __class__._responses['post'][403])
""" parse request data and use model attribute info"""
requestbody = request.json
log.debug(requestbody)
""" further validations:
test connection to ftp destination server
"""
validftp, validftpmessage = test_ftpconnection(
requestbody['Server'], requestbody['Port'], requestbody['User'],
requestbody['Password'], requestbody['ServerPath'])
if not validftp:
log.error(validftpmessage)
api.abort(
403,
__class__._responses['post'][403] + ': ' + validftpmessage)
""" create jobchain DownloadMessage -----------------------------------------------------------------"""
Content = {}
for key, value in requestbody.items():
if key in video.keys():
Content[video[key].attribute] = value
message = PushVideoMessage(**Content)
""" update otrkey filename as output """
message.otrkeyfile = os.path.splitext(
os.path.basename(message.sourcelink))[0]
message.videofile = os.path.splitext(
os.path.basename(message.otrkeyfile))[0]
""" put download queue message """
message = queue.put(message)
if message is None:
api.abort(403, __class__._responses['post'][403])
""" prepare return data """
message.otrpassword = '******'
message.password = '******'
""" add history entry """
recording = db.get(
Recording(PartitionKey='top', RowKey=str(data['epgid'])))
AddHistory(g.user, message.id, 'torrent', recording.Id,
recording.beginn, recording.sender, recording.titel,
recording.genre, recording.previewimagelink,
data['resolution'], message.sourcefile, request.remote_addr,
request.user_agent.platform, request.user_agent.browser,
request.user_agent.version, request.user_agent.language)
return message, 200
def get(self, id):
""" request top recording detail data """
""" logging """
log.info('select all details for recording: {!s}'.format(id))
""" retrieve board """
recording = db.get(Recording(PartitionKey='top', RowKey=str(id)))
if not db.exists(recording):
api.abort(404, __class__._responses['get'][404])
recording.Torrents = db.query(recording.Torrents)
""" return recording """
return recording, 200
def get(self, id):
""" request job detail data """
""" logging """
log.info('select history for job: {!s}'.format(id))
""" find history item for id """
history = db.get(History(PartitionKey=g.user.RowKey, RowKey=id))
log.debug('found history: {!s}'.format(history))
if (history is None) or not db.exists(history):
api.abort(404, __class__._responses['get'][404])
else:
""" return recording """
return history, 200
def verify_auth_token(token):
s = Serializer(config['SECRET_KEY'])
try:
data = s.loads(token)
except SignatureExpired:
return None # valid token, but expired
except BadSignature:
return None # invalid token
user = db.get(
User(PartitionKey=data['PartitionKey'], RowKey=data['RowKey']))
if db.exists(user):
return user
else:
return None
def wrapper(*args, **kwargs):
if g.deviceuuid == '' or g.clientid == '':
g.user = None
else:
loginuser = db.get(
User(PartitionKey=g.clientid, RowKey=g.deviceuuid))
""" user exists ? Create a new and """
if not db.exists(loginuser):
loginuser.created = datetime.now()
db.insert(loginuser)
""" login user """
g.user = loginuser
if backtoindex and not g.user:
return redirect(url_for('ui.index', messageid=4))
else:
return func(*args, **kwargs)
def post(self):
""" create push job in queue for torrent/video file """
""" parse request data and use model attribute info"""
data = request.json
log.debug(data)
for key, value in push.items():
if key in data:
data[value.attribute] = data.pop(key)
""" further validations:
test connection to ftp destination server
"""
validftp, validftpmessage = test_ftpconnection(data['server'],
data['port'],
data['user'],
data['password'],
data['destpath'])
if not validftp:
log.error(validftpmessage)
api.abort(
403,
__class__._responses['post'][403] + ': ' + validftpmessage)
""" init a PushMessage instance and put it into queue """
message = PushMessage(**data)
message = queue.put(message)
if message is None:
api.abort(403, __class__._responses['post'][403])
""" return message """
message.password = '******'
""" add history """
recording = db.get(
Recording(PartitionKey='top', RowKey=str(data['epgid'])))
AddHistory(g.user, message.id, 'torrent', recording.Id,
recording.beginn, recording.sender, recording.titel,
recording.genre, recording.previewimagelink,
data['resolution'], message.sourcefile, request.remote_addr,
request.user_agent.platform, request.user_agent.browser,
request.user_agent.version, request.user_agent.language)
log.debug('mesage input: {!s}, {!s}, {!s}'.format(
message.getmessage(), message.id, message.insertion_time))
return message, 200
def verify_password(clientid_or_token, fingerprint):
# first try to authenticate by token
user = verify_auth_token(clientid_or_token)
if not user:
# try to authenticate with username/password
user = db.get(User(PartitionKey=clientid_or_token, RowKey=fingerprint))
if not db.exists(user):
return False
""" retrieve user and set global """
g.user = user
session['authtoken'] = generate_auth_token(user)
""" verify by clientid """
if user.ProUser:
g.clientrole = 'PRO'
else:
g.clientrole = 'BASIC'
log.debug('client identified with role {}'.format(g.clientrole))
return True
def update_toprecordings(config, log):
""" Rufe alle top bewerteten otr Aufzeichnungen ab:
https://www.onlinetvrecorder.com/v2/?go=list&tab=toplist&tlview=all&listid=104&start=0
in 20er Paketen wird der content durchsucht und bei hohen Bewertungen das Recording in eine neue Partition verschoben
"""
log.debug('try to update toprecordings webcontent...')
stopflag = False
start = 0
toplist = []
db.register_model(Recording())
while not stopflag:
""" download webcontent into content"""
with urllib.request.urlopen(
'https://www.onlinetvrecorder.com/v2/?go=list&tab=toplist&tlview=all&listid=104&start='
+ str(start)) as response:
content = response.read()
""" für jeden Eintrag in ID= searchrow """
content = str(content.decode('utf-8',
'ignore')).split("<tr id='serchrow")
for index in range(1, len(content)):
lines = content[index].split('<td oncontextmenu="showNewTabMenu(')
""" parse option #1 """
parsed = False
try:
epg_id = lines[1].split(',')[0]
previewimagelink = lines[10].split('<img src=')[1].split(
' width=')[0]
primarykey = datetime.strptime(
lines[4].split('>')[1].split('<')[0],
'%d.%m.%y').date().strftime('%Y_%m_%d')
rating = lines[8].split('Beliebtheit: ')[1].split("'")[0]
log.debug(
'parsed recording: {} with rating: {} and preview = {}'.
format(epg_id, rating, previewimagelink))
parsed = True
except Exception as e:
log.error(
'parsing option #1 not possible {!s} due to {!s}'.format(
lines, e))
""" parse option #2 """
if not parsed:
try:
epg_id = lines[1].split(',')[0]
previewimagelink = lines[8].split('<img src=')[1].split(
' width=')[0]
primarykey = datetime.strptime(
lines[2].split('>')[1].split('<')[0],
'%d.%m.%y').date().strftime('%Y_%m_%d')
rating = lines[6].split('Beliebtheit: ')[1].split("'")[0]
log.debug(
'parsed recording: {} with rating: {} and preview = {}'
.format(epg_id, rating, previewimagelink))
parsed = True
except Exception as e:
log.error(
'parsing option #2 not possible {!s} due to {!s}'.
format(lines, e))
""" save top records to storage """
if parsed:
top = db.get(Recording(PartitionKey=primarykey, RowKey=epg_id))
if not top is None:
top.rating = rating
top.previewimagelink = previewimagelink.replace(
'http://', 'https://')
top.PartitionKey = 'top'
db.insert(top)
if top.rating not in ['sehr hoch', 'hoch']:
stopflag = True
log.info(
'recording {} moved or is already moved successfully ({}, {!s}, at {})'
.format(epg_id, top.titel, top.beginn, top.sender))
else:
log.info(
'epg not found: {} with rating: {} and preview = {}'.
format(epg_id, rating, previewimagelink))
start = start + 20
log.info('toprecordings successfully retireved!')
def details(epgid):
""" request top recording detail data """
""" logging """
log.info('select all details for recording: {!s} with method {!s}'.format(
epgid, request.method))
""" determine template & messages """
pathtemplate = g.platform + '/' + 'details.html'
#get message
message = g.message
g.pop('message')
""" retrieve recording from storage """
recording = db.get(Recording(PartitionKey='top', RowKey=str(epgid)))
""" show details including all torrents """
if not db.exists(recording):
return index()
recording.Torrents = db.query(recording.Torrents)
recording.startdate = recording.beginn.strftime('%d.%m.%Y')
recording.starttime = recording.beginn.strftime('%H:%M')
recording.previewimagelink = recording.previewimagelink.replace(
'http://', 'https://', 1)
if request.method == 'POST':
""" retrieve Form Data """
data = request.form.to_dict()
log.debug(data)
""" post method = push only available for logged in users """
if not g.user:
message.show = True
message.error = True
message.header = 'Fehler'
message.text = 'Bitte loggen Sie sich ein!'
elif (not g.user.ProUser) and (g.user.PushVideo):
message.show = True
message.error = True
message.header = 'Fehler'
message.text = 'Um Videos zu decodieren müssen Sie Pro Status haben!'
elif ExistsHistory(g.user.RowKey, epgid):
""" already in History ? """
message.show = True
message.error = True
message.header = 'Fehler'
message.text = 'Sie haben diese Aufnahme bereits erfolgreich gepushed!'
elif (g.user.PushVideo):
""" successfully configured ? """
if not g.user.FtpConnectionChecked:
return redirect(url_for('ui.settings', messageid=5))
""" push video """
job, errormessage = PushVideo(epgid, data['Resolution'],
data['TorrentFile'],
data['TorrentLink'], g.user)
if not job:
message.show = True
message.error = True
message.header = 'Fehler'
message.text = errormessage
else:
""" success """
message.show = True
message.header = 'Erfolg'
message.text = '{!s} wird heruntergeladen, decodiert und danach an Ihren Endpoint gepushed. Die Aufgabe {!s} wurde dazu erfolgreich angelegt'.format(
job.sourcefile, job.id)
""" add history """
AddHistory(g.user, job.id, 'video', epgid, recording.beginn,
recording.sender, recording.titel, recording.genre,
recording.previewimagelink, data['Resolution'],
job.sourcefile, request.remote_addr,
request.user_agent.platform,
request.user_agent.browser,
request.user_agent.version,
request.user_agent.language)
else:
""" successfully configured ? """
if not g.user.FtpConnectionChecked:
return redirect(url_for('ui.settings', messageid=5))
""" push torrent """
job, errormessage = PushTorrent(epgid, data['Resolution'],
data['TorrentFile'],
data['TorrentLink'], g.user)
if not job:
message.show = True
message.error = True
message.header = 'Fehler'
message.text = errormessage
else:
""" success """
message.show = True
message.header = 'Erfolg'
message.text = '{!s} wird heruntergeladen und an Ihren Endpoint gepushed. Die Aufgabe {!s} wurde dazu erfolgreich angelegt'.format(
job.sourcefile, job.id)
""" add history """
AddHistory(g.user, job.id, 'torrent', epgid, recording.beginn,
recording.sender, recording.titel, recording.genre,
recording.previewimagelink, data['Resolution'],
job.sourcefile, request.remote_addr,
request.user_agent.platform,
request.user_agent.browser,
request.user_agent.version,
request.user_agent.language)
""" render platform template """
return render_template(pathtemplate,
title='OTR Aufnahme Details',
pagetitle='details',
item=recording,
message=message)
def do_pushtorrent_queue_message(config, log):
""" retrieve and process all visible push queue messages
- if link is local check if file exist then push to ftp endpoint
- if link is url then download torrentfile and push to endpoint
"""
queue.register_model(PushMessage())
db.register_model(History())
if config['APPLICATION_ENVIRONMENT'] in ['Development', 'Test']:
queuehide = 1
else:
queuehide = 5 * 60
""" loop all push queue messages """
message = queue.get(PushMessage(), queuehide)
while not message is None:
""" get history entry for message for an status update """
historylist = StorageTableCollection('history',
"RowKey eq '" + message.id + "'")
historylist = db.query(historylist)
for item in historylist:
history = db.get(
History(PartitionKey=item.PartitionKey, RowKey=message.id))
if not history:
history = History(PartitionKey='torrent', RowKey=message.id)
history.created = datetime.now()
history.epgid = message.epgid
history.sourcefile = message.sourcefile
if message.sourcelink in ['', 'string']:
""" no sourcelink ? """
""" delete queue message and tmp file """
queue.delete(message)
else:
""" push torrentfile
---------------------------------------------------------------------
1) download torrent to local tmp folder
2) pushfile to ftp
3) delete torrent from local tmp folder
4) delete queue message
"""
""" 1) download torrent to local tmp folder """
filename, localfile = get_torrentfile(
message.sourcelink, config['APPLICATION_PATH_TMP'])
if (not filename is None) and (not localfile is None):
downloaded, errormessage = download_fromurl(
message.sourcelink, localfile)
if downloaded:
""" 2) pushfile to ftp """
uploaded, errormessage = ftp_upload_file2(
log, message.server, message.port, message.user,
message.password, message.destpath, filename,
localfile)
if uploaded:
""" 3) delete torrent from local tmp folder,
4) delete queue message
"""
queue.delete(message)
if os.path.exists(localfile):
os.remove(localfile)
history.status = 'finished'
if not errormessage is None:
""" delete message after 3 tries """
log.error('push failed because {}'.format(errormessage))
history.status = 'error'
if (config['APPLICATION_ENVIRONMENT']
== 'Production') and (message.dequeue_count >= 3):
queue.delete(message)
history.status = 'deleted'
if os.path.exists(localfile):
os.remove(localfile)
""" update history entry """
history.updated = datetime.now()
db.insert(history)
""" next message """
message = queue.get(PushMessage(), queuehide)
pass
def do_pushvideo_queue_message(config, log):
""" retrieve and process all visible download queue mess ages
- if link is url then download torrentfile
---------------------------------------------------------------------
1) if videofile is in place:
1a) push video to ftp
1b) delete videofile, otrkeyfile, torrentfile
1c) delete queue message
2) OR if otrkeyfile is in place
2a) init decodingprocess to videofile
3) ELSE if transmission job is not running
3a) add transmission torrent
"""
queue.register_model(PushVideoMessage())
db.register_model(History())
if config['APPLICATION_ENVIRONMENT'] in ['Development', 'Test']:
queuehide = 60
else:
queuehide = 5 * 60
""" housekeeping array of files and transmission-queues to be deleted """
houskeeping = []
housekeepingTransmission = []
""" get transmission status """
transmissionstatus = get_transmissionstatus(log)
""" loop all push queue messages """
message = queue.get(PushVideoMessage(), queuehide)
while not message is None:
""" get history entry for message for an status update """
historylist = StorageTableCollection('history',
"RowKey eq '" + message.id + "'")
historylist = db.query(historylist)
for item in historylist:
history = db.get(
History(PartitionKey=item.PartitionKey, RowKey=message.id))
if not history:
history = History(PartitionKey='video', RowKey=message.id)
history.created = datetime.now()
history.epgid = message.epgid
history.sourcefile = message.videofile
user = None
else:
""" get user """
user = None
userlist = db.query(
StorageTableCollection(
'userprofile', "RowKey eq '" + history.PartitionKey + "'"))
for item in userlist:
user = db.get(
User(PartitionKey=item.PartitionKey,
RowKey=history.PartitionKey))
#log.debug('{!s}'.format(history.dict()))
""" get single transmission download status """
downloadstatus = [
element for element in transmissionstatus
if element['Name'] == message.otrkeyfile
]
if downloadstatus != []:
downloadstatus = downloadstatus[0]
else:
downloadstatus = None
#log.debug('{!s}'.format(downloadstatus))
if message.sourcelink in ['', 'string']:
""" no sourcelink ? """
""" delete queue message and tmp file """
queue.delete(message)
history.status = 'deleted'
else:
""" process push video """
try:
localvideofile = os.path.join(
config['APPLICATION_PATH_VIDEOS'], message.videofile)
localotrkeyfile = os.path.join(
config['APPLICATION_PATH_OTRKEYS'], message.otrkeyfile)
message.sourcefile, localtorrentfile = get_torrentfile(
message.sourcelink, config['APPLICATION_PATH_TORRENTS'])
if os.path.exists(localvideofile):
""" 1) videofile is in place:
1a) push video to ftp
1b) delete videofile, otrkeyfile, torrentfile
1c) delete queue message """
""" 1a) push video to ftp """
uploaded, errormessage = ftp_upload_file2(
log, message.server, message.port, message.user,
message.password, message.destpath, message.videofile,
localvideofile)
if uploaded:
""" 1b) delete videofile, otrkeyfile, torrentfile """
houskeeping.append(localvideofile)
houskeeping.append(localotrkeyfile)
houskeeping.append(localtorrentfile)
""" 1c) delete queue message """
queue.delete(message)
log.info(
'push video queue message {!s} for {!s} successfully processed!'
.format(message.id, message.videofile))
history.status = 'finished'
else:
raise Exception(
'push failed because {}'.format(errormessage))
elif os.path.exists(localotrkeyfile):
""" 2) OR if otrkeyfile is in place
2a) init decodingprocess to videofile
2b) delete transmission queue """
""" 2a) init decodingprocess to videofile """
if message.usecutlist:
localcutlistfile = get_cutlist(
message.otrkeyfile, message.videofile,
config['APPLICATION_PATH_TMP'], log)
else:
localcutlistfile = None
decoded, errormessage = decode(
log, message.otruser, message.otrpassword,
message.usecutlist, localotrkeyfile,
config['APPLICATION_PATH_VIDEOS'], localcutlistfile)
if decoded == 0:
""" successfully decoded """
houskeeping.append(localcutlistfile)
if not downloadstatus is None:
housekeepingTransmission.append(downloadstatus)
log.info(
'decoding otrkeyfile {!s} successfully processed!'.
format(message.otrkeyfile))
history.status = 'decoded'
if not user is None:
user.FtpConnectionChecked = False
db.insert(user)
elif decoded == 255:
""" otr credentials not worked """
if not user is None:
user.OtrCredentialsChecked = False
db.insert(user)
raise Exception(errormessage)
else:
""" other error """
raise Exception(errormessage)
else:
""" 3) ELSE if transmission job is not running
3a) add transmission torrent """
if not downloadstatus is None:
history.status = downloadstatus[
'Status'] + ' ' + downloadstatus[
'Done'] + ' (ETA ' + downloadstatus['ETA'] + ')'
log.info('otrkeyfile {!s} {}'.format(
message.otrkeyfile, history.status))
else:
""" 3a) add transmission torrent """
if os.path.exists(localtorrentfile):
downloaded = True
else:
downloaded, errormessage = download_fromurl(
message.sourcelink, localtorrentfile)
if downloaded:
log.info(
'downloading torrentfile {!s} successfully initiated!'
.format(message.sourcefile))
history.status = 'download started'
else:
raise Exception(errormessage)
except Exception as e:
if isinstance(e, subprocess.CalledProcessError):
errormessage = 'cmd {!s} failed because {!s}, {!s}'.format(
e.cmd, e.stderr, e.stdout)
else:
errormessage = e
log.exception(
'push video failed because {!s}'.format(errormessage))
history.status = 'error'
""" delete message after 3 tries """
if (config['APPLICATION_ENVIRONMENT']
== 'Production') and (message.dequeue_count >= 3):
queue.delete(message)
history.status = 'deleted'
""" update history entry """
history.updated = datetime.now()
db.insert(history)
""" next message """
message = queue.get(PushVideoMessage(), queuehide)
""" housekeeping temporary files """
for file in houskeeping:
if not file is None:
if os.path.exists(file):
os.remove(file)
""" houskeeping torrent queue """
for torrentsinglestate in housekeepingTransmission:
call = 'transmission-remote -t ' + torrentsinglestate['ID'] + ' -r'
process = subprocess.run(call,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.debug('{} finished with {}'.format(
call, process.stdout.decode(encoding='utf-8')))
for torrentsinglestate in transmissionstatus:
""" restart queue entries """
if torrentsinglestate['Status'] == 'Stopped':
call = 'transmission-remote -t ' + torrentsinglestate['ID'] + ' -s'
process = subprocess.run(call,
shell=True,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.debug('{} finished with {}'.format(
call, process.stdout.decode(encoding='utf-8')))
pass
