def post(self): """ parse request data and use model attribute info""" data = request.json for key, value in login.items(): if key in data: data[value.attribute] = data.pop(key) #log.debug('{!s}: {!s}'.format(value.attribute, data[value.attribute])) """ retrieve user info """ if 'PartitionKey' in data: if data['PartitionKey'] == "": data['PartitionKey'] = config['APPLICATION_CLIENT_ID'] else: data['PartitionKey'] = config['APPLICATION_CLIENT_ID'] loginuser = db.get(User(**data)) """ user exists ? Create a new and """ if not db.exists(loginuser): loginuser.created = datetime.now() db.insert(loginuser) """ login user """ #log.debug(loginuser.dict()) g.user = loginuser token = generate_auth_token(loginuser) session['authtoken'] = token """ prepare return dict """ data['loggedin'] = True data['timeout'] = 600 log.debug(session) return data, 200
def post(self): """ create a new push task for a decoded video in queue to be worked off (only available for PRO clients) """ """ retrieve client role """ AuthUser = g.get('clientrole') if AuthUser is None: api.abort(401, __class__._responses['post'][401]) if AuthUser != 'PRO': api.abort(403, __class__._responses['post'][403]) """ parse request data and use model attribute info""" requestbody = request.json log.debug(requestbody) """ further validations: test connection to ftp destination server """ validftp, validftpmessage = test_ftpconnection( requestbody['Server'], requestbody['Port'], requestbody['User'], requestbody['Password'], requestbody['ServerPath']) if not validftp: log.error(validftpmessage) api.abort( 403, __class__._responses['post'][403] + ': ' + validftpmessage) """ create jobchain DownloadMessage -----------------------------------------------------------------""" Content = {} for key, value in requestbody.items(): if key in video.keys(): Content[video[key].attribute] = value message = PushVideoMessage(**Content) """ update otrkey filename as output """ message.otrkeyfile = os.path.splitext( os.path.basename(message.sourcelink))[0] message.videofile = os.path.splitext( os.path.basename(message.otrkeyfile))[0] """ put download queue message """ message = queue.put(message) if message is None: api.abort(403, __class__._responses['post'][403]) """ prepare return data """ message.otrpassword = '******' message.password = '******' """ add history entry """ recording = db.get( Recording(PartitionKey='top', RowKey=str(data['epgid']))) AddHistory(g.user, message.id, 'torrent', recording.Id, recording.beginn, recording.sender, recording.titel, recording.genre, recording.previewimagelink, data['resolution'], message.sourcefile, request.remote_addr, request.user_agent.platform, request.user_agent.browser, request.user_agent.version, request.user_agent.language) return message, 200
def get(self, id): """ request top recording detail data """ """ logging """ log.info('select all details for recording: {!s}'.format(id)) """ retrieve board """ recording = db.get(Recording(PartitionKey='top', RowKey=str(id))) if not db.exists(recording): api.abort(404, __class__._responses['get'][404]) recording.Torrents = db.query(recording.Torrents) """ return recording """ return recording, 200
def get(self, id): """ request job detail data """ """ logging """ log.info('select history for job: {!s}'.format(id)) """ find history item for id """ history = db.get(History(PartitionKey=g.user.RowKey, RowKey=id)) log.debug('found history: {!s}'.format(history)) if (history is None) or not db.exists(history): api.abort(404, __class__._responses['get'][404]) else: """ return recording """ return history, 200
def verify_auth_token(token): s = Serializer(config['SECRET_KEY']) try: data = s.loads(token) except SignatureExpired: return None # valid token, but expired except BadSignature: return None # invalid token user = db.get( User(PartitionKey=data['PartitionKey'], RowKey=data['RowKey'])) if db.exists(user): return user else: return None
def wrapper(*args, **kwargs): if g.deviceuuid == '' or g.clientid == '': g.user = None else: loginuser = db.get( User(PartitionKey=g.clientid, RowKey=g.deviceuuid)) """ user exists ? Create a new and """ if not db.exists(loginuser): loginuser.created = datetime.now() db.insert(loginuser) """ login user """ g.user = loginuser if backtoindex and not g.user: return redirect(url_for('ui.index', messageid=4)) else: return func(*args, **kwargs)
def post(self): """ create push job in queue for torrent/video file """ """ parse request data and use model attribute info""" data = request.json log.debug(data) for key, value in push.items(): if key in data: data[value.attribute] = data.pop(key) """ further validations: test connection to ftp destination server """ validftp, validftpmessage = test_ftpconnection(data['server'], data['port'], data['user'], data['password'], data['destpath']) if not validftp: log.error(validftpmessage) api.abort( 403, __class__._responses['post'][403] + ': ' + validftpmessage) """ init a PushMessage instance and put it into queue """ message = PushMessage(**data) message = queue.put(message) if message is None: api.abort(403, __class__._responses['post'][403]) """ return message """ message.password = '******' """ add history """ recording = db.get( Recording(PartitionKey='top', RowKey=str(data['epgid']))) AddHistory(g.user, message.id, 'torrent', recording.Id, recording.beginn, recording.sender, recording.titel, recording.genre, recording.previewimagelink, data['resolution'], message.sourcefile, request.remote_addr, request.user_agent.platform, request.user_agent.browser, request.user_agent.version, request.user_agent.language) log.debug('mesage input: {!s}, {!s}, {!s}'.format( message.getmessage(), message.id, message.insertion_time)) return message, 200
def verify_password(clientid_or_token, fingerprint): # first try to authenticate by token user = verify_auth_token(clientid_or_token) if not user: # try to authenticate with username/password user = db.get(User(PartitionKey=clientid_or_token, RowKey=fingerprint)) if not db.exists(user): return False """ retrieve user and set global """ g.user = user session['authtoken'] = generate_auth_token(user) """ verify by clientid """ if user.ProUser: g.clientrole = 'PRO' else: g.clientrole = 'BASIC' log.debug('client identified with role {}'.format(g.clientrole)) return True
def update_toprecordings(config, log): """ Rufe alle top bewerteten otr Aufzeichnungen ab: https://www.onlinetvrecorder.com/v2/?go=list&tab=toplist&tlview=all&listid=104&start=0 in 20er Paketen wird der content durchsucht und bei hohen Bewertungen das Recording in eine neue Partition verschoben """ log.debug('try to update toprecordings webcontent...') stopflag = False start = 0 toplist = [] db.register_model(Recording()) while not stopflag: """ download webcontent into content""" with urllib.request.urlopen( 'https://www.onlinetvrecorder.com/v2/?go=list&tab=toplist&tlview=all&listid=104&start=' + str(start)) as response: content = response.read() """ für jeden Eintrag in ID= searchrow """ content = str(content.decode('utf-8', 'ignore')).split("<tr id='serchrow") for index in range(1, len(content)): lines = content[index].split('<td oncontextmenu="showNewTabMenu(') """ parse option #1 """ parsed = False try: epg_id = lines[1].split(',')[0] previewimagelink = lines[10].split('<img src=')[1].split( ' width=')[0] primarykey = datetime.strptime( lines[4].split('>')[1].split('<')[0], '%d.%m.%y').date().strftime('%Y_%m_%d') rating = lines[8].split('Beliebtheit: ')[1].split("'")[0] log.debug( 'parsed recording: {} with rating: {} and preview = {}'. format(epg_id, rating, previewimagelink)) parsed = True except Exception as e: log.error( 'parsing option #1 not possible {!s} due to {!s}'.format( lines, e)) """ parse option #2 """ if not parsed: try: epg_id = lines[1].split(',')[0] previewimagelink = lines[8].split('<img src=')[1].split( ' width=')[0] primarykey = datetime.strptime( lines[2].split('>')[1].split('<')[0], '%d.%m.%y').date().strftime('%Y_%m_%d') rating = lines[6].split('Beliebtheit: ')[1].split("'")[0] log.debug( 'parsed recording: {} with rating: {} and preview = {}' .format(epg_id, rating, previewimagelink)) parsed = True except Exception as e: log.error( 'parsing option #2 not possible {!s} due to {!s}'. format(lines, e)) """ save top records to storage """ if parsed: top = db.get(Recording(PartitionKey=primarykey, RowKey=epg_id)) if not top is None: top.rating = rating top.previewimagelink = previewimagelink.replace( 'http://', 'https://') top.PartitionKey = 'top' db.insert(top) if top.rating not in ['sehr hoch', 'hoch']: stopflag = True log.info( 'recording {} moved or is already moved successfully ({}, {!s}, at {})' .format(epg_id, top.titel, top.beginn, top.sender)) else: log.info( 'epg not found: {} with rating: {} and preview = {}'. format(epg_id, rating, previewimagelink)) start = start + 20 log.info('toprecordings successfully retireved!')
def details(epgid): """ request top recording detail data """ """ logging """ log.info('select all details for recording: {!s} with method {!s}'.format( epgid, request.method)) """ determine template & messages """ pathtemplate = g.platform + '/' + 'details.html' #get message message = g.message g.pop('message') """ retrieve recording from storage """ recording = db.get(Recording(PartitionKey='top', RowKey=str(epgid))) """ show details including all torrents """ if not db.exists(recording): return index() recording.Torrents = db.query(recording.Torrents) recording.startdate = recording.beginn.strftime('%d.%m.%Y') recording.starttime = recording.beginn.strftime('%H:%M') recording.previewimagelink = recording.previewimagelink.replace( 'http://', 'https://', 1) if request.method == 'POST': """ retrieve Form Data """ data = request.form.to_dict() log.debug(data) """ post method = push only available for logged in users """ if not g.user: message.show = True message.error = True message.header = 'Fehler' message.text = 'Bitte loggen Sie sich ein!' elif (not g.user.ProUser) and (g.user.PushVideo): message.show = True message.error = True message.header = 'Fehler' message.text = 'Um Videos zu decodieren müssen Sie Pro Status haben!' elif ExistsHistory(g.user.RowKey, epgid): """ already in History ? """ message.show = True message.error = True message.header = 'Fehler' message.text = 'Sie haben diese Aufnahme bereits erfolgreich gepushed!' elif (g.user.PushVideo): """ successfully configured ? """ if not g.user.FtpConnectionChecked: return redirect(url_for('ui.settings', messageid=5)) """ push video """ job, errormessage = PushVideo(epgid, data['Resolution'], data['TorrentFile'], data['TorrentLink'], g.user) if not job: message.show = True message.error = True message.header = 'Fehler' message.text = errormessage else: """ success """ message.show = True message.header = 'Erfolg' message.text = '{!s} wird heruntergeladen, decodiert und danach an Ihren Endpoint gepushed. Die Aufgabe {!s} wurde dazu erfolgreich angelegt'.format( job.sourcefile, job.id) """ add history """ AddHistory(g.user, job.id, 'video', epgid, recording.beginn, recording.sender, recording.titel, recording.genre, recording.previewimagelink, data['Resolution'], job.sourcefile, request.remote_addr, request.user_agent.platform, request.user_agent.browser, request.user_agent.version, request.user_agent.language) else: """ successfully configured ? """ if not g.user.FtpConnectionChecked: return redirect(url_for('ui.settings', messageid=5)) """ push torrent """ job, errormessage = PushTorrent(epgid, data['Resolution'], data['TorrentFile'], data['TorrentLink'], g.user) if not job: message.show = True message.error = True message.header = 'Fehler' message.text = errormessage else: """ success """ message.show = True message.header = 'Erfolg' message.text = '{!s} wird heruntergeladen und an Ihren Endpoint gepushed. Die Aufgabe {!s} wurde dazu erfolgreich angelegt'.format( job.sourcefile, job.id) """ add history """ AddHistory(g.user, job.id, 'torrent', epgid, recording.beginn, recording.sender, recording.titel, recording.genre, recording.previewimagelink, data['Resolution'], job.sourcefile, request.remote_addr, request.user_agent.platform, request.user_agent.browser, request.user_agent.version, request.user_agent.language) """ render platform template """ return render_template(pathtemplate, title='OTR Aufnahme Details', pagetitle='details', item=recording, message=message)
def do_pushtorrent_queue_message(config, log): """ retrieve and process all visible push queue messages - if link is local check if file exist then push to ftp endpoint - if link is url then download torrentfile and push to endpoint """ queue.register_model(PushMessage()) db.register_model(History()) if config['APPLICATION_ENVIRONMENT'] in ['Development', 'Test']: queuehide = 1 else: queuehide = 5 * 60 """ loop all push queue messages """ message = queue.get(PushMessage(), queuehide) while not message is None: """ get history entry for message for an status update """ historylist = StorageTableCollection('history', "RowKey eq '" + message.id + "'") historylist = db.query(historylist) for item in historylist: history = db.get( History(PartitionKey=item.PartitionKey, RowKey=message.id)) if not history: history = History(PartitionKey='torrent', RowKey=message.id) history.created = datetime.now() history.epgid = message.epgid history.sourcefile = message.sourcefile if message.sourcelink in ['', 'string']: """ no sourcelink ? """ """ delete queue message and tmp file """ queue.delete(message) else: """ push torrentfile --------------------------------------------------------------------- 1) download torrent to local tmp folder 2) pushfile to ftp 3) delete torrent from local tmp folder 4) delete queue message """ """ 1) download torrent to local tmp folder """ filename, localfile = get_torrentfile( message.sourcelink, config['APPLICATION_PATH_TMP']) if (not filename is None) and (not localfile is None): downloaded, errormessage = download_fromurl( message.sourcelink, localfile) if downloaded: """ 2) pushfile to ftp """ uploaded, errormessage = ftp_upload_file2( log, message.server, message.port, message.user, message.password, message.destpath, filename, localfile) if uploaded: """ 3) delete torrent from local tmp folder, 4) delete queue message """ queue.delete(message) if os.path.exists(localfile): os.remove(localfile) history.status = 'finished' if not errormessage is None: """ delete message after 3 tries """ log.error('push failed because {}'.format(errormessage)) history.status = 'error' if (config['APPLICATION_ENVIRONMENT'] == 'Production') and (message.dequeue_count >= 3): queue.delete(message) history.status = 'deleted' if os.path.exists(localfile): os.remove(localfile) """ update history entry """ history.updated = datetime.now() db.insert(history) """ next message """ message = queue.get(PushMessage(), queuehide) pass
def do_pushvideo_queue_message(config, log): """ retrieve and process all visible download queue mess ages - if link is url then download torrentfile --------------------------------------------------------------------- 1) if videofile is in place: 1a) push video to ftp 1b) delete videofile, otrkeyfile, torrentfile 1c) delete queue message 2) OR if otrkeyfile is in place 2a) init decodingprocess to videofile 3) ELSE if transmission job is not running 3a) add transmission torrent """ queue.register_model(PushVideoMessage()) db.register_model(History()) if config['APPLICATION_ENVIRONMENT'] in ['Development', 'Test']: queuehide = 60 else: queuehide = 5 * 60 """ housekeeping array of files and transmission-queues to be deleted """ houskeeping = [] housekeepingTransmission = [] """ get transmission status """ transmissionstatus = get_transmissionstatus(log) """ loop all push queue messages """ message = queue.get(PushVideoMessage(), queuehide) while not message is None: """ get history entry for message for an status update """ historylist = StorageTableCollection('history', "RowKey eq '" + message.id + "'") historylist = db.query(historylist) for item in historylist: history = db.get( History(PartitionKey=item.PartitionKey, RowKey=message.id)) if not history: history = History(PartitionKey='video', RowKey=message.id) history.created = datetime.now() history.epgid = message.epgid history.sourcefile = message.videofile user = None else: """ get user """ user = None userlist = db.query( StorageTableCollection( 'userprofile', "RowKey eq '" + history.PartitionKey + "'")) for item in userlist: user = db.get( User(PartitionKey=item.PartitionKey, RowKey=history.PartitionKey)) #log.debug('{!s}'.format(history.dict())) """ get single transmission download status """ downloadstatus = [ element for element in transmissionstatus if element['Name'] == message.otrkeyfile ] if downloadstatus != []: downloadstatus = downloadstatus[0] else: downloadstatus = None #log.debug('{!s}'.format(downloadstatus)) if message.sourcelink in ['', 'string']: """ no sourcelink ? """ """ delete queue message and tmp file """ queue.delete(message) history.status = 'deleted' else: """ process push video """ try: localvideofile = os.path.join( config['APPLICATION_PATH_VIDEOS'], message.videofile) localotrkeyfile = os.path.join( config['APPLICATION_PATH_OTRKEYS'], message.otrkeyfile) message.sourcefile, localtorrentfile = get_torrentfile( message.sourcelink, config['APPLICATION_PATH_TORRENTS']) if os.path.exists(localvideofile): """ 1) videofile is in place: 1a) push video to ftp 1b) delete videofile, otrkeyfile, torrentfile 1c) delete queue message """ """ 1a) push video to ftp """ uploaded, errormessage = ftp_upload_file2( log, message.server, message.port, message.user, message.password, message.destpath, message.videofile, localvideofile) if uploaded: """ 1b) delete videofile, otrkeyfile, torrentfile """ houskeeping.append(localvideofile) houskeeping.append(localotrkeyfile) houskeeping.append(localtorrentfile) """ 1c) delete queue message """ queue.delete(message) log.info( 'push video queue message {!s} for {!s} successfully processed!' .format(message.id, message.videofile)) history.status = 'finished' else: raise Exception( 'push failed because {}'.format(errormessage)) elif os.path.exists(localotrkeyfile): """ 2) OR if otrkeyfile is in place 2a) init decodingprocess to videofile 2b) delete transmission queue """ """ 2a) init decodingprocess to videofile """ if message.usecutlist: localcutlistfile = get_cutlist( message.otrkeyfile, message.videofile, config['APPLICATION_PATH_TMP'], log) else: localcutlistfile = None decoded, errormessage = decode( log, message.otruser, message.otrpassword, message.usecutlist, localotrkeyfile, config['APPLICATION_PATH_VIDEOS'], localcutlistfile) if decoded == 0: """ successfully decoded """ houskeeping.append(localcutlistfile) if not downloadstatus is None: housekeepingTransmission.append(downloadstatus) log.info( 'decoding otrkeyfile {!s} successfully processed!'. format(message.otrkeyfile)) history.status = 'decoded' if not user is None: user.FtpConnectionChecked = False db.insert(user) elif decoded == 255: """ otr credentials not worked """ if not user is None: user.OtrCredentialsChecked = False db.insert(user) raise Exception(errormessage) else: """ other error """ raise Exception(errormessage) else: """ 3) ELSE if transmission job is not running 3a) add transmission torrent """ if not downloadstatus is None: history.status = downloadstatus[ 'Status'] + ' ' + downloadstatus[ 'Done'] + ' (ETA ' + downloadstatus['ETA'] + ')' log.info('otrkeyfile {!s} {}'.format( message.otrkeyfile, history.status)) else: """ 3a) add transmission torrent """ if os.path.exists(localtorrentfile): downloaded = True else: downloaded, errormessage = download_fromurl( message.sourcelink, localtorrentfile) if downloaded: log.info( 'downloading torrentfile {!s} successfully initiated!' .format(message.sourcefile)) history.status = 'download started' else: raise Exception(errormessage) except Exception as e: if isinstance(e, subprocess.CalledProcessError): errormessage = 'cmd {!s} failed because {!s}, {!s}'.format( e.cmd, e.stderr, e.stdout) else: errormessage = e log.exception( 'push video failed because {!s}'.format(errormessage)) history.status = 'error' """ delete message after 3 tries """ if (config['APPLICATION_ENVIRONMENT'] == 'Production') and (message.dequeue_count >= 3): queue.delete(message) history.status = 'deleted' """ update history entry """ history.updated = datetime.now() db.insert(history) """ next message """ message = queue.get(PushVideoMessage(), queuehide) """ housekeeping temporary files """ for file in houskeeping: if not file is None: if os.path.exists(file): os.remove(file) """ houskeeping torrent queue """ for torrentsinglestate in housekeepingTransmission: call = 'transmission-remote -t ' + torrentsinglestate['ID'] + ' -r' process = subprocess.run(call, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.debug('{} finished with {}'.format( call, process.stdout.decode(encoding='utf-8'))) for torrentsinglestate in transmissionstatus: """ restart queue entries """ if torrentsinglestate['Status'] == 'Stopped': call = 'transmission-remote -t ' + torrentsinglestate['ID'] + ' -s' process = subprocess.run(call, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.debug('{} finished with {}'.format( call, process.stdout.decode(encoding='utf-8'))) pass