def write(self): """ Make a copy of the stored config and write it to the configured file """ new_config = ConfigObj(encoding="UTF-8") new_config.filename = self._config_file # first copy over everything from the old config, even if it is not # correctly defined to keep from losing data for key, subkeys in self._config.items(): if key not in new_config: new_config[key] = {} for subkey, value in subkeys.items(): new_config[key][subkey] = value # next make sure that everything we expect to have defined is so for key in _CONFIG_DEFINITIONS.keys(): key, definition_type, section, ini_key, default = self._define(key) self.check_setting(key) if section not in new_config: new_config[section] = {} new_config[section][ini_key] = self._config[section][ini_key] # Write it to file logger.info("Config :: Writing configuration to file") try: new_config.write() except IOError as e: logger.error("Config :: Error writing configuration file: %s", e)
def sig_handler(signum=None, frame=None): global SR if signum is not None: logger.info( "sig_handler: Signal %i caught, Shutting Down SpeakReader...", signum) SR.SIGNAL = 'shutdown'
def runHandler(self): if self._STARTED: logger.warn('Transcript Queue Handler already started') return logger.info('Transcript Queue Handler starting') self._STARTED = True while self._STARTED: try: transcript = self._receiverQueue.get(timeout=2) if transcript is None: break except queue.Empty: if self._STARTED: transcript = {"event": "ping"} else: break for queueElement in list(self._listenerQueues.values()): try: queueElement.put_nowait(transcript) except queue.Full: self.removeListener(sessionID=queueElement.sessionID) self._STARTED = False self.closeAllListeners() logger.info('Transcript Queue Handler terminated')
def runHandler(self): if self._STARTED: logger.warn('Sound Meter Queue Handler already started') return logger.info('Sound Meter Queue Handler starting') self._STARTED = True while self._STARTED: try: meterRecord = self._receiverQueue.get(timeout=2) if meterRecord is None: break data = { "event": "meterrecord", "final": True, "record": meterRecord, } except queue.Empty: if self._STARTED: data = {"event": "ping"} else: break for queueElement in list(self._listenerQueues.values()): try: queueElement.put_nowait(data) except queue.Full: self.removeListener(sessionID=queueElement.sessionID) self._STARTED = False self.closeAllListeners() logger.info('Sound Meter Queue Handler terminated')
def checkForUpdate(self, **kwargs): logger.info("Checking for Updates") self.SR.versionInfo.checkForUpdate() versionInfo = { "latest_release": self.SR.versionInfo.LATEST_RELEASE, "latest_release_url": self.SR.versionInfo.LATEST_RELEASE_URL, "update_available": self.SR.versionInfo.UPDATE_AVAILABLE, } return versionInfo
def __init__(self): if self._INITIALIZED: logger.warn('Queue Manager already Initialized') return logger.info('Queue Manager Initializing') self.transcriptHandler = TranscriptHandler("TranscriptQueueHandler") self.logHandler = LogHandler("LogQueueHandler") self.meterHandler = MeterHandler("SoundMeterQueueHandler") self._INITIALIZED = True
def pip_update(self): logger.info("Running pip_update to update the installation tools.") try: cmd = sys.executable + ' -m pip install --upgrade pip setuptools wheel pip-tools' output = self._exec_command(cmd) for line in output: logger.info('pip_update output: %s' % line) return True except Exception as e: logger.error('Command failed: %s' % e) return False
def pip_sync(self): logger.info("Running pip-sync to synchronize the environment.") try: cmd = sys.executable + ' -m piptools sync requirements.txt' output = self._exec_command(cmd) for line in output: logger.info('pip-sync output: %s' % line) return True except Exception as e: logger.error('Command failed: %s' % e) return False
def runHandler(self): if self._STARTED: logger.warn('Log Queue Handler already started') return logger.info('Log Queue Handler starting') self._STARTED = True mainLogger = logging.getLogger("SpeakReader") self.queueHandler = handlers.QueueHandler(self._receiverQueue) self.queueHandler.setFormatter(logger.log_format) self.queueHandler.setLevel(logger.log_level) mainLogger.addHandler(self.queueHandler) for handler in mainLogger.handlers[:]: if isinstance(handler, handlers.RotatingFileHandler): self.fileName = handler.baseFilename break while self._STARTED: try: logRecord = self._receiverQueue.get(timeout=2) if logRecord is None: break # Python 3.6.8 doesn't seem to return a formatted message while 3.7.3 does. logMessage = logRecord.getMessage() formatted_logMessage = self.queueHandler.format(logRecord) logRecord.msg = "" formatted_header = self.queueHandler.format(logRecord) if formatted_header not in logMessage: logMessage = formatted_logMessage data = { "event": "logrecord", "final": True, "record": logMessage, } except queue.Empty: if self._STARTED: data = {"event": "ping"} else: break for queueElement in list(self._listenerQueues.values()): try: queueElement.put_nowait(data) except queue.Full: self.removeListener(sessionID=queueElement.sessionID) self._STARTED = False self.closeAllListeners() logger.info('Log Queue Handler terminated')
def addListener(self, type=None, remoteIP=None, sessionID=None): if not self._STARTED or not remoteIP or not sessionID: return None logger.info("Adding " + type.capitalize() + " Listener Queue for IP: " + remoteIP + " with SessionID: " + sessionID) if type == 'meter': maxsize = 200 else: maxsize = 20 queueElement = QueueElement(type=type, remoteIP=remoteIP, sessionID=sessionID, maxsize=maxsize) self._listenerQueues[sessionID] = queueElement data = { "event": "open", "sessionID": sessionID, } queueElement.put_nowait(data) data = None if type == "log": if self.fileName: with open(self.fileName) as f: records = '<p>' + f.read().replace("\n", "</p><p>") + '</p>' data = { "event": "logrecord", "final": "reload", "record": records, } elif type == "transcript": if self.fileName: with open(self.fileName, 'r') as f: records = "<p>" + f.read().rstrip("\n\n").replace( "\n\n", "</p><p>") + "</p>" data = { "event": "transcript", "final": "reload", "record": records, } if data: queueElement.put_nowait(data) return queueElement.listenerQueue
def startTranscribeEngine(self): if self.transcribeEngine.is_online: logger.info("Transcribe Engine already started.") return if self.get_input_device() is None: logger.warn("No Input Devices Available. Can't start Transcribe Engine.") return if CONFIG.SPEECH_TO_TEXT_SERVICE == 'google': if CONFIG.GOOGLE_CREDENTIALS_FILE == "": logger.warn("API Credentials not available. Can't start Transcribe Engine.") return try: with open(CONFIG.GOOGLE_CREDENTIALS_FILE) as f: json.loads(f.read()) except json.decoder.JSONDecodeError: logger.warn("API Credentials does not appear to be a valid JSON file. Can't start Transcribe Engine.") return elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM': if CONFIG.IBM_CREDENTIALS_FILE == "": logger.warn("API Credentials not available. Can't start Transcribe Engine.") return APIKEY = None URL = None try: with open(CONFIG.IBM_CREDENTIALS_FILE) as f: for line in f.read().splitlines(): parm = line.split('=') if parm[0] == 'SPEECH_TO_TEXT_APIKEY': APIKEY = parm[1] if parm[0] == 'SPEECH_TO_TEXT_URL': URL = parm[1] except: pass if APIKEY is None or URL is None: logger.warn("APIKEY or URL not found in IBM credentials file. Can't start Transcribe Engine.") return elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft': if CONFIG.MICROSOFT_SERVICE_APIKEY == "" or CONFIG.MICROSOFT_SERVICE_REGION == "": logger.warn("Microsoft Azure APIKEY and Region are required. Can't start Transcribe Engine.") return else: return self.transcribeEngine.start()
def daemonize(myPidFile): if threading.activeCount() != 1: logger.warn( "There are %r active threads. Daemonizing may cause" " strange behavior.", threading.enumerate()) sys.stdout.flush() sys.stderr.flush() # Do first fork try: pid = os.fork() # @UndefinedVariable - only available in UNIX if pid != 0: sys.exit(0) except OSError as e: raise RuntimeError("1st fork failed: %s [%d]", e.strerror, e.errno) os.setsid() # Make sure I can read my own files and shut out others prev = os.umask(0) # @UndefinedVariable - only available in UNIX os.umask(prev and int('077', 8)) # Make the child a session-leader by detaching from the terminal try: pid = os.fork() # @UndefinedVariable - only available in UNIX if pid != 0: sys.exit(0) except OSError as e: raise RuntimeError("2nd fork failed: %s [%d]", e.strerror, e.errno) dev_null = open('/dev/null', 'r') os.dup2(dev_null.fileno(), sys.stdin.fileno()) si = open('/dev/null', "r") so = open('/dev/null', "a+") se = open('/dev/null', "a+") os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) pid = os.getpid() myPidFile.seek(0) myPidFile.write(str(pid)) myPidFile.flush() logger.info("Daemonized to PID: %d", pid)
def signin(self, username=None, password=None, remember_me='0', *args, **kwargs): if cherrypy.request.method != 'POST': cherrypy.response.status = 405 return {'status': 'error', 'message': 'Sign in using POST.'} error_message = {'status': 'error', 'message': 'Invalid credentials.'} valid_login = check_credentials(username=username, password=password) if valid_login: time_delta = timedelta( days=30) if remember_me == '1' else timedelta(minutes=60) expiry = datetime.utcnow() + time_delta payload = {'user': username, 'exp': expiry} jwt_token = jwt.encode(payload, speakreader.CONFIG.JWT_SECRET, algorithm=JWT_ALGORITHM).decode('utf-8') self.on_login( username=username, success=True, ) jwt_cookie = JWT_COOKIE_NAME cherrypy.response.cookie[jwt_cookie] = jwt_token cherrypy.response.cookie[jwt_cookie]['expires'] = int( time_delta.total_seconds()) cherrypy.response.cookie[jwt_cookie]['path'] = '/' cherrypy.request.login = payload cherrypy.response.status = 200 return {'status': 'success'} else: self.on_login(username=username) logger.info("WebAuth :: Invalid admin login attempt from '%s'." % username) cherrypy.response.status = 401 return error_message
def removeListener(self, sessionID=None, listenerQueue=None): queueElement = None if sessionID is not None: queueElement = self._listenerQueues[sessionID] elif listenerQueue is not None: for sessionID, queueElement in self._listenerQueues.items(): if queueElement.listenerQueue == listenerQueue: break if queueElement is not None: logger.info("Removing " + queueElement.type.capitalize() + " Listener Queue for IP: " + queueElement.remoteIP + " with SessionID: " + sessionID) queueElement.clear() queueElement.put_nowait(None) self._listenerQueues.pop(sessionID)
def checkout_git_branch(self): if self.INSTALL_TYPE == 'git': output, err = self.runGit('fetch %s' % speakreader.CONFIG.GIT_REMOTE) output, err = self.runGit('checkout %s' % speakreader.CONFIG.GIT_BRANCH) if not output: logger.error('Unable to change git branch.') return for line in output.split('\n'): if line.endswith(('Aborting', 'Aborting.')): logger.error('Unable to checkout from git: ' + line) logger.info('Output: ' + str(output)) output, err = self.runGit( 'pull %s %s' % (speakreader.CONFIG.GIT_REMOTE, speakreader.CONFIG.GIT_BRANCH)) self.pip_update() self.pip_sync()
def __init__(self): if TranscribeEngine._INITIALIZED: logger.warn("Transcribe Engine already Initialized") return logger.info("Transcribe Engine Initializing") ################################################################################################### # Set Supported Platforms ################################################################################################### self.GOOGLE_SERVICE = GOOGLE_SERVICE self.IBM_SERVICE = IBM_SERVICE self.MICROSOFT_SERVICE = MICROSOFT_SERVICE ################################################################################################### # Initialize the Queue Manager ################################################################################################### self.queueManager = QueueManager() self.transcriptQueue = self.queueManager.transcriptHandler.getReceiverQueue( ) TranscribeEngine._INITIALIZED = True
def run(self): if self._ONLINE: logger.warn("Transcribe Engine already Started") return logger.info( "Transcribe Engine Starting with the %s%s Speech-To-Text Service" % (speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE[0].upper(), speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE[1:])) FILENAME_DATESTRING = datetime.datetime.now().strftime( FILENAME_DATE_FORMAT) TRANSCRIPT_FILENAME = FILENAME_PREFIX + FILENAME_DATESTRING + "." + TRANSCRIPT_FILENAME_SUFFIX RECORDING_FILENAME = FILENAME_PREFIX + FILENAME_DATESTRING + "." + RECORDING_FILENAME_SUFFIX tf = os.path.join(speakreader.CONFIG.TRANSCRIPTS_FOLDER, TRANSCRIPT_FILENAME) self.queueManager.transcriptHandler.setFileName(tf) try: self.microphoneStream = MicrophoneStream( speakreader.CONFIG.INPUT_DEVICE) self.microphoneStream.recordingFilename = RECORDING_FILENAME self.microphoneStream.meterQueue = self.queueManager.meterHandler.getReceiverQueue( ) except Exception as e: logger.debug("MicrophoneStream Exception: %s" % e) self.transcriptQueue.put_nowait(self.OFFLINE_MESSAGE) return if speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'google' and self.GOOGLE_SERVICE: transcribeService = googleTranscribe(self.microphoneStream) elif speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM' and self.IBM_SERVICE: transcribeService = ibmTranscribe(self.microphoneStream) elif speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft' and self.MICROSOFT_SERVICE: transcribeService = microsoftTranscribe(self.microphoneStream) else: logger.warn( "No Supported Transcribe Service Selected. Can't start Transcribe Engine." ) return self.transcriptFile = open(tf, "a+") self.transcriptQueue.put_nowait(self.ONLINE_MESSAGE) self._ONLINE = True try: with self.microphoneStream as stream: while self._ONLINE: responses = transcribeService.transcribe() self.process_responses(responses) logger.info("Transcription Engine Stream Closed") except Exception as e: logger.error(e) self.transcriptFile.close() self.transcriptQueue.put_nowait(self.OFFLINE_MESSAGE) self._ONLINE = False logger.info("Transcribe Engine Terminated")
def cleanup_files(self): logger.info("Running File Cleanup") def delete(path, days): try: days = int(days) except ValueError: return delete_date = datetime.datetime.now() - datetime.timedelta(days=days) with os.scandir(path=path) as files: for file in files: file_info = file.stat() if datetime.datetime.fromtimestamp(file_info.st_ctime) < delete_date: filename = os.path.join(path, file.name) logger.debug("Deleting: %s" % filename) os.remove(filename) if CONFIG.LOG_RETENTION_DAYS != "": delete(CONFIG.LOG_DIR, CONFIG.LOG_RETENTION_DAYS) if CONFIG.TRANSCRIPT_RETENTION_DAYS != "": delete(CONFIG.TRANSCRIPTS_FOLDER, CONFIG.TRANSCRIPT_RETENTION_DAYS) if CONFIG.RECORDING_RETENTION_DAYS != "": delete(CONFIG.RECORDINGS_FOLDER, CONFIG.RECORDING_RETENTION_DAYS)
def shutdown(self): self._INITIALIZED = False self.transcriptHandler.shutdown() self.logHandler.shutdown() self.meterHandler.shutdown() logger.info("Queue Manager terminated")
def __init__(self, initOptions): if SpeakReader._INITIALIZED: return with INIT_LOCK: global PROG_DIR PROG_DIR = initOptions['prog_dir'] global DATA_DIR DATA_DIR = initOptions['data_dir'] global CONFIG CONFIG = initOptions['config'] assert CONFIG is not None if isinstance(initOptions['http_port'], int): self.HTTP_PORT = initOptions['http_port'] else: self.HTTP_PORT = int(CONFIG.HTTP_PORT) if self.HTTP_PORT < 21 or self.HTTP_PORT > 65535: logger.warn("HTTP_PORT out of bounds: 21 < %s < 65535", self.HTTP_PORT) self.HTTP_PORT = 8880 # Check if pyOpenSSL is installed. It is required for certificate generation # and for CherryPy. if CONFIG.ENABLE_HTTPS: try: import OpenSSL except ImportError: logger.warn("The pyOpenSSL module is missing. Install this " "module to enable HTTPS. HTTPS will be disabled.") CONFIG.ENABLE_HTTPS = False if not CONFIG.HTTPS_CERT: CONFIG.HTTPS_CERT = os.path.join(DATA_DIR, 'server.crt') if not CONFIG.HTTPS_KEY: CONFIG.HTTPS_KEY = os.path.join(DATA_DIR, 'server.key') if not (os.path.exists(CONFIG.HTTPS_CERT) and os.path.exists(CONFIG.HTTPS_KEY)): logger.warn("Disabled HTTPS because of missing certificate and key.") CONFIG.ENABLE_HTTPS = False # Check if we has a jwt_secret if CONFIG.JWT_SECRET == '' or not CONFIG.JWT_SECRET: logger.debug("Generating JWT secret...") CONFIG.JWT_SECRET = generate_uuid() CONFIG.write() ################################################################################################### # Get Version Information and check for updates ################################################################################################### self.versionInfo = Version() ################################################################################################### # Get the Input Device ################################################################################################### self.get_input_device() ################################################################################################### # Initialize the Transcribe Engine ################################################################################################### self.transcribeEngine = TranscribeEngine() if CONFIG.START_TRANSCRIBE_ON_STARTUP : self.startTranscribeEngine() ################################################################################################### # Initialize the webserver ################################################################################################### logger.info('WebServer Initializing') webServerOptions = { 'config': CONFIG, 'prog_dir': PROG_DIR, 'data_dir': DATA_DIR, 'http_port': self.HTTP_PORT, } self.webServer = webstart.initialize(webServerOptions) self.webServer.root.SR = self cherrypy.server.start() # Launch the WebBrowser if CONFIG.LAUNCH_BROWSER and not initOptions['nolaunch']: launch_browser(CONFIG.HTTP_HOST, self.HTTP_PORT, CONFIG.HTTP_ROOT + 'manage') ################################################################################################### # Run cleanup of old logs, transcripts, and recordings and start a scheduler to run every 24 hours ################################################################################################### self.cleanup_files() self.scheduler = BackgroundScheduler() self.scheduler.add_job(self.cleanup_files, 'interval', hours=24) self.scheduler.start() SpeakReader._INITIALIZED = True
def _check_github(self): self.COMMITS_BEHIND = 0 # Get the latest version available from github logger.debug('Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/%s/%s/commits/%s' % ( speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO, speakreader.CONFIG.GIT_BRANCH) if speakreader.CONFIG.GIT_TOKEN: url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN try: response = requests.get(url, timeout=10) except Exception as e: logger.warn('Failed to establish a connection to GitHub') return if response.ok: version = response.json() else: logger.warn( 'Could not get the latest version information from GitHub for ' + speakreader.CONFIG.GIT_REMOTE + '/' + speakreader.CONFIG.GIT_BRANCH + '. Are you running a local development version?') return self.LATEST_VERSION_HASH = version['sha'] # See how many commits behind we are if not self.INSTALLED_VERSION_HASH: logger.info( 'You are running an unknown version of SpeakReader. Run the updater to identify your version' ) self.LATEST_RELEASE = "Unknown" return # Get latest release tag logger.debug('Retrieving latest release information from GitHub') url = 'https://api.github.com/repos/%s/%s/releases' % ( speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO) if speakreader.CONFIG.GIT_TOKEN: url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN try: response = requests.get(url, timeout=10) except Exception as e: logger.warn('Failed to establish a connection to GitHub') return if response.ok: releases = response.json() else: logger.warn('Could not get releases from GitHub.') return if speakreader.CONFIG.GIT_BRANCH == 'master': release = next((r for r in releases if not r['prerelease']), releases[0]) elif speakreader.CONFIG.GIT_BRANCH == 'beta': release = next((r for r in releases), releases[0]) else: release = releases[0] self.LATEST_RELEASE = release['tag_name'] url = 'https://github.com/%s/%s/releases/tag/%s' \ % (speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO, self.LATEST_RELEASE) if speakreader.CONFIG.GIT_TOKEN: url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN self.LATEST_RELEASE_URL = url logger.info("Installed release is %s - %s" % (self.INSTALLED_RELEASE, self.INSTALLED_VERSION_HASH)) logger.info(" Latest release is %s - %s" % (self.LATEST_RELEASE, self.LATEST_VERSION_HASH)) if self.LATEST_VERSION_HASH == self.INSTALLED_VERSION_HASH: logger.info('SpeakReader is up to date') return logger.debug( 'Comparing currently installed version with latest GitHub version') url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % ( speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO, self.LATEST_VERSION_HASH, self.INSTALLED_VERSION_HASH) if speakreader.CONFIG.GIT_TOKEN: url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN try: response = requests.get(url, timeout=10) except Exception as e: logger.warn('Failed to establish a connection to GitHub') return if response.ok: commits = response.json() else: logger.warn('Could not get commits behind from GitHub.') return try: self.COMMITS_BEHIND = int(commits['behind_by']) logger.debug("In total, %d commits behind", self.COMMITS_BEHIND) except KeyError: logger.info( 'Cannot compare versions. Are you running a local development version?' ) self.COMMITS_BEHIND = 0 if self.COMMITS_BEHIND > 0: logger.info('Updates Available. You are %s commits behind' % self.COMMITS_BEHIND) elif self.COMMITS_BEHIND == 0: logger.info('SpeakReader is up to date')
def update(self): if speakreader.CONFIG.SERVER_ENVIRONMENT.lower() != 'production': logger.info( "Updating bypassed because this is not a production environment" ) return False if not self.UPDATE_AVAILABLE: logger.info("No Updates Available") return False if self.INSTALL_TYPE == 'git': output, err = self.runGit( 'diff --name-only %s/%s' % (speakreader.CONFIG.GIT_REMOTE, speakreader.CONFIG.GIT_BRANCH)) if output == '': logger.debug("No differences found from the origin") elif output == 'requirements.txt': logger.warn( 'Requirements file is out of sync. Restoring to original.') output, err = self.runGit('checkout %s/%s requirements.txt' % (speakreader.CONFIG.GIT_REMOTE, speakreader.CONFIG.GIT_BRANCH)) else: logger.error("Differences Found. Unable to update.") logger.info('Output: ' + str(output)) return False output, err = self.runGit('pull ' + speakreader.CONFIG.GIT_REMOTE + ' ' + speakreader.CONFIG.GIT_BRANCH) if not output: logger.error('Unable to download latest version') return False for line in output.split('\n'): if 'Already up-to-date.' in line: logger.info('No update available, not updating') logger.info('Output: ' + str(output)) return False elif line.endswith(('Aborting', 'Aborting.')): logger.error('Unable to update from git: ' + line) logger.info('Output: ' + str(output)) return False else: tar_download_url = 'https://api.github.com/repos/{}/{}/tarball/{}'.format( speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO, speakreader.CONFIG.GIT_BRANCH) if speakreader.CONFIG.GIT_TOKEN: tar_download_url = tar_download_url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN update_dir = os.path.join(speakreader.PROG_DIR, 'update') version_path = os.path.join(speakreader.PROG_DIR, 'version.txt') logger.info('Downloading update from: ' + tar_download_url) try: data = requests.get(tar_download_url, timeout=10) except Exception as e: logger.warn('Failed to establish a connection to GitHub') return False if not data: logger.error( "Unable to retrieve new version from '%s', can't update", tar_download_url) return False download_name = speakreader.CONFIG.GIT_BRANCH + '-github' tar_download_path = os.path.join(speakreader.PROG_DIR, download_name) # Save tar to disk with open(tar_download_path, 'wb') as f: f.write(data.content) # Extract the tar to update folder logger.info('Extracting file: ' + tar_download_path) tar = tarfile.open(tar_download_path) tar.extractall(update_dir) tar.close() # Delete the tar.gz logger.info('Deleting file: ' + tar_download_path) os.remove(tar_download_path) # Find update dir name update_dir_contents = [ x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x)) ] if len(update_dir_contents) != 1: logger.error("Invalid update data, update failed: " + str(update_dir_contents)) return False content_dir = os.path.join(update_dir, update_dir_contents[0]) # walk temp folder and move files to main folder for dirname, dirnames, filenames in os.walk(content_dir): dirname = dirname[len(content_dir) + 1:] for curfile in filenames: old_path = os.path.join(content_dir, dirname, curfile) new_path = os.path.join(speakreader.PROG_DIR, dirname, curfile) if os.path.isfile(new_path): os.remove(new_path) os.renames(old_path, new_path) # Update version.txt try: with open(version_path, 'w') as f: f.write(str(self.LATEST_VERSION_HASH)) except IOError as e: logger.error( "Unable to write current version to version.txt, update not complete: %s" % e) return False self.pip_update() self.pip_sync() logger.info("Update Complete") return True
def on_login(self, username=None, success=False): """Called on successful login""" if success: logger.info("WebAuth :: Admin user '%s' logged into SpeakReader." % (username))
def on_logout(self, username): """Called on logout""" logger.info("WebAuth :: Admin user '%s' logged out of SpeakReader." % (username))
def update(self, **kwargs): if self.SR.versionInfo.UPDATE_AVAILABLE: return self.do_state_change('update', 'Updating', 180) else: logger.info("No Updates Available") return self.manage()
def configUpdate(self, **kwargs): logger.info("Processing configUpdate.") restartTranscribeEngine = False restartSpeakReader = False logout = False cleanup = False checked_configs = [ "start_transcribe_on_startup", "launch_browser", "enable_https", "save_recordings", "show_interim_results", "enable_censorship", "http_hash_password", "http_basic_auth", "check_github", ] for checked_config in checked_configs: if checked_config not in kwargs: # checked items should be zero or one. if they were not sent then the item was not checked kwargs[checked_config] = 0 else: kwargs[checked_config] = 1 upload_google_credentials_file = kwargs.pop( 'upload_google_credentials_file', None) upload_ibm_credentials_file = kwargs.pop('upload_ibm_credentials_file', None) if upload_google_credentials_file.file is not None: self.upload_credentials_file(upload_google_credentials_file) kwargs['google_credentials_file'] = os.path.join( speakreader.DATA_DIR, upload_google_credentials_file.filename) if upload_ibm_credentials_file.file is not None: self.upload_credentials_file(upload_ibm_credentials_file) kwargs['ibm_credentials_file'] = os.path.join( speakreader.DATA_DIR, upload_ibm_credentials_file.filename) if kwargs.get('speech_to_text_service') != speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE \ or kwargs.get('google_credentials_file') != speakreader.CONFIG.GOOGLE_CREDENTIALS_FILE \ or kwargs.get('ibm_credentials_file') != speakreader.CONFIG.IBM_CREDENTIALS_FILE \ or kwargs.get('microsoft_service_apikey') != speakreader.CONFIG.MICROSOFT_SERVICE_APIKEY \ or kwargs.get('microsoft_service_region') != speakreader.CONFIG.MICROSOFT_SERVICE_REGION \ or kwargs.get('enable_censorship') != speakreader.CONFIG.ENABLE_CENSORSHIP \ or kwargs.get('input_device') != speakreader.CONFIG.INPUT_DEVICE \ or kwargs.get('save_recordings') != speakreader.CONFIG.SAVE_RECORDINGS: restartTranscribeEngine = True if kwargs.get('http_port') != str(speakreader.CONFIG.HTTP_PORT) \ or kwargs.get('enable_https') != speakreader.CONFIG.ENABLE_HTTPS \ or kwargs.get('https_cert') != speakreader.CONFIG.HTTPS_CERT \ or kwargs.get('https_cert_chain') != speakreader.CONFIG.HTTPS_CERT_CHAIN \ or kwargs.get('https_key') != speakreader.CONFIG.HTTPS_KEY \ or kwargs.get('transcripts_folder') != speakreader.CONFIG.TRANSCRIPTS_FOLDER \ or kwargs.get('recordings_folder') != speakreader.CONFIG.RECORDINGS_FOLDER \ or kwargs.get('log_dir') != speakreader.CONFIG.LOG_DIR: restartSpeakReader = True kwargs['censored_words'] = kwargs['censored_words'].rstrip( '\r\n').replace('\r\n', ',').split(',') while ("" in kwargs['censored_words']): kwargs['censored_words'].remove("") set_http_password = int(kwargs.pop('set_http_password', 0)) if kwargs.get('http_username') == "": kwargs['http_password'] = "" else: if set_http_password: if kwargs.get('http_hash_password'): kwargs['http_password'] = pbkdf2_sha256.hash( kwargs['http_password']) else: kwargs.pop('http_password', 0) if kwargs.get('http_hash_password' ) != speakreader.CONFIG.HTTP_HASH_PASSWORD: if kwargs.get('http_hash_password' ) and speakreader.CONFIG.HTTP_PASSWORD != "": kwargs['http_password'] = pbkdf2_sha256.hash( speakreader.CONFIG.HTTP_PASSWORD) if kwargs[ 'http_username'] != speakreader.CONFIG.HTTP_USERNAME or set_http_password: logout = True if kwargs.get('log_retention_days') != speakreader.CONFIG.LOG_RETENTION_DAYS \ or kwargs.get('transcript_retention_days') != speakreader.CONFIG.TRANSCRIPT_RETENTION_DAYS \ or kwargs.get('recording_retention_days') != speakreader.CONFIG.RECORDING_RETENTION_DAYS: cleanup = True speakreader.CONFIG.process_kwargs(kwargs) speakreader.CONFIG.write() if cleanup: self.SR.cleanup_files() if restartSpeakReader: #self.restart() return {'portchanged': True} if restartTranscribeEngine and self.SR.transcribeEngine.is_online: self.SR.stopTranscribeEngine() self.SR.startTranscribeEngine() return { 'result': 'success', 'google_credentials_file': kwargs['google_credentials_file'], 'ibm_credentials_file': kwargs['ibm_credentials_file'], 'logout': logout, 'portchanged': False, }
def main(): """ SpeakReader application entry point. Parses arguments, setups encoding and initializes the application. """ DAEMON = False NOFORK = False QUIET = False VERBOSE = False NOLAUNCH = False HTTP_PORT = None PLATFORM = system() PLATFORM_RELEASE = release() PLATFORM_VERSION = version() PLATFORM_LINUX_DISTRO = None PLATFORM_PROCESSOR = processor() PLATFORM_MACHINE = machine() PLATFORM_IS_64BITS = sys.maxsize > 2**32 SYS_PLATFORM = sys.platform # Fixed paths to application if hasattr(sys, 'frozen'): FULL_PATH = os.path.abspath(sys.executable) else: FULL_PATH = os.path.abspath(__file__) PROG_DIR = os.path.dirname(FULL_PATH) # Ensure only one instance of SpeakReader running. PIDFILE = os.path.join(PROG_DIR, 'pidfile') myPid = os.getpid() if os.path.exists(PIDFILE) and os.path.isfile(PIDFILE): for p in psutil.process_iter(): if 'python' in p.name() and p.pid != myPid: for f in p.open_files(): if f.path == PIDFILE: logger.error( "SpeakReader is already Running. Exiting.") sys.exit(0) myPidFile = open(PIDFILE, 'w+') myPidFile.write(str(myPid)) myPidFile.flush() try: locale.setlocale(locale.LC_ALL, "") except (locale.Error, IOError): pass try: SYS_TIMEZONE = str(tzlocal.get_localzone()) SYS_UTC_OFFSET = datetime.datetime.now( pytz.timezone(SYS_TIMEZONE)).strftime('%z') except (pytz.UnknownTimeZoneError, LookupError, ValueError) as e: logger.error("Could not determine system timezone: %s" % e) SYS_TIMEZONE = 'Unknown' SYS_UTC_OFFSET = '+0000' # Parse any passed startup arguments ARGS = sys.argv[1:] # Set up and gather command line arguments parser = argparse.ArgumentParser( description= 'A Python based monitoring and tracking tool for Plex Media Server.') parser.add_argument('-v', '--verbose', action='store_true', help='Increase console logging verbosity') parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging') parser.add_argument('-d', '--daemon', action='store_true', help='Run as a daemon') parser.add_argument('-p', '--port', type=int, help='Force SpeakReader to run on a specified port') parser.add_argument( '--datadir', help='Specify a directory where to store your data files') parser.add_argument('--config', help='Specify a config file to use') parser.add_argument('--nolaunch', action='store_true', help='Prevent browser from launching on startup') parser.add_argument( '--nofork', action='store_true', help='Start SpeakReader as a service, do not fork when restarting') args = parser.parse_args() # Force the http port if necessary if args.port: HTTP_PORT = args.port logger.info('Using forced web server port: %i', HTTP_PORT) # Don't launch the browser if args.nolaunch: NOLAUNCH = True if args.verbose: VERBOSE = True if args.quiet: QUIET = True if args.daemon: if SYS_PLATFORM == 'win32': sys.stderr.write( "Daemonizing not supported under Windows, starting normally\n") else: DAEMON = True QUIET = True if args.nofork: NOFORK = True logger.info( "SpeakReader is running as a service, it will not fork when restarted." ) # Determine which data directory and config file to use if args.datadir: DATA_DIR = args.datadir else: DATA_DIR = os.path.join(PROG_DIR, 'data') if args.config: CONFIG_FILE = args.config else: CONFIG_FILE = os.path.join(DATA_DIR, config.FILENAME) # Try to create the DATA_DIR if it doesn't exist if not os.path.exists(DATA_DIR): try: os.makedirs(DATA_DIR) except OSError: raise SystemExit('Could not create data directory: ' + DATA_DIR + '. Exiting....') # Make sure the DATA_DIR is writeable if not os.access(DATA_DIR, os.W_OK): raise SystemExit('Cannot write to the data directory: ' + DATA_DIR + '. Exiting...') # Do an initial setup of the logging. logger.initLogger(console=not QUIET, log_dir=False, verbose=VERBOSE) # Initialize the configuration from the config file CONFIG = config.Config(CONFIG_FILE) assert CONFIG is not None if CONFIG.SERVER_ENVIRONMENT.lower() != 'production': VERBOSE = True CONFIG.LOG_DIR, log_writable = check_folder_writable( CONFIG.LOG_DIR, os.path.join(DATA_DIR, 'logs'), 'logs') if not log_writable and not QUIET: sys.stderr.write( "Unable to create the log directory. Logging to screen only.\n") # Start the logger, disable console if needed logger.initLogger(console=not QUIET, log_dir=CONFIG.LOG_DIR if log_writable else None, verbose=VERBOSE) logger.info("Initializing {} {}".format(speakreader.PRODUCT, speakreader.VERSION_RELEASE)) logger.info("{} {} ({}{})".format( PLATFORM, PLATFORM_RELEASE, PLATFORM_VERSION, ' - {}'.format(PLATFORM_LINUX_DISTRO) if PLATFORM_LINUX_DISTRO else '')) logger.info("{} ({} {})".format( PLATFORM_PROCESSOR, PLATFORM_MACHINE, '{}'.format('64-BIT') if PLATFORM_IS_64BITS else '32-BIT')) logger.info("Python {}".format(sys.version)) logger.info("{} (UTC{})".format(SYS_TIMEZONE, SYS_UTC_OFFSET)) logger.info("Program Dir: {}".format(PROG_DIR)) logger.info("Config File: {}".format(CONFIG_FILE)) CONFIG.TRANSCRIPTS_FOLDER, _ = check_folder_writable( CONFIG.TRANSCRIPTS_FOLDER, os.path.join(DATA_DIR, 'transcripts'), 'transcripts') CONFIG.RECORDINGS_FOLDER, _ = check_folder_writable( CONFIG.RECORDINGS_FOLDER, os.path.join(DATA_DIR, 'recordings'), 'recordings') if DAEMON: daemonize(myPidFile) # Store the original umask UMASK = os.umask(0) os.umask(UMASK) initOptions = { 'config': CONFIG, 'http_port': HTTP_PORT, 'nolaunch': NOLAUNCH, 'prog_dir': PROG_DIR, 'data_dir': DATA_DIR, } # Read config and start logging global SR SR = SpeakReader(initOptions) # Wait endlessly for a signal to happen restart = False checkout = False update = False while True: if not SR.SIGNAL: try: time.sleep(1) except KeyboardInterrupt: SR.SIGNAL = 'shutdown' else: logger.info('Received signal: %s', SR.SIGNAL) if SR.SIGNAL == 'shutdown': break elif SR.SIGNAL == 'restart': restart = True break elif SR.SIGNAL == 'update': restart = True update = True break elif SR.SIGNAL == 'checkout': restart = True checkout = True break else: SR.SIGNAL = None SR.shutdown(restart=restart, update=update, checkout=checkout) myPidFile.close() os.remove(PIDFILE) if restart: logger.info("SpeakReader is restarting...") exe = sys.executable args = [exe, FULL_PATH] args += ARGS if '--nolaunch' not in args: args += ['--nolaunch'] # Separate out logger so we can shutdown logger after if NOFORK: logger.info('Running as service, not forking. Exiting...') elif os.name == 'nt': logger.info('Restarting SpeakReader with %s', args) else: logger.info('Restarting SpeakReader with %s', args) logger.shutdown() if NOFORK: pass elif os.name == 'nt': subprocess.Popen(args, cwd=os.getcwd()) else: os.execv(exe, args) else: logger.info("SpeakReader Terminated") logger.shutdown() sys.exit(0)
def shutdown(self, restart=False, update=False, checkout=False): SpeakReader._INITIALIZED = False self.transcribeEngine.shutdown() self.scheduler.shutdown() CONFIG.write() if not restart and not update and not checkout: logger.info("Shutting Down SpeakReader") if update: logger.info("********************************") logger.info("* SpeakReader is updating... *") logger.info("********************************") try: self.versionInfo.update() except Exception as e: logger.warn("SpeakReader failed to update: %s. Restarting." % e) logger.info('WebServer Terminating') cherrypy.engine.exit() if checkout: logger.info("SpeakReader is switching the git branch...") try: self.versionInfo.checkout_git_branch() except Exception as e: logger.warn("SpeakReader failed to switch git branch: %s. Restarting." % e)
def initialize(options): from speakreader import webauth from speakreader.webserve import WebInterface CONFIG = options['config'] options_dict = { 'server.socket_port': options['http_port'], 'server.socket_host': CONFIG.HTTP_HOST, 'environment': 'production', 'server.thread_pool': 50, 'tools.encode.on': True, 'tools.encode.encoding': 'utf-8', 'tools.decode.on': True } if CONFIG.ENABLE_HTTPS: options_dict['server.ssl_certificate'] = CONFIG.HTTPS_CERT options_dict['server.ssl_certificate_chain'] = CONFIG.HTTPS_CERT_CHAIN options_dict['server.ssl_private_key'] = CONFIG.HTTPS_KEY protocol = "https" else: protocol = "http" if CONFIG.HTTP_PROXY: # Overwrite cherrypy.tools.proxy with our own proxy handler cherrypy.tools.proxy = cherrypy.Tool('before_handler', proxy, priority=1) if CONFIG.HTTP_PASSWORD: login_allowed = [ "SpeakReader admin (username is '%s')" % CONFIG.HTTP_USERNAME ] logger.info("WebStart :: Web server authentication is enabled: %s.", ' and '.join(login_allowed)) if CONFIG.HTTP_BASIC_AUTH: auth_enabled = False basic_auth_enabled = True else: auth_enabled = True basic_auth_enabled = False cherrypy.tools.auth = cherrypy.Tool('before_handler', webauth.check_auth, priority=2) else: auth_enabled = basic_auth_enabled = False cherrypy.config.update(options_dict) conf = { '/': { 'tools.staticdir.root': os.path.join(options['prog_dir'], 'html'), 'tools.proxy.on': bool(CONFIG.HTTP_PROXY), 'tools.gzip.on': True, 'tools.gzip.mime_types': [ 'text/html', 'text/plain', 'text/css', 'text/javascript', 'application/json', 'application/javascript' ], 'tools.auth.on': False, 'tools.auth_basic.on': False, 'tools.sessions.on': True, }, '/manage': { 'tools.staticdir.root': os.path.join(options['prog_dir'], 'html'), 'tools.proxy.on': bool(CONFIG.HTTP_PROXY), 'tools.gzip.on': True, 'tools.gzip.mime_types': [ 'text/html', 'text/plain', 'text/css', 'text/javascript', 'application/json', 'application/javascript' ], 'tools.auth.on': auth_enabled, 'tools.auth_basic.on': basic_auth_enabled, 'tools.auth_basic.realm': 'SpeakReader web server', 'tools.auth_basic.checkpassword': cherrypy.lib.auth_basic.checkpassword_dict( {CONFIG.HTTP_USERNAME: CONFIG.HTTP_PASSWORD}), 'tools.sessions.on': True, }, '/images': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "images", 'tools.staticdir.content_types': { 'svg': 'image/svg+xml' }, 'tools.caching.on': True, 'tools.caching.force': True, 'tools.caching.delay': 0, 'tools.expires.on': True, 'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days 'tools.sessions.on': False, 'tools.auth.on': False }, '/css': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "css", 'tools.caching.on': True, 'tools.caching.force': True, 'tools.caching.delay': 0, 'tools.expires.on': True, 'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days 'tools.sessions.on': False, 'tools.auth.on': False }, '/fonts': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "fonts", 'tools.caching.on': True, 'tools.caching.force': True, 'tools.caching.delay': 0, 'tools.expires.on': True, 'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days 'tools.sessions.on': False, 'tools.auth.on': False }, '/js': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "js", 'tools.caching.on': True, 'tools.caching.force': True, 'tools.caching.delay': 0, 'tools.expires.on': True, 'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days 'tools.sessions.on': False, 'tools.auth.on': False }, '/favicon.ico': { 'tools.staticfile.on': True, 'tools.staticfile.filename': os.path.abspath( os.path.join(options['prog_dir'], '/html/images/favicon/favicon.ico')), 'tools.caching.on': True, 'tools.caching.force': True, 'tools.caching.delay': 0, 'tools.expires.on': True, 'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days 'tools.sessions.on': False, 'tools.auth.on': False } } # Prevent time-outs appTree = cherrypy.tree.mount(WebInterface(), CONFIG.HTTP_ROOT, config=conf) if CONFIG.HTTP_ROOT != '/': cherrypy.tree.mount(BaseRedirect(), '/') try: logger.info("WebStart :: Starting Web Server on %s://%s:%d%s", protocol, CONFIG.HTTP_HOST, options['http_port'], CONFIG.HTTP_ROOT) portend.free(str(CONFIG.HTTP_HOST), options['http_port']) except IOError: sys.stderr.write( 'Failed to start on port: %i. Is something else running?\n' % (options['http_port'])) sys.exit(1) # return appTree