def write(self):
""" Make a copy of the stored config and write it to the configured file """
new_config = ConfigObj(encoding="UTF-8")
new_config.filename = self._config_file
# first copy over everything from the old config, even if it is not
# correctly defined to keep from losing data
for key, subkeys in self._config.items():
if key not in new_config:
new_config[key] = {}
for subkey, value in subkeys.items():
new_config[key][subkey] = value
# next make sure that everything we expect to have defined is so
for key in _CONFIG_DEFINITIONS.keys():
key, definition_type, section, ini_key, default = self._define(key)
self.check_setting(key)
if section not in new_config:
new_config[section] = {}
new_config[section][ini_key] = self._config[section][ini_key]
# Write it to file
logger.info("Config :: Writing configuration to file")
try:
new_config.write()
except IOError as e:
logger.error("Config :: Error writing configuration file: %s", e)
def sig_handler(signum=None, frame=None):
global SR
if signum is not None:
logger.info(
"sig_handler: Signal %i caught, Shutting Down SpeakReader...",
signum)
SR.SIGNAL = 'shutdown'
def runHandler(self):
if self._STARTED:
logger.warn('Transcript Queue Handler already started')
return
logger.info('Transcript Queue Handler starting')
self._STARTED = True
while self._STARTED:
try:
transcript = self._receiverQueue.get(timeout=2)
if transcript is None:
break
except queue.Empty:
if self._STARTED:
transcript = {"event": "ping"}
else:
break
for queueElement in list(self._listenerQueues.values()):
try:
queueElement.put_nowait(transcript)
except queue.Full:
self.removeListener(sessionID=queueElement.sessionID)
self._STARTED = False
self.closeAllListeners()
logger.info('Transcript Queue Handler terminated')
def runHandler(self):
if self._STARTED:
logger.warn('Sound Meter Queue Handler already started')
return
logger.info('Sound Meter Queue Handler starting')
self._STARTED = True
while self._STARTED:
try:
meterRecord = self._receiverQueue.get(timeout=2)
if meterRecord is None:
break
data = {
"event": "meterrecord",
"final": True,
"record": meterRecord,
}
except queue.Empty:
if self._STARTED:
data = {"event": "ping"}
else:
break
for queueElement in list(self._listenerQueues.values()):
try:
queueElement.put_nowait(data)
except queue.Full:
self.removeListener(sessionID=queueElement.sessionID)
self._STARTED = False
self.closeAllListeners()
logger.info('Sound Meter Queue Handler terminated')
def checkForUpdate(self, **kwargs):
logger.info("Checking for Updates")
self.SR.versionInfo.checkForUpdate()
versionInfo = {
"latest_release": self.SR.versionInfo.LATEST_RELEASE,
"latest_release_url": self.SR.versionInfo.LATEST_RELEASE_URL,
"update_available": self.SR.versionInfo.UPDATE_AVAILABLE,
}
return versionInfo
def __init__(self):
if self._INITIALIZED:
logger.warn('Queue Manager already Initialized')
return
logger.info('Queue Manager Initializing')
self.transcriptHandler = TranscriptHandler("TranscriptQueueHandler")
self.logHandler = LogHandler("LogQueueHandler")
self.meterHandler = MeterHandler("SoundMeterQueueHandler")
self._INITIALIZED = True
def pip_update(self):
logger.info("Running pip_update to update the installation tools.")
try:
cmd = sys.executable + ' -m pip install --upgrade pip setuptools wheel pip-tools'
output = self._exec_command(cmd)
for line in output:
logger.info('pip_update output: %s' % line)
return True
except Exception as e:
logger.error('Command failed: %s' % e)
return False
def pip_sync(self):
logger.info("Running pip-sync to synchronize the environment.")
try:
cmd = sys.executable + ' -m piptools sync requirements.txt'
output = self._exec_command(cmd)
for line in output:
logger.info('pip-sync output: %s' % line)
return True
except Exception as e:
logger.error('Command failed: %s' % e)
return False
def runHandler(self):
if self._STARTED:
logger.warn('Log Queue Handler already started')
return
logger.info('Log Queue Handler starting')
self._STARTED = True
mainLogger = logging.getLogger("SpeakReader")
self.queueHandler = handlers.QueueHandler(self._receiverQueue)
self.queueHandler.setFormatter(logger.log_format)
self.queueHandler.setLevel(logger.log_level)
mainLogger.addHandler(self.queueHandler)
for handler in mainLogger.handlers[:]:
if isinstance(handler, handlers.RotatingFileHandler):
self.fileName = handler.baseFilename
break
while self._STARTED:
try:
logRecord = self._receiverQueue.get(timeout=2)
if logRecord is None:
break
# Python 3.6.8 doesn't seem to return a formatted message while 3.7.3 does.
logMessage = logRecord.getMessage()
formatted_logMessage = self.queueHandler.format(logRecord)
logRecord.msg = ""
formatted_header = self.queueHandler.format(logRecord)
if formatted_header not in logMessage:
logMessage = formatted_logMessage
data = {
"event": "logrecord",
"final": True,
"record": logMessage,
}
except queue.Empty:
if self._STARTED:
data = {"event": "ping"}
else:
break
for queueElement in list(self._listenerQueues.values()):
try:
queueElement.put_nowait(data)
except queue.Full:
self.removeListener(sessionID=queueElement.sessionID)
self._STARTED = False
self.closeAllListeners()
logger.info('Log Queue Handler terminated')
def addListener(self, type=None, remoteIP=None, sessionID=None):
if not self._STARTED or not remoteIP or not sessionID:
return None
logger.info("Adding " + type.capitalize() +
" Listener Queue for IP: " + remoteIP +
" with SessionID: " + sessionID)
if type == 'meter':
maxsize = 200
else:
maxsize = 20
queueElement = QueueElement(type=type,
remoteIP=remoteIP,
sessionID=sessionID,
maxsize=maxsize)
self._listenerQueues[sessionID] = queueElement
data = {
"event": "open",
"sessionID": sessionID,
}
queueElement.put_nowait(data)
data = None
if type == "log":
if self.fileName:
with open(self.fileName) as f:
records = '<p>' + f.read().replace("\n",
"</p><p>") + '</p>'
data = {
"event": "logrecord",
"final": "reload",
"record": records,
}
elif type == "transcript":
if self.fileName:
with open(self.fileName, 'r') as f:
records = "<p>" + f.read().rstrip("\n\n").replace(
"\n\n", "</p><p>") + "</p>"
data = {
"event": "transcript",
"final": "reload",
"record": records,
}
if data:
queueElement.put_nowait(data)
return queueElement.listenerQueue
def startTranscribeEngine(self):
if self.transcribeEngine.is_online:
logger.info("Transcribe Engine already started.")
return
if self.get_input_device() is None:
logger.warn("No Input Devices Available. Can't start Transcribe Engine.")
return
if CONFIG.SPEECH_TO_TEXT_SERVICE == 'google':
if CONFIG.GOOGLE_CREDENTIALS_FILE == "":
logger.warn("API Credentials not available. Can't start Transcribe Engine.")
return
try:
with open(CONFIG.GOOGLE_CREDENTIALS_FILE) as f:
json.loads(f.read())
except json.decoder.JSONDecodeError:
logger.warn("API Credentials does not appear to be a valid JSON file. Can't start Transcribe Engine.")
return
elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM':
if CONFIG.IBM_CREDENTIALS_FILE == "":
logger.warn("API Credentials not available. Can't start Transcribe Engine.")
return
APIKEY = None
URL = None
try:
with open(CONFIG.IBM_CREDENTIALS_FILE) as f:
for line in f.read().splitlines():
parm = line.split('=')
if parm[0] == 'SPEECH_TO_TEXT_APIKEY':
APIKEY = parm[1]
if parm[0] == 'SPEECH_TO_TEXT_URL':
URL = parm[1]
except:
pass
if APIKEY is None or URL is None:
logger.warn("APIKEY or URL not found in IBM credentials file. Can't start Transcribe Engine.")
return
elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft':
if CONFIG.MICROSOFT_SERVICE_APIKEY == "" or CONFIG.MICROSOFT_SERVICE_REGION == "":
logger.warn("Microsoft Azure APIKEY and Region are required. Can't start Transcribe Engine.")
return
else:
return
self.transcribeEngine.start()
def daemonize(myPidFile):
if threading.activeCount() != 1:
logger.warn(
"There are %r active threads. Daemonizing may cause"
" strange behavior.", threading.enumerate())
sys.stdout.flush()
sys.stderr.flush()
# Do first fork
try:
pid = os.fork() # @UndefinedVariable - only available in UNIX
if pid != 0:
sys.exit(0)
except OSError as e:
raise RuntimeError("1st fork failed: %s [%d]", e.strerror, e.errno)
os.setsid()
# Make sure I can read my own files and shut out others
prev = os.umask(0) # @UndefinedVariable - only available in UNIX
os.umask(prev and int('077', 8))
# Make the child a session-leader by detaching from the terminal
try:
pid = os.fork() # @UndefinedVariable - only available in UNIX
if pid != 0:
sys.exit(0)
except OSError as e:
raise RuntimeError("2nd fork failed: %s [%d]", e.strerror, e.errno)
dev_null = open('/dev/null', 'r')
os.dup2(dev_null.fileno(), sys.stdin.fileno())
si = open('/dev/null', "r")
so = open('/dev/null', "a+")
se = open('/dev/null', "a+")
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
pid = os.getpid()
myPidFile.seek(0)
myPidFile.write(str(pid))
myPidFile.flush()
logger.info("Daemonized to PID: %d", pid)
def signin(self,
username=None,
password=None,
remember_me='0',
*args,
**kwargs):
if cherrypy.request.method != 'POST':
cherrypy.response.status = 405
return {'status': 'error', 'message': 'Sign in using POST.'}
error_message = {'status': 'error', 'message': 'Invalid credentials.'}
valid_login = check_credentials(username=username, password=password)
if valid_login:
time_delta = timedelta(
days=30) if remember_me == '1' else timedelta(minutes=60)
expiry = datetime.utcnow() + time_delta
payload = {'user': username, 'exp': expiry}
jwt_token = jwt.encode(payload,
speakreader.CONFIG.JWT_SECRET,
algorithm=JWT_ALGORITHM).decode('utf-8')
self.on_login(
username=username,
success=True,
)
jwt_cookie = JWT_COOKIE_NAME
cherrypy.response.cookie[jwt_cookie] = jwt_token
cherrypy.response.cookie[jwt_cookie]['expires'] = int(
time_delta.total_seconds())
cherrypy.response.cookie[jwt_cookie]['path'] = '/'
cherrypy.request.login = payload
cherrypy.response.status = 200
return {'status': 'success'}
else:
self.on_login(username=username)
logger.info("WebAuth :: Invalid admin login attempt from '%s'." %
username)
cherrypy.response.status = 401
return error_message
def removeListener(self, sessionID=None, listenerQueue=None):
queueElement = None
if sessionID is not None:
queueElement = self._listenerQueues[sessionID]
elif listenerQueue is not None:
for sessionID, queueElement in self._listenerQueues.items():
if queueElement.listenerQueue == listenerQueue:
break
if queueElement is not None:
logger.info("Removing " + queueElement.type.capitalize() +
" Listener Queue for IP: " + queueElement.remoteIP +
" with SessionID: " + sessionID)
queueElement.clear()
queueElement.put_nowait(None)
self._listenerQueues.pop(sessionID)
def checkout_git_branch(self):
if self.INSTALL_TYPE == 'git':
output, err = self.runGit('fetch %s' %
speakreader.CONFIG.GIT_REMOTE)
output, err = self.runGit('checkout %s' %
speakreader.CONFIG.GIT_BRANCH)
if not output:
logger.error('Unable to change git branch.')
return
for line in output.split('\n'):
if line.endswith(('Aborting', 'Aborting.')):
logger.error('Unable to checkout from git: ' + line)
logger.info('Output: ' + str(output))
output, err = self.runGit(
'pull %s %s' %
(speakreader.CONFIG.GIT_REMOTE, speakreader.CONFIG.GIT_BRANCH))
self.pip_update()
self.pip_sync()
def __init__(self):
if TranscribeEngine._INITIALIZED:
logger.warn("Transcribe Engine already Initialized")
return
logger.info("Transcribe Engine Initializing")
###################################################################################################
# Set Supported Platforms
###################################################################################################
self.GOOGLE_SERVICE = GOOGLE_SERVICE
self.IBM_SERVICE = IBM_SERVICE
self.MICROSOFT_SERVICE = MICROSOFT_SERVICE
###################################################################################################
# Initialize the Queue Manager
###################################################################################################
self.queueManager = QueueManager()
self.transcriptQueue = self.queueManager.transcriptHandler.getReceiverQueue(
)
TranscribeEngine._INITIALIZED = True
def run(self):
if self._ONLINE:
logger.warn("Transcribe Engine already Started")
return
logger.info(
"Transcribe Engine Starting with the %s%s Speech-To-Text Service" %
(speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE[0].upper(),
speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE[1:]))
FILENAME_DATESTRING = datetime.datetime.now().strftime(
FILENAME_DATE_FORMAT)
TRANSCRIPT_FILENAME = FILENAME_PREFIX + FILENAME_DATESTRING + "." + TRANSCRIPT_FILENAME_SUFFIX
RECORDING_FILENAME = FILENAME_PREFIX + FILENAME_DATESTRING + "." + RECORDING_FILENAME_SUFFIX
tf = os.path.join(speakreader.CONFIG.TRANSCRIPTS_FOLDER,
TRANSCRIPT_FILENAME)
self.queueManager.transcriptHandler.setFileName(tf)
try:
self.microphoneStream = MicrophoneStream(
speakreader.CONFIG.INPUT_DEVICE)
self.microphoneStream.recordingFilename = RECORDING_FILENAME
self.microphoneStream.meterQueue = self.queueManager.meterHandler.getReceiverQueue(
)
except Exception as e:
logger.debug("MicrophoneStream Exception: %s" % e)
self.transcriptQueue.put_nowait(self.OFFLINE_MESSAGE)
return
if speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'google' and self.GOOGLE_SERVICE:
transcribeService = googleTranscribe(self.microphoneStream)
elif speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM' and self.IBM_SERVICE:
transcribeService = ibmTranscribe(self.microphoneStream)
elif speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft' and self.MICROSOFT_SERVICE:
transcribeService = microsoftTranscribe(self.microphoneStream)
else:
logger.warn(
"No Supported Transcribe Service Selected. Can't start Transcribe Engine."
)
return
self.transcriptFile = open(tf, "a+")
self.transcriptQueue.put_nowait(self.ONLINE_MESSAGE)
self._ONLINE = True
try:
with self.microphoneStream as stream:
while self._ONLINE:
responses = transcribeService.transcribe()
self.process_responses(responses)
logger.info("Transcription Engine Stream Closed")
except Exception as e:
logger.error(e)
self.transcriptFile.close()
self.transcriptQueue.put_nowait(self.OFFLINE_MESSAGE)
self._ONLINE = False
logger.info("Transcribe Engine Terminated")
def cleanup_files(self):
logger.info("Running File Cleanup")
def delete(path, days):
try:
days = int(days)
except ValueError:
return
delete_date = datetime.datetime.now() - datetime.timedelta(days=days)
with os.scandir(path=path) as files:
for file in files:
file_info = file.stat()
if datetime.datetime.fromtimestamp(file_info.st_ctime) < delete_date:
filename = os.path.join(path, file.name)
logger.debug("Deleting: %s" % filename)
os.remove(filename)
if CONFIG.LOG_RETENTION_DAYS != "":
delete(CONFIG.LOG_DIR, CONFIG.LOG_RETENTION_DAYS)
if CONFIG.TRANSCRIPT_RETENTION_DAYS != "":
delete(CONFIG.TRANSCRIPTS_FOLDER, CONFIG.TRANSCRIPT_RETENTION_DAYS)
if CONFIG.RECORDING_RETENTION_DAYS != "":
delete(CONFIG.RECORDINGS_FOLDER, CONFIG.RECORDING_RETENTION_DAYS)
def shutdown(self):
self._INITIALIZED = False
self.transcriptHandler.shutdown()
self.logHandler.shutdown()
self.meterHandler.shutdown()
logger.info("Queue Manager terminated")
def __init__(self, initOptions):
if SpeakReader._INITIALIZED:
return
with INIT_LOCK:
global PROG_DIR
PROG_DIR = initOptions['prog_dir']
global DATA_DIR
DATA_DIR = initOptions['data_dir']
global CONFIG
CONFIG = initOptions['config']
assert CONFIG is not None
if isinstance(initOptions['http_port'], int):
self.HTTP_PORT = initOptions['http_port']
else:
self.HTTP_PORT = int(CONFIG.HTTP_PORT)
if self.HTTP_PORT < 21 or self.HTTP_PORT > 65535:
logger.warn("HTTP_PORT out of bounds: 21 < %s < 65535", self.HTTP_PORT)
self.HTTP_PORT = 8880
# Check if pyOpenSSL is installed. It is required for certificate generation
# and for CherryPy.
if CONFIG.ENABLE_HTTPS:
try:
import OpenSSL
except ImportError:
logger.warn("The pyOpenSSL module is missing. Install this "
"module to enable HTTPS. HTTPS will be disabled.")
CONFIG.ENABLE_HTTPS = False
if not CONFIG.HTTPS_CERT:
CONFIG.HTTPS_CERT = os.path.join(DATA_DIR, 'server.crt')
if not CONFIG.HTTPS_KEY:
CONFIG.HTTPS_KEY = os.path.join(DATA_DIR, 'server.key')
if not (os.path.exists(CONFIG.HTTPS_CERT) and os.path.exists(CONFIG.HTTPS_KEY)):
logger.warn("Disabled HTTPS because of missing certificate and key.")
CONFIG.ENABLE_HTTPS = False
# Check if we has a jwt_secret
if CONFIG.JWT_SECRET == '' or not CONFIG.JWT_SECRET:
logger.debug("Generating JWT secret...")
CONFIG.JWT_SECRET = generate_uuid()
CONFIG.write()
###################################################################################################
# Get Version Information and check for updates
###################################################################################################
self.versionInfo = Version()
###################################################################################################
# Get the Input Device
###################################################################################################
self.get_input_device()
###################################################################################################
# Initialize the Transcribe Engine
###################################################################################################
self.transcribeEngine = TranscribeEngine()
if CONFIG.START_TRANSCRIBE_ON_STARTUP :
self.startTranscribeEngine()
###################################################################################################
# Initialize the webserver
###################################################################################################
logger.info('WebServer Initializing')
webServerOptions = {
'config': CONFIG,
'prog_dir': PROG_DIR,
'data_dir': DATA_DIR,
'http_port': self.HTTP_PORT,
}
self.webServer = webstart.initialize(webServerOptions)
self.webServer.root.SR = self
cherrypy.server.start()
# Launch the WebBrowser
if CONFIG.LAUNCH_BROWSER and not initOptions['nolaunch']:
launch_browser(CONFIG.HTTP_HOST, self.HTTP_PORT, CONFIG.HTTP_ROOT + 'manage')
###################################################################################################
# Run cleanup of old logs, transcripts, and recordings and start a scheduler to run every 24 hours
###################################################################################################
self.cleanup_files()
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(self.cleanup_files, 'interval', hours=24)
self.scheduler.start()
SpeakReader._INITIALIZED = True
def _check_github(self):
self.COMMITS_BEHIND = 0
# Get the latest version available from github
logger.debug('Retrieving latest version information from GitHub')
url = 'https://api.github.com/repos/%s/%s/commits/%s' % (
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO,
speakreader.CONFIG.GIT_BRANCH)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
try:
response = requests.get(url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return
if response.ok:
version = response.json()
else:
logger.warn(
'Could not get the latest version information from GitHub for '
+ speakreader.CONFIG.GIT_REMOTE + '/' +
speakreader.CONFIG.GIT_BRANCH +
'. Are you running a local development version?')
return
self.LATEST_VERSION_HASH = version['sha']
# See how many commits behind we are
if not self.INSTALLED_VERSION_HASH:
logger.info(
'You are running an unknown version of SpeakReader. Run the updater to identify your version'
)
self.LATEST_RELEASE = "Unknown"
return
# Get latest release tag
logger.debug('Retrieving latest release information from GitHub')
url = 'https://api.github.com/repos/%s/%s/releases' % (
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
try:
response = requests.get(url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return
if response.ok:
releases = response.json()
else:
logger.warn('Could not get releases from GitHub.')
return
if speakreader.CONFIG.GIT_BRANCH == 'master':
release = next((r for r in releases if not r['prerelease']),
releases[0])
elif speakreader.CONFIG.GIT_BRANCH == 'beta':
release = next((r for r in releases), releases[0])
else:
release = releases[0]
self.LATEST_RELEASE = release['tag_name']
url = 'https://github.com/%s/%s/releases/tag/%s' \
% (speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO, self.LATEST_RELEASE)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
self.LATEST_RELEASE_URL = url
logger.info("Installed release is %s - %s" %
(self.INSTALLED_RELEASE, self.INSTALLED_VERSION_HASH))
logger.info(" Latest release is %s - %s" %
(self.LATEST_RELEASE, self.LATEST_VERSION_HASH))
if self.LATEST_VERSION_HASH == self.INSTALLED_VERSION_HASH:
logger.info('SpeakReader is up to date')
return
logger.debug(
'Comparing currently installed version with latest GitHub version')
url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % (
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO,
self.LATEST_VERSION_HASH, self.INSTALLED_VERSION_HASH)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
try:
response = requests.get(url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return
if response.ok:
commits = response.json()
else:
logger.warn('Could not get commits behind from GitHub.')
return
try:
self.COMMITS_BEHIND = int(commits['behind_by'])
logger.debug("In total, %d commits behind", self.COMMITS_BEHIND)
except KeyError:
logger.info(
'Cannot compare versions. Are you running a local development version?'
)
self.COMMITS_BEHIND = 0
if self.COMMITS_BEHIND > 0:
logger.info('Updates Available. You are %s commits behind' %
self.COMMITS_BEHIND)
elif self.COMMITS_BEHIND == 0:
logger.info('SpeakReader is up to date')
def update(self):
if speakreader.CONFIG.SERVER_ENVIRONMENT.lower() != 'production':
logger.info(
"Updating bypassed because this is not a production environment"
)
return False
if not self.UPDATE_AVAILABLE:
logger.info("No Updates Available")
return False
if self.INSTALL_TYPE == 'git':
output, err = self.runGit(
'diff --name-only %s/%s' %
(speakreader.CONFIG.GIT_REMOTE, speakreader.CONFIG.GIT_BRANCH))
if output == '':
logger.debug("No differences found from the origin")
elif output == 'requirements.txt':
logger.warn(
'Requirements file is out of sync. Restoring to original.')
output, err = self.runGit('checkout %s/%s requirements.txt' %
(speakreader.CONFIG.GIT_REMOTE,
speakreader.CONFIG.GIT_BRANCH))
else:
logger.error("Differences Found. Unable to update.")
logger.info('Output: ' + str(output))
return False
output, err = self.runGit('pull ' + speakreader.CONFIG.GIT_REMOTE +
' ' + speakreader.CONFIG.GIT_BRANCH)
if not output:
logger.error('Unable to download latest version')
return False
for line in output.split('\n'):
if 'Already up-to-date.' in line:
logger.info('No update available, not updating')
logger.info('Output: ' + str(output))
return False
elif line.endswith(('Aborting', 'Aborting.')):
logger.error('Unable to update from git: ' + line)
logger.info('Output: ' + str(output))
return False
else:
tar_download_url = 'https://api.github.com/repos/{}/{}/tarball/{}'.format(
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO,
speakreader.CONFIG.GIT_BRANCH)
if speakreader.CONFIG.GIT_TOKEN:
tar_download_url = tar_download_url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
update_dir = os.path.join(speakreader.PROG_DIR, 'update')
version_path = os.path.join(speakreader.PROG_DIR, 'version.txt')
logger.info('Downloading update from: ' + tar_download_url)
try:
data = requests.get(tar_download_url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return False
if not data:
logger.error(
"Unable to retrieve new version from '%s', can't update",
tar_download_url)
return False
download_name = speakreader.CONFIG.GIT_BRANCH + '-github'
tar_download_path = os.path.join(speakreader.PROG_DIR,
download_name)
# Save tar to disk
with open(tar_download_path, 'wb') as f:
f.write(data.content)
# Extract the tar to update folder
logger.info('Extracting file: ' + tar_download_path)
tar = tarfile.open(tar_download_path)
tar.extractall(update_dir)
tar.close()
# Delete the tar.gz
logger.info('Deleting file: ' + tar_download_path)
os.remove(tar_download_path)
# Find update dir name
update_dir_contents = [
x for x in os.listdir(update_dir)
if os.path.isdir(os.path.join(update_dir, x))
]
if len(update_dir_contents) != 1:
logger.error("Invalid update data, update failed: " +
str(update_dir_contents))
return False
content_dir = os.path.join(update_dir, update_dir_contents[0])
# walk temp folder and move files to main folder
for dirname, dirnames, filenames in os.walk(content_dir):
dirname = dirname[len(content_dir) + 1:]
for curfile in filenames:
old_path = os.path.join(content_dir, dirname, curfile)
new_path = os.path.join(speakreader.PROG_DIR, dirname,
curfile)
if os.path.isfile(new_path):
os.remove(new_path)
os.renames(old_path, new_path)
# Update version.txt
try:
with open(version_path, 'w') as f:
f.write(str(self.LATEST_VERSION_HASH))
except IOError as e:
logger.error(
"Unable to write current version to version.txt, update not complete: %s"
% e)
return False
self.pip_update()
self.pip_sync()
logger.info("Update Complete")
return True
def on_login(self, username=None, success=False):
"""Called on successful login"""
if success:
logger.info("WebAuth :: Admin user '%s' logged into SpeakReader." %
(username))
def on_logout(self, username):
"""Called on logout"""
logger.info("WebAuth :: Admin user '%s' logged out of SpeakReader." %
(username))
def update(self, **kwargs):
if self.SR.versionInfo.UPDATE_AVAILABLE:
return self.do_state_change('update', 'Updating', 180)
else:
logger.info("No Updates Available")
return self.manage()
def configUpdate(self, **kwargs):
logger.info("Processing configUpdate.")
restartTranscribeEngine = False
restartSpeakReader = False
logout = False
cleanup = False
checked_configs = [
"start_transcribe_on_startup",
"launch_browser",
"enable_https",
"save_recordings",
"show_interim_results",
"enable_censorship",
"http_hash_password",
"http_basic_auth",
"check_github",
]
for checked_config in checked_configs:
if checked_config not in kwargs:
# checked items should be zero or one. if they were not sent then the item was not checked
kwargs[checked_config] = 0
else:
kwargs[checked_config] = 1
upload_google_credentials_file = kwargs.pop(
'upload_google_credentials_file', None)
upload_ibm_credentials_file = kwargs.pop('upload_ibm_credentials_file',
None)
if upload_google_credentials_file.file is not None:
self.upload_credentials_file(upload_google_credentials_file)
kwargs['google_credentials_file'] = os.path.join(
speakreader.DATA_DIR, upload_google_credentials_file.filename)
if upload_ibm_credentials_file.file is not None:
self.upload_credentials_file(upload_ibm_credentials_file)
kwargs['ibm_credentials_file'] = os.path.join(
speakreader.DATA_DIR, upload_ibm_credentials_file.filename)
if kwargs.get('speech_to_text_service') != speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE \
or kwargs.get('google_credentials_file') != speakreader.CONFIG.GOOGLE_CREDENTIALS_FILE \
or kwargs.get('ibm_credentials_file') != speakreader.CONFIG.IBM_CREDENTIALS_FILE \
or kwargs.get('microsoft_service_apikey') != speakreader.CONFIG.MICROSOFT_SERVICE_APIKEY \
or kwargs.get('microsoft_service_region') != speakreader.CONFIG.MICROSOFT_SERVICE_REGION \
or kwargs.get('enable_censorship') != speakreader.CONFIG.ENABLE_CENSORSHIP \
or kwargs.get('input_device') != speakreader.CONFIG.INPUT_DEVICE \
or kwargs.get('save_recordings') != speakreader.CONFIG.SAVE_RECORDINGS:
restartTranscribeEngine = True
if kwargs.get('http_port') != str(speakreader.CONFIG.HTTP_PORT) \
or kwargs.get('enable_https') != speakreader.CONFIG.ENABLE_HTTPS \
or kwargs.get('https_cert') != speakreader.CONFIG.HTTPS_CERT \
or kwargs.get('https_cert_chain') != speakreader.CONFIG.HTTPS_CERT_CHAIN \
or kwargs.get('https_key') != speakreader.CONFIG.HTTPS_KEY \
or kwargs.get('transcripts_folder') != speakreader.CONFIG.TRANSCRIPTS_FOLDER \
or kwargs.get('recordings_folder') != speakreader.CONFIG.RECORDINGS_FOLDER \
or kwargs.get('log_dir') != speakreader.CONFIG.LOG_DIR:
restartSpeakReader = True
kwargs['censored_words'] = kwargs['censored_words'].rstrip(
'\r\n').replace('\r\n', ',').split(',')
while ("" in kwargs['censored_words']):
kwargs['censored_words'].remove("")
set_http_password = int(kwargs.pop('set_http_password', 0))
if kwargs.get('http_username') == "":
kwargs['http_password'] = ""
else:
if set_http_password:
if kwargs.get('http_hash_password'):
kwargs['http_password'] = pbkdf2_sha256.hash(
kwargs['http_password'])
else:
kwargs.pop('http_password', 0)
if kwargs.get('http_hash_password'
) != speakreader.CONFIG.HTTP_HASH_PASSWORD:
if kwargs.get('http_hash_password'
) and speakreader.CONFIG.HTTP_PASSWORD != "":
kwargs['http_password'] = pbkdf2_sha256.hash(
speakreader.CONFIG.HTTP_PASSWORD)
if kwargs[
'http_username'] != speakreader.CONFIG.HTTP_USERNAME or set_http_password:
logout = True
if kwargs.get('log_retention_days') != speakreader.CONFIG.LOG_RETENTION_DAYS \
or kwargs.get('transcript_retention_days') != speakreader.CONFIG.TRANSCRIPT_RETENTION_DAYS \
or kwargs.get('recording_retention_days') != speakreader.CONFIG.RECORDING_RETENTION_DAYS:
cleanup = True
speakreader.CONFIG.process_kwargs(kwargs)
speakreader.CONFIG.write()
if cleanup:
self.SR.cleanup_files()
if restartSpeakReader:
#self.restart()
return {'portchanged': True}
if restartTranscribeEngine and self.SR.transcribeEngine.is_online:
self.SR.stopTranscribeEngine()
self.SR.startTranscribeEngine()
return {
'result': 'success',
'google_credentials_file': kwargs['google_credentials_file'],
'ibm_credentials_file': kwargs['ibm_credentials_file'],
'logout': logout,
'portchanged': False,
}
def main():
"""
SpeakReader application entry point. Parses arguments, setups encoding and
initializes the application.
"""
DAEMON = False
NOFORK = False
QUIET = False
VERBOSE = False
NOLAUNCH = False
HTTP_PORT = None
PLATFORM = system()
PLATFORM_RELEASE = release()
PLATFORM_VERSION = version()
PLATFORM_LINUX_DISTRO = None
PLATFORM_PROCESSOR = processor()
PLATFORM_MACHINE = machine()
PLATFORM_IS_64BITS = sys.maxsize > 2**32
SYS_PLATFORM = sys.platform
# Fixed paths to application
if hasattr(sys, 'frozen'):
FULL_PATH = os.path.abspath(sys.executable)
else:
FULL_PATH = os.path.abspath(__file__)
PROG_DIR = os.path.dirname(FULL_PATH)
# Ensure only one instance of SpeakReader running.
PIDFILE = os.path.join(PROG_DIR, 'pidfile')
myPid = os.getpid()
if os.path.exists(PIDFILE) and os.path.isfile(PIDFILE):
for p in psutil.process_iter():
if 'python' in p.name() and p.pid != myPid:
for f in p.open_files():
if f.path == PIDFILE:
logger.error(
"SpeakReader is already Running. Exiting.")
sys.exit(0)
myPidFile = open(PIDFILE, 'w+')
myPidFile.write(str(myPid))
myPidFile.flush()
try:
locale.setlocale(locale.LC_ALL, "")
except (locale.Error, IOError):
pass
try:
SYS_TIMEZONE = str(tzlocal.get_localzone())
SYS_UTC_OFFSET = datetime.datetime.now(
pytz.timezone(SYS_TIMEZONE)).strftime('%z')
except (pytz.UnknownTimeZoneError, LookupError, ValueError) as e:
logger.error("Could not determine system timezone: %s" % e)
SYS_TIMEZONE = 'Unknown'
SYS_UTC_OFFSET = '+0000'
# Parse any passed startup arguments
ARGS = sys.argv[1:]
# Set up and gather command line arguments
parser = argparse.ArgumentParser(
description=
'A Python based monitoring and tracking tool for Plex Media Server.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Increase console logging verbosity')
parser.add_argument('-q',
'--quiet',
action='store_true',
help='Turn off console logging')
parser.add_argument('-d',
'--daemon',
action='store_true',
help='Run as a daemon')
parser.add_argument('-p',
'--port',
type=int,
help='Force SpeakReader to run on a specified port')
parser.add_argument(
'--datadir', help='Specify a directory where to store your data files')
parser.add_argument('--config', help='Specify a config file to use')
parser.add_argument('--nolaunch',
action='store_true',
help='Prevent browser from launching on startup')
parser.add_argument(
'--nofork',
action='store_true',
help='Start SpeakReader as a service, do not fork when restarting')
args = parser.parse_args()
# Force the http port if necessary
if args.port:
HTTP_PORT = args.port
logger.info('Using forced web server port: %i', HTTP_PORT)
# Don't launch the browser
if args.nolaunch:
NOLAUNCH = True
if args.verbose:
VERBOSE = True
if args.quiet:
QUIET = True
if args.daemon:
if SYS_PLATFORM == 'win32':
sys.stderr.write(
"Daemonizing not supported under Windows, starting normally\n")
else:
DAEMON = True
QUIET = True
if args.nofork:
NOFORK = True
logger.info(
"SpeakReader is running as a service, it will not fork when restarted."
)
# Determine which data directory and config file to use
if args.datadir:
DATA_DIR = args.datadir
else:
DATA_DIR = os.path.join(PROG_DIR, 'data')
if args.config:
CONFIG_FILE = args.config
else:
CONFIG_FILE = os.path.join(DATA_DIR, config.FILENAME)
# Try to create the DATA_DIR if it doesn't exist
if not os.path.exists(DATA_DIR):
try:
os.makedirs(DATA_DIR)
except OSError:
raise SystemExit('Could not create data directory: ' + DATA_DIR +
'. Exiting....')
# Make sure the DATA_DIR is writeable
if not os.access(DATA_DIR, os.W_OK):
raise SystemExit('Cannot write to the data directory: ' + DATA_DIR +
'. Exiting...')
# Do an initial setup of the logging.
logger.initLogger(console=not QUIET, log_dir=False, verbose=VERBOSE)
# Initialize the configuration from the config file
CONFIG = config.Config(CONFIG_FILE)
assert CONFIG is not None
if CONFIG.SERVER_ENVIRONMENT.lower() != 'production':
VERBOSE = True
CONFIG.LOG_DIR, log_writable = check_folder_writable(
CONFIG.LOG_DIR, os.path.join(DATA_DIR, 'logs'), 'logs')
if not log_writable and not QUIET:
sys.stderr.write(
"Unable to create the log directory. Logging to screen only.\n")
# Start the logger, disable console if needed
logger.initLogger(console=not QUIET,
log_dir=CONFIG.LOG_DIR if log_writable else None,
verbose=VERBOSE)
logger.info("Initializing {} {}".format(speakreader.PRODUCT,
speakreader.VERSION_RELEASE))
logger.info("{} {} ({}{})".format(
PLATFORM, PLATFORM_RELEASE, PLATFORM_VERSION,
' - {}'.format(PLATFORM_LINUX_DISTRO)
if PLATFORM_LINUX_DISTRO else ''))
logger.info("{} ({} {})".format(
PLATFORM_PROCESSOR, PLATFORM_MACHINE,
'{}'.format('64-BIT') if PLATFORM_IS_64BITS else '32-BIT'))
logger.info("Python {}".format(sys.version))
logger.info("{} (UTC{})".format(SYS_TIMEZONE, SYS_UTC_OFFSET))
logger.info("Program Dir: {}".format(PROG_DIR))
logger.info("Config File: {}".format(CONFIG_FILE))
CONFIG.TRANSCRIPTS_FOLDER, _ = check_folder_writable(
CONFIG.TRANSCRIPTS_FOLDER, os.path.join(DATA_DIR, 'transcripts'),
'transcripts')
CONFIG.RECORDINGS_FOLDER, _ = check_folder_writable(
CONFIG.RECORDINGS_FOLDER, os.path.join(DATA_DIR, 'recordings'),
'recordings')
if DAEMON:
daemonize(myPidFile)
# Store the original umask
UMASK = os.umask(0)
os.umask(UMASK)
initOptions = {
'config': CONFIG,
'http_port': HTTP_PORT,
'nolaunch': NOLAUNCH,
'prog_dir': PROG_DIR,
'data_dir': DATA_DIR,
}
# Read config and start logging
global SR
SR = SpeakReader(initOptions)
# Wait endlessly for a signal to happen
restart = False
checkout = False
update = False
while True:
if not SR.SIGNAL:
try:
time.sleep(1)
except KeyboardInterrupt:
SR.SIGNAL = 'shutdown'
else:
logger.info('Received signal: %s', SR.SIGNAL)
if SR.SIGNAL == 'shutdown':
break
elif SR.SIGNAL == 'restart':
restart = True
break
elif SR.SIGNAL == 'update':
restart = True
update = True
break
elif SR.SIGNAL == 'checkout':
restart = True
checkout = True
break
else:
SR.SIGNAL = None
SR.shutdown(restart=restart, update=update, checkout=checkout)
myPidFile.close()
os.remove(PIDFILE)
if restart:
logger.info("SpeakReader is restarting...")
exe = sys.executable
args = [exe, FULL_PATH]
args += ARGS
if '--nolaunch' not in args:
args += ['--nolaunch']
# Separate out logger so we can shutdown logger after
if NOFORK:
logger.info('Running as service, not forking. Exiting...')
elif os.name == 'nt':
logger.info('Restarting SpeakReader with %s', args)
else:
logger.info('Restarting SpeakReader with %s', args)
logger.shutdown()
if NOFORK:
pass
elif os.name == 'nt':
subprocess.Popen(args, cwd=os.getcwd())
else:
os.execv(exe, args)
else:
logger.info("SpeakReader Terminated")
logger.shutdown()
sys.exit(0)
def shutdown(self, restart=False, update=False, checkout=False):
SpeakReader._INITIALIZED = False
self.transcribeEngine.shutdown()
self.scheduler.shutdown()
CONFIG.write()
if not restart and not update and not checkout:
logger.info("Shutting Down SpeakReader")
if update:
logger.info("********************************")
logger.info("* SpeakReader is updating... *")
logger.info("********************************")
try:
self.versionInfo.update()
except Exception as e:
logger.warn("SpeakReader failed to update: %s. Restarting." % e)
logger.info('WebServer Terminating')
cherrypy.engine.exit()
if checkout:
logger.info("SpeakReader is switching the git branch...")
try:
self.versionInfo.checkout_git_branch()
except Exception as e:
logger.warn("SpeakReader failed to switch git branch: %s. Restarting." % e)
def initialize(options):
from speakreader import webauth
from speakreader.webserve import WebInterface
CONFIG = options['config']
options_dict = {
'server.socket_port': options['http_port'],
'server.socket_host': CONFIG.HTTP_HOST,
'environment': 'production',
'server.thread_pool': 50,
'tools.encode.on': True,
'tools.encode.encoding': 'utf-8',
'tools.decode.on': True
}
if CONFIG.ENABLE_HTTPS:
options_dict['server.ssl_certificate'] = CONFIG.HTTPS_CERT
options_dict['server.ssl_certificate_chain'] = CONFIG.HTTPS_CERT_CHAIN
options_dict['server.ssl_private_key'] = CONFIG.HTTPS_KEY
protocol = "https"
else:
protocol = "http"
if CONFIG.HTTP_PROXY:
# Overwrite cherrypy.tools.proxy with our own proxy handler
cherrypy.tools.proxy = cherrypy.Tool('before_handler',
proxy,
priority=1)
if CONFIG.HTTP_PASSWORD:
login_allowed = [
"SpeakReader admin (username is '%s')" % CONFIG.HTTP_USERNAME
]
logger.info("WebStart :: Web server authentication is enabled: %s.",
' and '.join(login_allowed))
if CONFIG.HTTP_BASIC_AUTH:
auth_enabled = False
basic_auth_enabled = True
else:
auth_enabled = True
basic_auth_enabled = False
cherrypy.tools.auth = cherrypy.Tool('before_handler',
webauth.check_auth,
priority=2)
else:
auth_enabled = basic_auth_enabled = False
cherrypy.config.update(options_dict)
conf = {
'/': {
'tools.staticdir.root':
os.path.join(options['prog_dir'], 'html'),
'tools.proxy.on':
bool(CONFIG.HTTP_PROXY),
'tools.gzip.on':
True,
'tools.gzip.mime_types': [
'text/html', 'text/plain', 'text/css', 'text/javascript',
'application/json', 'application/javascript'
],
'tools.auth.on':
False,
'tools.auth_basic.on':
False,
'tools.sessions.on':
True,
},
'/manage': {
'tools.staticdir.root':
os.path.join(options['prog_dir'], 'html'),
'tools.proxy.on':
bool(CONFIG.HTTP_PROXY),
'tools.gzip.on':
True,
'tools.gzip.mime_types': [
'text/html', 'text/plain', 'text/css', 'text/javascript',
'application/json', 'application/javascript'
],
'tools.auth.on':
auth_enabled,
'tools.auth_basic.on':
basic_auth_enabled,
'tools.auth_basic.realm':
'SpeakReader web server',
'tools.auth_basic.checkpassword':
cherrypy.lib.auth_basic.checkpassword_dict(
{CONFIG.HTTP_USERNAME: CONFIG.HTTP_PASSWORD}),
'tools.sessions.on':
True,
},
'/images': {
'tools.staticdir.on': True,
'tools.staticdir.dir': "images",
'tools.staticdir.content_types': {
'svg': 'image/svg+xml'
},
'tools.caching.on': True,
'tools.caching.force': True,
'tools.caching.delay': 0,
'tools.expires.on': True,
'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days
'tools.sessions.on': False,
'tools.auth.on': False
},
'/css': {
'tools.staticdir.on': True,
'tools.staticdir.dir': "css",
'tools.caching.on': True,
'tools.caching.force': True,
'tools.caching.delay': 0,
'tools.expires.on': True,
'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days
'tools.sessions.on': False,
'tools.auth.on': False
},
'/fonts': {
'tools.staticdir.on': True,
'tools.staticdir.dir': "fonts",
'tools.caching.on': True,
'tools.caching.force': True,
'tools.caching.delay': 0,
'tools.expires.on': True,
'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days
'tools.sessions.on': False,
'tools.auth.on': False
},
'/js': {
'tools.staticdir.on': True,
'tools.staticdir.dir': "js",
'tools.caching.on': True,
'tools.caching.force': True,
'tools.caching.delay': 0,
'tools.expires.on': True,
'tools.expires.secs': 60 * 60 * 24 * 30, # 30 days
'tools.sessions.on': False,
'tools.auth.on': False
},
'/favicon.ico': {
'tools.staticfile.on':
True,
'tools.staticfile.filename':
os.path.abspath(
os.path.join(options['prog_dir'],
'/html/images/favicon/favicon.ico')),
'tools.caching.on':
True,
'tools.caching.force':
True,
'tools.caching.delay':
0,
'tools.expires.on':
True,
'tools.expires.secs':
60 * 60 * 24 * 30, # 30 days
'tools.sessions.on':
False,
'tools.auth.on':
False
}
}
# Prevent time-outs
appTree = cherrypy.tree.mount(WebInterface(),
CONFIG.HTTP_ROOT,
config=conf)
if CONFIG.HTTP_ROOT != '/':
cherrypy.tree.mount(BaseRedirect(), '/')
try:
logger.info("WebStart :: Starting Web Server on %s://%s:%d%s",
protocol, CONFIG.HTTP_HOST, options['http_port'],
CONFIG.HTTP_ROOT)
portend.free(str(CONFIG.HTTP_HOST), options['http_port'])
except IOError:
sys.stderr.write(
'Failed to start on port: %i. Is something else running?\n' %
(options['http_port']))
sys.exit(1)
#
return appTree
