def runHandler(self):
if self._STARTED:
logger.warn('Sound Meter Queue Handler already started')
return
logger.info('Sound Meter Queue Handler starting')
self._STARTED = True
while self._STARTED:
try:
meterRecord = self._receiverQueue.get(timeout=2)
if meterRecord is None:
break
data = {
"event": "meterrecord",
"final": True,
"record": meterRecord,
}
except queue.Empty:
if self._STARTED:
data = {"event": "ping"}
else:
break
for queueElement in list(self._listenerQueues.values()):
try:
queueElement.put_nowait(data)
except queue.Full:
self.removeListener(sessionID=queueElement.sessionID)
self._STARTED = False
self.closeAllListeners()
logger.info('Sound Meter Queue Handler terminated')
def runHandler(self):
if self._STARTED:
logger.warn('Transcript Queue Handler already started')
return
logger.info('Transcript Queue Handler starting')
self._STARTED = True
while self._STARTED:
try:
transcript = self._receiverQueue.get(timeout=2)
if transcript is None:
break
except queue.Empty:
if self._STARTED:
transcript = {"event": "ping"}
else:
break
for queueElement in list(self._listenerQueues.values()):
try:
queueElement.put_nowait(transcript)
except queue.Full:
self.removeListener(sessionID=queueElement.sessionID)
self._STARTED = False
self.closeAllListeners()
logger.info('Transcript Queue Handler terminated')
def __init__(self, audio_device):
self.is_supported = is_supported
if not self.is_supported:
return
self.audio_device = audio_device
APIKEY = None
URL = None
with open(speakreader.CONFIG.IBM_CREDENTIALS_FILE) as f:
for line in f.read().splitlines():
parm = line.split('=')
if parm[0] == 'SPEECH_TO_TEXT_APIKEY':
APIKEY = parm[1]
if parm[0] == 'SPEECH_TO_TEXT_URL':
URL = parm[1]
if APIKEY is None or URL is None:
logger.warn('ibmTranscribe: APIKEY or URL not found in credentials file')
# initialize speech to text service
self.authenticator = IAMAuthenticator(APIKEY)
self.speech_to_text = SpeechToTextV1(authenticator=self.authenticator)
self.speech_to_text.set_service_url(URL)
self.mycallback = ProcessResponses()
self.audio_source = AudioSource(audio_device._streamBuff, is_recording=True, is_buffer=True)
def check_folder_writable(folder, fallback, name):
if not folder:
folder = fallback
if not os.path.exists(folder):
try:
os.makedirs(folder)
except OSError as e:
logger.error("Could not create %s dir '%s': %s" %
(name, folder, e))
if folder != fallback:
logger.warn("Falling back to %s dir '%s'" % (name, fallback))
return check_folder_writable(None, fallback, name)
else:
return folder, None
if not os.access(folder, os.W_OK):
logger.error("Cannot write to %s dir '%s'" % (name, folder))
if folder != fallback:
logger.warn("Falling back to %s dir '%s'" % (name, fallback))
return check_folder_writable(None, fallback, name)
else:
return folder, False
return folder, True
def shutdown(self, restart=False, update=False, checkout=False):
SpeakReader._INITIALIZED = False
self.transcribeEngine.shutdown()
self.scheduler.shutdown()
CONFIG.write()
if not restart and not update and not checkout:
logger.info("Shutting Down SpeakReader")
if update:
logger.info("********************************")
logger.info("* SpeakReader is updating... *")
logger.info("********************************")
try:
self.versionInfo.update()
except Exception as e:
logger.warn("SpeakReader failed to update: %s. Restarting." % e)
logger.info('WebServer Terminating')
cherrypy.engine.exit()
if checkout:
logger.info("SpeakReader is switching the git branch...")
try:
self.versionInfo.checkout_git_branch()
except Exception as e:
logger.warn("SpeakReader failed to switch git branch: %s. Restarting." % e)
def run(self):
if self._ONLINE:
logger.warn("Transcribe Engine already Started")
return
logger.info(
"Transcribe Engine Starting with the %s%s Speech-To-Text Service" %
(speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE[0].upper(),
speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE[1:]))
FILENAME_DATESTRING = datetime.datetime.now().strftime(
FILENAME_DATE_FORMAT)
TRANSCRIPT_FILENAME = FILENAME_PREFIX + FILENAME_DATESTRING + "." + TRANSCRIPT_FILENAME_SUFFIX
RECORDING_FILENAME = FILENAME_PREFIX + FILENAME_DATESTRING + "." + RECORDING_FILENAME_SUFFIX
tf = os.path.join(speakreader.CONFIG.TRANSCRIPTS_FOLDER,
TRANSCRIPT_FILENAME)
self.queueManager.transcriptHandler.setFileName(tf)
try:
self.microphoneStream = MicrophoneStream(
speakreader.CONFIG.INPUT_DEVICE)
self.microphoneStream.recordingFilename = RECORDING_FILENAME
self.microphoneStream.meterQueue = self.queueManager.meterHandler.getReceiverQueue(
)
except Exception as e:
logger.debug("MicrophoneStream Exception: %s" % e)
self.transcriptQueue.put_nowait(self.OFFLINE_MESSAGE)
return
if speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'google' and self.GOOGLE_SERVICE:
transcribeService = googleTranscribe(self.microphoneStream)
elif speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM' and self.IBM_SERVICE:
transcribeService = ibmTranscribe(self.microphoneStream)
elif speakreader.CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft' and self.MICROSOFT_SERVICE:
transcribeService = microsoftTranscribe(self.microphoneStream)
else:
logger.warn(
"No Supported Transcribe Service Selected. Can't start Transcribe Engine."
)
return
self.transcriptFile = open(tf, "a+")
self.transcriptQueue.put_nowait(self.ONLINE_MESSAGE)
self._ONLINE = True
try:
with self.microphoneStream as stream:
while self._ONLINE:
responses = transcribeService.transcribe()
self.process_responses(responses)
logger.info("Transcription Engine Stream Closed")
except Exception as e:
logger.error(e)
self.transcriptFile.close()
self.transcriptQueue.put_nowait(self.OFFLINE_MESSAGE)
self._ONLINE = False
logger.info("Transcribe Engine Terminated")
def __init__(self):
if self._INITIALIZED:
logger.warn('Queue Manager already Initialized')
return
logger.info('Queue Manager Initializing')
self.transcriptHandler = TranscriptHandler("TranscriptQueueHandler")
self.logHandler = LogHandler("LogQueueHandler")
self.meterHandler = MeterHandler("SoundMeterQueueHandler")
self._INITIALIZED = True
def runHandler(self):
if self._STARTED:
logger.warn('Log Queue Handler already started')
return
logger.info('Log Queue Handler starting')
self._STARTED = True
mainLogger = logging.getLogger("SpeakReader")
self.queueHandler = handlers.QueueHandler(self._receiverQueue)
self.queueHandler.setFormatter(logger.log_format)
self.queueHandler.setLevel(logger.log_level)
mainLogger.addHandler(self.queueHandler)
for handler in mainLogger.handlers[:]:
if isinstance(handler, handlers.RotatingFileHandler):
self.fileName = handler.baseFilename
break
while self._STARTED:
try:
logRecord = self._receiverQueue.get(timeout=2)
if logRecord is None:
break
# Python 3.6.8 doesn't seem to return a formatted message while 3.7.3 does.
logMessage = logRecord.getMessage()
formatted_logMessage = self.queueHandler.format(logRecord)
logRecord.msg = ""
formatted_header = self.queueHandler.format(logRecord)
if formatted_header not in logMessage:
logMessage = formatted_logMessage
data = {
"event": "logrecord",
"final": True,
"record": logMessage,
}
except queue.Empty:
if self._STARTED:
data = {"event": "ping"}
else:
break
for queueElement in list(self._listenerQueues.values()):
try:
queueElement.put_nowait(data)
except queue.Full:
self.removeListener(sessionID=queueElement.sessionID)
self._STARTED = False
self.closeAllListeners()
logger.info('Log Queue Handler terminated')
def daemonize(myPidFile):
if threading.activeCount() != 1:
logger.warn(
"There are %r active threads. Daemonizing may cause"
" strange behavior.", threading.enumerate())
sys.stdout.flush()
sys.stderr.flush()
# Do first fork
try:
pid = os.fork() # @UndefinedVariable - only available in UNIX
if pid != 0:
sys.exit(0)
except OSError as e:
raise RuntimeError("1st fork failed: %s [%d]", e.strerror, e.errno)
os.setsid()
# Make sure I can read my own files and shut out others
prev = os.umask(0) # @UndefinedVariable - only available in UNIX
os.umask(prev and int('077', 8))
# Make the child a session-leader by detaching from the terminal
try:
pid = os.fork() # @UndefinedVariable - only available in UNIX
if pid != 0:
sys.exit(0)
except OSError as e:
raise RuntimeError("2nd fork failed: %s [%d]", e.strerror, e.errno)
dev_null = open('/dev/null', 'r')
os.dup2(dev_null.fileno(), sys.stdin.fileno())
si = open('/dev/null', "r")
so = open('/dev/null', "a+")
se = open('/dev/null', "a+")
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
pid = os.getpid()
myPidFile.seek(0)
myPidFile.write(str(pid))
myPidFile.flush()
logger.info("Daemonized to PID: %d", pid)
def __init__(self):
if TranscribeEngine._INITIALIZED:
logger.warn("Transcribe Engine already Initialized")
return
logger.info("Transcribe Engine Initializing")
###################################################################################################
# Set Supported Platforms
###################################################################################################
self.GOOGLE_SERVICE = GOOGLE_SERVICE
self.IBM_SERVICE = IBM_SERVICE
self.MICROSOFT_SERVICE = MICROSOFT_SERVICE
###################################################################################################
# Initialize the Queue Manager
###################################################################################################
self.queueManager = QueueManager()
self.transcriptQueue = self.queueManager.transcriptHandler.getReceiverQueue(
)
TranscribeEngine._INITIALIZED = True
def update(self):
if speakreader.CONFIG.SERVER_ENVIRONMENT.lower() != 'production':
logger.info(
"Updating bypassed because this is not a production environment"
)
return False
if not self.UPDATE_AVAILABLE:
logger.info("No Updates Available")
return False
if self.INSTALL_TYPE == 'git':
output, err = self.runGit(
'diff --name-only %s/%s' %
(speakreader.CONFIG.GIT_REMOTE, speakreader.CONFIG.GIT_BRANCH))
if output == '':
logger.debug("No differences found from the origin")
elif output == 'requirements.txt':
logger.warn(
'Requirements file is out of sync. Restoring to original.')
output, err = self.runGit('checkout %s/%s requirements.txt' %
(speakreader.CONFIG.GIT_REMOTE,
speakreader.CONFIG.GIT_BRANCH))
else:
logger.error("Differences Found. Unable to update.")
logger.info('Output: ' + str(output))
return False
output, err = self.runGit('pull ' + speakreader.CONFIG.GIT_REMOTE +
' ' + speakreader.CONFIG.GIT_BRANCH)
if not output:
logger.error('Unable to download latest version')
return False
for line in output.split('\n'):
if 'Already up-to-date.' in line:
logger.info('No update available, not updating')
logger.info('Output: ' + str(output))
return False
elif line.endswith(('Aborting', 'Aborting.')):
logger.error('Unable to update from git: ' + line)
logger.info('Output: ' + str(output))
return False
else:
tar_download_url = 'https://api.github.com/repos/{}/{}/tarball/{}'.format(
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO,
speakreader.CONFIG.GIT_BRANCH)
if speakreader.CONFIG.GIT_TOKEN:
tar_download_url = tar_download_url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
update_dir = os.path.join(speakreader.PROG_DIR, 'update')
version_path = os.path.join(speakreader.PROG_DIR, 'version.txt')
logger.info('Downloading update from: ' + tar_download_url)
try:
data = requests.get(tar_download_url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return False
if not data:
logger.error(
"Unable to retrieve new version from '%s', can't update",
tar_download_url)
return False
download_name = speakreader.CONFIG.GIT_BRANCH + '-github'
tar_download_path = os.path.join(speakreader.PROG_DIR,
download_name)
# Save tar to disk
with open(tar_download_path, 'wb') as f:
f.write(data.content)
# Extract the tar to update folder
logger.info('Extracting file: ' + tar_download_path)
tar = tarfile.open(tar_download_path)
tar.extractall(update_dir)
tar.close()
# Delete the tar.gz
logger.info('Deleting file: ' + tar_download_path)
os.remove(tar_download_path)
# Find update dir name
update_dir_contents = [
x for x in os.listdir(update_dir)
if os.path.isdir(os.path.join(update_dir, x))
]
if len(update_dir_contents) != 1:
logger.error("Invalid update data, update failed: " +
str(update_dir_contents))
return False
content_dir = os.path.join(update_dir, update_dir_contents[0])
# walk temp folder and move files to main folder
for dirname, dirnames, filenames in os.walk(content_dir):
dirname = dirname[len(content_dir) + 1:]
for curfile in filenames:
old_path = os.path.join(content_dir, dirname, curfile)
new_path = os.path.join(speakreader.PROG_DIR, dirname,
curfile)
if os.path.isfile(new_path):
os.remove(new_path)
os.renames(old_path, new_path)
# Update version.txt
try:
with open(version_path, 'w') as f:
f.write(str(self.LATEST_VERSION_HASH))
except IOError as e:
logger.error(
"Unable to write current version to version.txt, update not complete: %s"
% e)
return False
self.pip_update()
self.pip_sync()
logger.info("Update Complete")
return True
def _check_github(self):
self.COMMITS_BEHIND = 0
# Get the latest version available from github
logger.debug('Retrieving latest version information from GitHub')
url = 'https://api.github.com/repos/%s/%s/commits/%s' % (
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO,
speakreader.CONFIG.GIT_BRANCH)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
try:
response = requests.get(url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return
if response.ok:
version = response.json()
else:
logger.warn(
'Could not get the latest version information from GitHub for '
+ speakreader.CONFIG.GIT_REMOTE + '/' +
speakreader.CONFIG.GIT_BRANCH +
'. Are you running a local development version?')
return
self.LATEST_VERSION_HASH = version['sha']
# See how many commits behind we are
if not self.INSTALLED_VERSION_HASH:
logger.info(
'You are running an unknown version of SpeakReader. Run the updater to identify your version'
)
self.LATEST_RELEASE = "Unknown"
return
# Get latest release tag
logger.debug('Retrieving latest release information from GitHub')
url = 'https://api.github.com/repos/%s/%s/releases' % (
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
try:
response = requests.get(url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return
if response.ok:
releases = response.json()
else:
logger.warn('Could not get releases from GitHub.')
return
if speakreader.CONFIG.GIT_BRANCH == 'master':
release = next((r for r in releases if not r['prerelease']),
releases[0])
elif speakreader.CONFIG.GIT_BRANCH == 'beta':
release = next((r for r in releases), releases[0])
else:
release = releases[0]
self.LATEST_RELEASE = release['tag_name']
url = 'https://github.com/%s/%s/releases/tag/%s' \
% (speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO, self.LATEST_RELEASE)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
self.LATEST_RELEASE_URL = url
logger.info("Installed release is %s - %s" %
(self.INSTALLED_RELEASE, self.INSTALLED_VERSION_HASH))
logger.info(" Latest release is %s - %s" %
(self.LATEST_RELEASE, self.LATEST_VERSION_HASH))
if self.LATEST_VERSION_HASH == self.INSTALLED_VERSION_HASH:
logger.info('SpeakReader is up to date')
return
logger.debug(
'Comparing currently installed version with latest GitHub version')
url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % (
speakreader.CONFIG.GIT_USER, speakreader.CONFIG.GIT_REPO,
self.LATEST_VERSION_HASH, self.INSTALLED_VERSION_HASH)
if speakreader.CONFIG.GIT_TOKEN:
url = url + '?access_token=%s' % speakreader.CONFIG.GIT_TOKEN
try:
response = requests.get(url, timeout=10)
except Exception as e:
logger.warn('Failed to establish a connection to GitHub')
return
if response.ok:
commits = response.json()
else:
logger.warn('Could not get commits behind from GitHub.')
return
try:
self.COMMITS_BEHIND = int(commits['behind_by'])
logger.debug("In total, %d commits behind", self.COMMITS_BEHIND)
except KeyError:
logger.info(
'Cannot compare versions. Are you running a local development version?'
)
self.COMMITS_BEHIND = 0
if self.COMMITS_BEHIND > 0:
logger.info('Updates Available. You are %s commits behind' %
self.COMMITS_BEHIND)
elif self.COMMITS_BEHIND == 0:
logger.info('SpeakReader is up to date')
def on_error(self, error):
logger.warn('ibmTranscribe.ProcessResponses.Error Error received: {}'.format(error))
def __init__(self, initOptions):
if SpeakReader._INITIALIZED:
return
with INIT_LOCK:
global PROG_DIR
PROG_DIR = initOptions['prog_dir']
global DATA_DIR
DATA_DIR = initOptions['data_dir']
global CONFIG
CONFIG = initOptions['config']
assert CONFIG is not None
if isinstance(initOptions['http_port'], int):
self.HTTP_PORT = initOptions['http_port']
else:
self.HTTP_PORT = int(CONFIG.HTTP_PORT)
if self.HTTP_PORT < 21 or self.HTTP_PORT > 65535:
logger.warn("HTTP_PORT out of bounds: 21 < %s < 65535", self.HTTP_PORT)
self.HTTP_PORT = 8880
# Check if pyOpenSSL is installed. It is required for certificate generation
# and for CherryPy.
if CONFIG.ENABLE_HTTPS:
try:
import OpenSSL
except ImportError:
logger.warn("The pyOpenSSL module is missing. Install this "
"module to enable HTTPS. HTTPS will be disabled.")
CONFIG.ENABLE_HTTPS = False
if not CONFIG.HTTPS_CERT:
CONFIG.HTTPS_CERT = os.path.join(DATA_DIR, 'server.crt')
if not CONFIG.HTTPS_KEY:
CONFIG.HTTPS_KEY = os.path.join(DATA_DIR, 'server.key')
if not (os.path.exists(CONFIG.HTTPS_CERT) and os.path.exists(CONFIG.HTTPS_KEY)):
logger.warn("Disabled HTTPS because of missing certificate and key.")
CONFIG.ENABLE_HTTPS = False
# Check if we has a jwt_secret
if CONFIG.JWT_SECRET == '' or not CONFIG.JWT_SECRET:
logger.debug("Generating JWT secret...")
CONFIG.JWT_SECRET = generate_uuid()
CONFIG.write()
###################################################################################################
# Get Version Information and check for updates
###################################################################################################
self.versionInfo = Version()
###################################################################################################
# Get the Input Device
###################################################################################################
self.get_input_device()
###################################################################################################
# Initialize the Transcribe Engine
###################################################################################################
self.transcribeEngine = TranscribeEngine()
if CONFIG.START_TRANSCRIBE_ON_STARTUP :
self.startTranscribeEngine()
###################################################################################################
# Initialize the webserver
###################################################################################################
logger.info('WebServer Initializing')
webServerOptions = {
'config': CONFIG,
'prog_dir': PROG_DIR,
'data_dir': DATA_DIR,
'http_port': self.HTTP_PORT,
}
self.webServer = webstart.initialize(webServerOptions)
self.webServer.root.SR = self
cherrypy.server.start()
# Launch the WebBrowser
if CONFIG.LAUNCH_BROWSER and not initOptions['nolaunch']:
launch_browser(CONFIG.HTTP_HOST, self.HTTP_PORT, CONFIG.HTTP_ROOT + 'manage')
###################################################################################################
# Run cleanup of old logs, transcripts, and recordings and start a scheduler to run every 24 hours
###################################################################################################
self.cleanup_files()
self.scheduler = BackgroundScheduler()
self.scheduler.add_job(self.cleanup_files, 'interval', hours=24)
self.scheduler.start()
SpeakReader._INITIALIZED = True
def startTranscribeEngine(self):
if self.transcribeEngine.is_online:
logger.info("Transcribe Engine already started.")
return
if self.get_input_device() is None:
logger.warn("No Input Devices Available. Can't start Transcribe Engine.")
return
if CONFIG.SPEECH_TO_TEXT_SERVICE == 'google':
if CONFIG.GOOGLE_CREDENTIALS_FILE == "":
logger.warn("API Credentials not available. Can't start Transcribe Engine.")
return
try:
with open(CONFIG.GOOGLE_CREDENTIALS_FILE) as f:
json.loads(f.read())
except json.decoder.JSONDecodeError:
logger.warn("API Credentials does not appear to be a valid JSON file. Can't start Transcribe Engine.")
return
elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'IBM':
if CONFIG.IBM_CREDENTIALS_FILE == "":
logger.warn("API Credentials not available. Can't start Transcribe Engine.")
return
APIKEY = None
URL = None
try:
with open(CONFIG.IBM_CREDENTIALS_FILE) as f:
for line in f.read().splitlines():
parm = line.split('=')
if parm[0] == 'SPEECH_TO_TEXT_APIKEY':
APIKEY = parm[1]
if parm[0] == 'SPEECH_TO_TEXT_URL':
URL = parm[1]
except:
pass
if APIKEY is None or URL is None:
logger.warn("APIKEY or URL not found in IBM credentials file. Can't start Transcribe Engine.")
return
elif CONFIG.SPEECH_TO_TEXT_SERVICE == 'microsoft':
if CONFIG.MICROSOFT_SERVICE_APIKEY == "" or CONFIG.MICROSOFT_SERVICE_REGION == "":
logger.warn("Microsoft Azure APIKEY and Region are required. Can't start Transcribe Engine.")
return
else:
return
self.transcribeEngine.start()
