def __init__(self, path, stt_file, sr_keys):
"""Initializie."""
Thread.__init__(self)
self.mbox_queue = PollableQueue()
self.path = path
self.stt_file = stt_file
self.sr_keys = sr_keys
self.subdirs = []
self.speech = sr.Recognizer()
self.status = {}
self.cache = {}
self.lock = Lock()
if os.path.isfile(stt_file):
with open(stt_file, 'rb') as infile:
self.cache = pickle.load(infile)
self._save_cache()
self.inot = inotify.adapters.Inotify()
for subdir in ('INBOX', 'Old', 'Urgent'):
directory = os.path.join(self.path, subdir)
if os.path.isdir(directory):
self.subdirs.append(subdir)
logging.debug("Watching Directory: %s", directory)
self.inot.add_watch(directory.encode('utf-8'))
else:
logging.debug("Directory %s not found", directory)
def __init__(self, cdr_path, sleep=5.0):
"""Initialize variables."""
Thread.__init__(self)
self._cdr_queue = PollableQueue()
self._sleep = sleep
self._inot = None
self._watch = None
self._sql = None
self._cdr_file = None
self._entries = []
self._datefield = 'calldate'
if not cdr_path:
# Disable CDR handling
self._keymap = {}
elif os.path.isfile(cdr_path):
self._cdr_file = cdr_path
self._header = ['accountcode', 'src', 'dst', 'dcontext', 'clid',
'channel', 'dstchannel', 'lastapp', 'lastdata',
'start', 'answer', 'end', 'duration', 'billsec',
'disposition', 'amaflags', 'uniqueid', 'userfield']
self._keymap = {
'time': 'start',
'callerid': 'clid',
'src': 'src',
'dest': 'lastdata',
'context': 'dcontext',
'application': 'lastapp',
'duration': 'duration',
'disposition': 'disposition',
}
try:
if sys.platform.startswith('linux'):
import inotify.adapters
self._inot = inotify.adapters.Inotify()
self._inot.add_watch(self._cdr_file.encode('utf-8'))
finally:
pass
else:
import sqlalchemy
try:
self._sql = sqlalchemy.create_engine(
posixpath.dirname(cdr_path))
self._query = sqlalchemy.text(
"SELECT * from {} ORDER BY `{}` DESC"
.format(posixpath.basename(cdr_path), self._datefield))
except sqlalchemy.exc.OperationalError as exception:
logging.error(exception)
self._keymap = {
'time': 'calldate',
'callerid': 'clid',
'src': 'src',
'dest': 'lastdata',
'context': 'dcontext',
'application': 'lastapp',
'duration': 'duration',
'disposition': 'disposition',
}
self._keys = list(self._keymap.keys())
def __init__(self, conn, mbox, cdr, password):
"""Initialize."""
# pylint: disable=too-many-arguments
Thread.__init__(self)
self.conn = conn
self.mbox = mbox
self.cdr = cdr
self.mboxq = PollableQueue()
self.password = password
self.accept_pw = False
def __init__(self, ipaddr, port, password, callback=None, **kwargs):
"""constructor."""
self._ipaddr = ipaddr
self._port = port
self._password = encode_password(password).encode('utf-8')
self._callback = callback
self._soc = None
self._thread = None
self._status = {}
# Stop thread
self.signal = PollableQueue()
# Send data to the server
self.request_queue = PollableQueue()
# Receive data from the server
self.result_queue = PollableQueue()
if 'autostart' not in kwargs or kwargs['autostart']:
self.start()
class Connection(Thread):
"""Thread to handle a single TCP conection."""
def __init__(self, conn, mbox, cdr, password):
"""Initialize."""
# pylint: disable=too-many-arguments
Thread.__init__(self)
self.conn = conn
self.mbox = mbox
self.cdr = cdr
self.mboxq = PollableQueue()
self.password = password
self.accept_pw = False
def queue(self):
"""Fetch queue object."""
return self.mboxq
def _build_msg_list(self):
"""Send a sanitized list to the client."""
result = []
blacklist = ["wav"]
status = self.mbox.get_mbox_status()
for _k, ref in status.items():
result.append(
{key: val
for key, val in ref.items() if key not in blacklist})
return result
@staticmethod
def _build_cdr(entries, start=0, count=-1, sha=None):
"""Extract requested info from cdr list."""
if sha is not None:
msg = decode_from_sha(sha)
start, count = [int(item) for item in msg.split(b',')]
if not start or start < 0:
start = 0
if start >= len(entries) or count == 0:
return msg
if count < 0 or count + start > len(entries):
end = len(entries)
else:
end = start + count
return entries[start:end]
def _send(self, command, msg):
"""Send message prefixed by length."""
print('SEND command %s' % (command))
msglen = len(msg)
msglen = bytes([
msglen >> 24, 0xff & (msglen >> 16), 0xff & (msglen >> 8),
0xff & (msglen >> 0)
])
self.conn.send(bytes([command]) + bytes(msglen) + bytes(msg))
def _handle_request(self, request):
print(request)
if request['cmd'] == cmd.CMD_MESSAGE_PASSWORD:
self.accept_pw, msg = compare_password(self.password,
request['sha'])
if self.accept_pw:
self._send(cmd.CMD_MESSAGE_VERSION,
__version__.encode('utf-8'))
logging.info("Password accepted")
else:
self._send(cmd.CMD_MESSAGE_ERROR, msg.encode('utf-8'))
logging.warning("Password rejected: %s", msg)
elif not self.accept_pw:
logging.warning("Bad Password")
self._send(cmd.CMD_MESSAGE_ERROR, b'bad password')
elif request['cmd'] == cmd.CMD_MESSAGE_LIST:
logging.debug("Requested Message List")
self._send(cmd.CMD_MESSAGE_LIST,
json.dumps(self._build_msg_list()).encode('utf-8'))
elif request['cmd'] == cmd.CMD_MESSAGE_MP3:
msg = self.mbox.mp3(request['sha'])
if msg:
self._send(cmd.CMD_MESSAGE_MP3, msg)
else:
logging.warning("Couldn't find message for %s", request['sha'])
self._send(cmd.CMD_MESSAGE_ERROR,
"Could not find requested message")
elif request['cmd'] == cmd.CMD_MESSAGE_DELETE:
msg = self.mbox.delete(request['sha'])
self._send(cmd.CMD_MESSAGE_LIST,
json.dumps(self._build_msg_list()).encode('utf-8'))
elif request['cmd'] == cmd.CMD_MESSAGE_CDR_AVAILABLE:
if not self.cdr:
self._send(cmd.CMD_MESSAGE_ERROR, b'CDR Not enabled')
return
self._send(cmd.CMD_MESSAGE_CDR_AVAILABLE,
json.dumps({
'count': self.cdr.count()
}).encode('utf-8'))
elif request['cmd'] == cmd.CMD_MESSAGE_CDR:
if not self.cdr:
self._send(cmd.CMD_MESSAGE_ERROR, b'CDR Not enabled')
return
entries = self._build_cdr(self.cdr.entries(), sha=request['sha'])
try:
msg = {
'keys': self.cdr.keys(),
'entries': entries.decode('utf-8')
}
except:
msg = {'keys': self.cdr.keys(), 'entries': entries}
try:
for entry in entries:
if entry['time'] == '':
logging.exception('No timestamp for this: %s' %
(entry))
except:
asdf = ''
self._send(cmd.CMD_MESSAGE_CDR,
zlib.compress(json.dumps(msg).encode('utf-8')))
def run(self):
"""Thread main loop."""
while True:
readable, _w, _e = select.select([self.conn, self.mboxq], [], [])
if self.conn in readable:
try:
request = _parse_request(recv_blocking(self.conn, 66))
except RuntimeError:
logging.warning("Connection closed")
break
logging.debug(request)
self._handle_request(request)
if self.mboxq in readable:
msgtype = self.mboxq.get()
self.mboxq.task_done()
if msgtype == 'mbox':
self._send(
cmd.CMD_MESSAGE_LIST,
json.dumps(self._build_msg_list()).encode('utf-8'))
elif msgtype == 'cdr':
self._send(
cmd.CMD_MESSAGE_CDR_AVAILABLE,
json.dumps({
'count': self.cdr.count()
}).encode('utf-8'))
class WatchMailBox(Thread):
"""Thread to watch for new/removed messages in a mailbox."""
def __init__(self, path, stt_file, sr_keys):
"""Initializie."""
Thread.__init__(self)
self.mbox_queue = PollableQueue()
self.path = path
self.stt_file = stt_file
self.sr_keys = sr_keys
self.subdirs = []
self.speech = sr.Recognizer()
self.status = {}
self.cache = {}
self.lock = Lock()
if os.path.isfile(stt_file):
with open(stt_file, 'rb') as infile:
self.cache = pickle.load(infile)
self._save_cache()
self.inot = inotify.adapters.Inotify()
for subdir in ('INBOX', 'Old', 'Urgent'):
directory = os.path.join(self.path, subdir)
if os.path.isdir(directory):
self.subdirs.append(subdir)
logging.debug("Watching Directory: %s", directory)
self.inot.add_watch(directory.encode('utf-8'))
else:
logging.debug("Directory %s not found", directory)
def _save_cache(self):
"""Save cache data."""
with open(self.stt_file, 'wb') as outfile:
pickle.dump(self.cache, outfile, pickle.HIGHEST_PROTOCOL)
def _speech_to_text(self, fname):
"""Convert WAV to text."""
txt = ""
logging.debug('STT ' + fname)
try:
with sr.WavFile(fname) as source:
audio = self.speech.record(source) # read the entire WAV file
txt += self.speech.recognize_google(
audio, key=self.sr_keys['GOOGLE_KEY'])
except sr.UnknownValueError:
txt += "Google Speech Recognition could not understand audio"
except sr.RequestError as exception:
txt += "Could not request results from " \
"Google Speech Recognition service; {0}".format(exception)
logging.debug("Parsed %s --> %s", fname, txt)
return txt
def _sha_to_fname(self, sha):
"""Find fname from sha256."""
for fname in self.status:
if sha == self.status[fname]['sha']:
return fname
return None
@staticmethod
def _parse_msg_header(fname):
"""Parse asterisk voicemail metadata."""
ini = configparser.ConfigParser()
ini.read(fname)
try:
return dict(ini.items('message'))
except configparser.NoSectionError:
logging.exception("Couldn't parse: %s", fname)
return
@staticmethod
def _sha256(fname):
"""Get SHA256 sum of a file contents."""
sha = hashlib.sha256()
with open(fname, "rb") as infile:
for chunk in iter(lambda: infile.read(4096), b""):
sha.update(chunk)
return sha.hexdigest()
def _build_mbox_status(self):
"""Parse all messages in a mailbox."""
data = {}
for subdir in self.subdirs:
directory = os.path.join(self.path, subdir)
logging.debug('Reading: ' + directory)
if not os.path.isdir(directory):
continue
for filename in os.listdir(directory):
filename = os.path.join(directory, filename)
if not os.path.isfile(filename):
continue
basename, ext = os.path.splitext(filename)
logging.debug('Parsing: %s -- %s', basename, ext)
if basename not in data:
data[basename] = {'mbox': subdir}
if ext == '.txt':
data[basename]['info'] = self._parse_msg_header(filename)
elif ext == '.wav':
data[basename]['sha'] = self._sha256(filename)
data[basename]['wav'] = filename
for fname, ref in list(data.items()):
if ('info' not in ref or 'sha' not in ref
or not os.path.isfile(ref['wav'])):
logging.debug('Message is not complete: ' + fname)
del data[fname]
continue
sha = ref['sha']
if sha not in self.cache:
self.cache[sha] = {}
if 'txt' not in self.cache[sha]:
self.cache[sha]['txt'] = self._speech_to_text(ref['wav'])
self._save_cache()
logging.debug("SHA (%s): %s", fname, sha)
ref['text'] = self.cache[sha]['txt']
self.status = data
def delete(self, sha):
"""Delete message from mailbox."""
# This does introduce a race condition between
# Asterisk resequencing and us deleting, but there isn't
# much we can do about it
self.lock.acquire()
self._build_mbox_status()
fname = self._sha_to_fname(sha)
for fil in glob.glob(fname + ".*"):
os.unlink(fil)
dirname = os.path.basename(fname)
paths = {}
for fil in glob.iglob(os.path.join(dirname + "msg[0-9]*")):
base, = os.path.splitext(fil)
paths[base] = None
last_idx = 0
rename = []
with tempfile.TemporaryDirectory(dir=dirname) as tmp:
for base in sorted(paths):
if base != os.path.join(dirname + "msg%04d" % (last_idx)):
for fil in glob.glob(base + ".*"):
ext = os.path.splitext(fil)[1]
newfile = "msg%04d.%s" % (last_idx, ext)
logging.info("Renaming '" + fil + "' to '" +
os.path.join(dirname, newfile) + "'")
rename.append(newfile)
os.link(fil, os.path.join(tempfile, newfile))
for fil in rename:
finalname = os.path.join(dirname, fil)
if os.path.lexists(finalname):
os.unlink(finalname)
os.rename(os.path.join(tmp, fil), finalname)
self.lock.release()
def mp3(self, sha):
"""Convert WAV to MP3 using LAME."""
try:
fname = self._sha_to_fname(sha)
if not fname:
return None
logging.debug("Requested MP3 for %s ==> %s", sha, fname)
wav = self.status[fname]['wav']
process = subprocess.Popen([
"lame", "--abr", "24", "-mm", "-h", "-c", "--resample",
"22.050", "--quiet", wav, "-"
],
stdout=subprocess.PIPE)
result = process.communicate()[0]
return result
except OSError:
logging.exception("Failed To execute lame")
return
def update(self):
"""Rebuild mailbox data."""
self.lock.acquire()
self._build_mbox_status()
self.lock.release()
def get_mbox_status(self):
"""Return current mbox status."""
return self.status
def queue(self):
"""Fetch queue object."""
return self.mbox_queue
def run(self):
"""Execute main loop."""
self.mbox_queue.put("rebuild")
trigger_events = ('IN_DELETE', 'IN_CLOSE_WRITE', 'IN_MOVED_FROM',
'IN_MOVED_TO')
rebuild = False
try:
for event in self.inot.event_gen():
if event is not None:
(_header, type_names, _watch_path, _filename) = event
if set(type_names) & set(trigger_events):
rebuild = True
else:
if rebuild:
logging.debug("Rebuilding mailbox due to event: %s",
type_names)
self.mbox_queue.put("rebuild")
rebuild = False
finally:
for subdir in self.subdirs:
directory = os.path.join(self.path, subdir)
self.inot.remove_watch(directory.encode('utf-8'))
class WatchCDR(Thread):
"""Watch for new entries in the CDR database."""
def __init__(self, cdr_path, sleep=5.0):
"""Initialize variables."""
Thread.__init__(self)
self._cdr_queue = PollableQueue()
self._sleep = sleep
self._inot = None
self._watch = None
self._sql = None
self._cdr_file = None
self._entries = []
self._datefield = 'calldate'
if not cdr_path:
# Disable CDR handling
self._keymap = {}
elif os.path.isfile(cdr_path):
self._cdr_file = cdr_path
self._header = ['accountcode', 'src', 'dst', 'dcontext', 'clid',
'channel', 'dstchannel', 'lastapp', 'lastdata',
'start', 'answer', 'end', 'duration', 'billsec',
'disposition', 'amaflags', 'uniqueid', 'userfield']
self._keymap = {
'time': 'start',
'callerid': 'clid',
'src': 'src',
'dest': 'lastdata',
'context': 'dcontext',
'application': 'lastapp',
'duration': 'duration',
'disposition': 'disposition',
}
try:
if sys.platform.startswith('linux'):
import inotify.adapters
self._inot = inotify.adapters.Inotify()
self._inot.add_watch(self._cdr_file.encode('utf-8'))
finally:
pass
else:
import sqlalchemy
try:
self._sql = sqlalchemy.create_engine(
posixpath.dirname(cdr_path))
self._query = sqlalchemy.text(
"SELECT * from {} ORDER BY `{}` DESC"
.format(posixpath.basename(cdr_path), self._datefield))
except sqlalchemy.exc.OperationalError as exception:
logging.error(exception)
self._keymap = {
'time': 'calldate',
'callerid': 'clid',
'src': 'src',
'dest': 'lastdata',
'context': 'dcontext',
'application': 'lastapp',
'duration': 'duration',
'disposition': 'disposition',
}
self._keys = list(self._keymap.keys())
def _update_entries(self, entries):
entries = [{key: e.get(origkey, "")
for (key, origkey) in self._keymap.items()}
for e in entries]
if (len(entries) == len(self._entries)
and (not entries or entries[0] == self._entries[0])):
return False
self._entries = entries
return True
def _read_cdr_from_sql(self):
result = self._sql.execute(self._query)
# This isn't thread safe, but since the values should
# virtually never change, is safe in reality.
# keys = [str(key) for key in result.keys()]
entries = [{key: str(value) for (key, value)
in row.items()} for row in result]
return self._update_entries(entries)
def _read_cdr_file(self):
with open(self._cdr_file) as filp:
lines = filp.readlines()
keys = None
entries = csv.reader(lines, quotechar='"', delimiter=',',
quoting=csv.QUOTE_ALL,
skipinitialspace=True)
if entries:
length = len(entries[0])
if length < len(self._header):
keys = self._header[:length]
else:
keys = self._header
entries = [dict(zip(keys, e[:len(keys)])) for e in entries]
return self._update_entries(reversed(entries))
def _inotify_loop(self):
updated = False
trigger_events = ('IN_DELETE', 'IN_CLOSE_WRITE',
'IN_MOVED_FROM', 'IN_MOVED_TO')
try:
for event in self._inot.event_gen():
if event is not None:
(_header, type_names, _watch_path, _filename) = event
if set(type_names) & set(trigger_events):
updated = True
else:
if updated:
if self._read_cdr_file():
logging.debug(
"Reloading CDR file due to event: %s",
type_names)
self._cdr_queue.put("updated")
updated = False
finally:
self._inot.remove_watch(self._cdr_file.encode('utf-8'))
def run(self):
"""Thread main loop."""
if self._inot:
self._inotify_loop()
return
last_mtime = 0
while True:
updated = False
if self._sql:
updated = self._read_cdr_from_sql()
elif self._cdr_file:
mtime = os.path.getmtime(self._cdr_file)
if last_mtime != mtime:
updated = self._read_cdr_file()
last_mtime = mtime
if updated:
self._cdr_queue.put("updated")
time.sleep(self._sleep)
def entries(self):
"""Retrieve current CDR log."""
return self._entries
def keys(self):
"""Retrieve CDR entity keys."""
return self._keys
def count(self):
"""Retrieve number of current CDR entries."""
return len(self._entries)
def queue(self):
"""Fetch queue object."""
return self._cdr_queue
class Client:
"""asterisk_mbox client."""
def __init__(self, ipaddr, port, password, callback=None, **kwargs):
"""constructor."""
self._ipaddr = ipaddr
self._port = port
self._password = encode_password(password).encode('utf-8')
self._callback = callback
self._soc = None
self._thread = None
self._status = {}
# Stop thread
self.signal = PollableQueue()
# Send data to the server
self.request_queue = PollableQueue()
# Receive data from the server
self.result_queue = PollableQueue()
if 'autostart' not in kwargs or kwargs['autostart']:
self.start()
def start(self):
"""Start thread."""
if not self._thread:
logging.info("Starting asterisk mbox thread")
# Ensure signal queue is empty
try:
while True:
self.signal.get(False)
except queue.Empty:
pass
self._thread = threading.Thread(target=self._loop)
self._thread.setDaemon(True)
self._thread.start()
def stop(self):
"""Stop thread."""
if self._thread:
self.signal.put("Stop")
self._thread.join()
if self._soc:
self._soc.shutdown()
self._soc.close()
self._thread = None
def _connect(self):
"""Connect to server."""
self._soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._soc.connect((self._ipaddr, self._port))
self._soc.send(
_build_request({
'cmd': cmd.CMD_MESSAGE_PASSWORD,
'sha': self._password
}))
def _recv_msg(self):
"""Read a message from the server."""
command = ord(recv_blocking(self._soc, 1))
msglen = recv_blocking(self._soc, 4)
msglen = ((msglen[0] << 24) + (msglen[1] << 16) + (msglen[2] << 8) +
msglen[3])
msg = recv_blocking(self._soc, msglen)
return command, msg
def _handle_msg(self, command, msg, request):
if command == cmd.CMD_MESSAGE_ERROR:
logging.warning("Received error: %s", msg.decode('utf-8'))
elif command == cmd.CMD_MESSAGE_VERSION:
min_ver = StrictVersion(__min_server_version__)
server_ver = StrictVersion(msg.decode('utf-8'))
if server_ver < min_ver:
raise ServerError("Server version is too low: {} < {}".format(
msg.decode('utf-8'), __min_server_version__))
elif command == cmd.CMD_MESSAGE_LIST:
self._status = json.loads(msg.decode('utf-8'))
msg = self._status
elif command == cmd.CMD_MESSAGE_CDR:
self._status = json.loads(zlib.decompress(msg).decode('utf-8'))
msg = self._status
if self._callback and 'sync' not in request:
self._callback(command, msg)
elif request and (command == request.get('cmd')
or command == cmd.CMD_MESSAGE_ERROR):
logging.debug("Got command: %s", cmd.commandstr(command))
self.result_queue.put([command, msg])
request.clear()
else:
logging.debug("Got unhandled command: %s", cmd.commandstr(command))
def _clear_request(self, request):
if not self._callback or 'sync' in request:
self.result_queue.put(
[cmd.CMD_MESSAGE_ERROR, "Not connected to server"])
request.clear()
def _loop(self):
"""Handle data."""
request = {}
connected = False
while True:
timeout = None
sockets = [self.request_queue, self.signal]
if not connected:
try:
self._clear_request(request)
self._connect()
self._soc.send(
_build_request({'cmd': cmd.CMD_MESSAGE_LIST}))
self._soc.send(
_build_request({'cmd': cmd.CMD_MESSAGE_CDR_AVAILABLE}))
connected = True
except ConnectionRefusedError:
timeout = 5.0
if connected:
sockets.append(self._soc)
readable, _writable, _errored = select.select(
sockets, [], [], timeout)
if self.signal in readable:
break
if self._soc in readable:
# We have incoming data
try:
command, msg = self._recv_msg()
self._handle_msg(command, msg, request)
except (RuntimeError, ConnectionResetError):
logging.warning("Lost connection")
connected = False
self._clear_request(request)
if self.request_queue in readable:
request = self.request_queue.get()
self.request_queue.task_done()
if not connected:
self._clear_request(request)
else:
if (request['cmd'] == cmd.CMD_MESSAGE_LIST and self._status
and (not self._callback or 'sync' in request)):
self.result_queue.put(
[cmd.CMD_MESSAGE_LIST, self._status])
request = {}
else:
self._soc.send(_build_request(request))
def _queue_msg(self, item, **kwargs):
if not self._thread:
raise ServerError("Client not running")
if not self._callback or kwargs.get('sync'):
item['sync'] = True
self.request_queue.put(item)
command, msg = self.result_queue.get()
if command == cmd.CMD_MESSAGE_ERROR:
raise ServerError(msg)
return msg
else:
self.request_queue.put(item)
def messages(self, **kwargs):
"""Get list of messages with metadata."""
return self._queue_msg({'cmd': cmd.CMD_MESSAGE_LIST}, **kwargs)
def mp3(self, sha, **kwargs):
"""Get raw MP3 of a message."""
return self._queue_msg(
{
'cmd': cmd.CMD_MESSAGE_MP3,
'sha': _get_bytes(sha)
}, **kwargs)
def delete(self, sha, **kwargs):
"""Delete a message."""
return self._queue_msg(
{
'cmd': cmd.CMD_MESSAGE_DELETE,
'sha': _get_bytes(sha)
}, **kwargs)
def cdr_count(self, **kwargs):
"""Request count of CDR entries"""
return self._queue_msg({'cmd': cmd.CMD_MESSAGE_CDR_AVAILABLE},
**kwargs)
def get_cdr(self, start=0, count=-1, **kwargs):
"""Request range of CDR messages"""
sha = encode_to_sha("{:d},{:d}".format(start, count))
return self._queue_msg({
'cmd': cmd.CMD_MESSAGE_CDR,
'sha': sha
}, **kwargs)