def threadReceiveHandler(self):
threadName = 'ReceiverThread'
log.debug('%s::started', threadName)
while True:
queues = []
for key in self._queues:
queues.append(self._queues[key])
if not queues:
time.sleep(2) # sleep for 2 seconds
continue
try:
with BrokerConnection(conf.amqpConnection) as conn:
conn.connect()
conn.ensure_connection()
log.debug('%s::Connected to %s',
threadName, conf.amqpConnection)
try:
with conn.Consumer(queues, callbacks = [
self.threadReceiverOnMessage
]) as consumer:
while True:
conn.ensure_connection()
try:
conn.drain_events(timeout = 5) # 5 seconds
except socket.timeout:
pass
self.appendNewQueues(consumer)
except Exception as E:
log.error('%s::%s', threadName, E)
conn.release()
log.debug('%s::Disconnected', threadName)
except Exception as E:
log.error('%s::%s', threadName, E)
time.sleep(30) # sleep for 30 seconds after exception
def processData(self, data):
"""
Processing of data from socket / storage.
Might be overridden in child classes
@param data: Data from socket
"""
if self._packetsFactory:
try:
if self._buffer is None:
self._buffer = b''
self._buffer += data
protocolPackets = (
self._packetsFactory.getPacketsFromBuffer(self._buffer)
)
for protocolPacket in protocolPackets:
self.processProtocolPacket(protocolPacket)
self._buffer = None
except NeedMoreDataException as E:
log.info('[%s] Need more data...', self.handlerId)
return
except Exception as E:
log.error("[%s] processData error: %s", self.handlerId, E)
log.debug('[%s] Checking handler commands', self.handlerId)
if not self.needProcessCommands():
return self
log.debug('[%s] Ok we can process commands!', self.handlerId)
self.processCommands()
return self
def getCommands(self, handler):
"""
Receives packets from the message broker.
Runs until receives packet or timeout passes
@param handler: AbstractHandler
@return: received packets
"""
content = None
try:
with BrokerConnection(conf.amqpConnection) as conn:
routing_key = conf.environment + '.mon.device.command.' + \
str(handler.uid)
log.debug('[%s] Check commands queue %s',
handler.handlerId, routing_key)
command_queue = Queue(
routing_key,
exchange = self._exchanges['mon.device'],
routing_key = routing_key)
conn.connect()
with conn.Consumer([command_queue],
callbacks = [self.onCommand]):
conn.ensure_connection()
conn.drain_events(timeout = 1)
command = self.getCommand(handler)
if command:
log.debug('[%s] We got command: %s',
handler.handlerId, command)
content = command
else:
log.debug('[%s] No commands found', handler.handlerId)
conn.release()
except Exception as E:
log.error('[%s] %s', handler.handlerId, E)
return content
def receiveImage(self, packet):
"""
Receives an image from tracker.
Sends it to the observer server, when totally received.
"""
log.error('Image receiving...')
log.info('[IS NOT IMPLEMENTED]')
def threadHandler(self):
"""
Thread handler
"""
commandRoutingKey = conf.environment + '.mon.device.command.' + \
self._protocolAlias
commandQueue = Queue(
commandRoutingKey,
exchange = broker._exchanges['mon.device'],
routing_key = commandRoutingKey
)
while True:
try:
with BrokerConnection(conf.amqpConnection) as conn:
conn.connect()
conn.ensure_connection()
log.debug('[%s] Connected to %s',
self._protocolAlias, conf.amqpConnection)
with conn.Consumer([commandQueue],
callbacks = [self.onCommand]):
while True:
conn.ensure_connection()
conn.drain_events()
conn.release()
except Exception as E:
log.error('[%s] %s', self._protocolAlias, E)
time.sleep(60) # sleep for 60 seconds after exception
def threadSignalRequestHandler(self):
threadName = 'SignalRequestThread'
signalQueueName = QUEUE_PREFIX + '.signal.request'
signalRoutingKey = QUEUE_PREFIX + '.create.#'
signalQueue = Queue(
signalQueueName,
exchange = broker._exchanges['mon.device'],
routing_key = signalRoutingKey
)
log.debug('%s::started', threadName)
while True:
try:
with BrokerConnection(conf.amqpConnection) as conn:
conn.connect()
conn.ensure_connection()
log.debug('%s::Connected to %s',
threadName, conf.amqpConnection)
try:
with conn.Consumer([signalQueue],
callbacks = [self.threadSignalRequestOnMessage]):
while True:
conn.ensure_connection()
conn.drain_events()
except Exception as E:
log.error('%s::%s', threadName, E)
conn.release()
log.debug('%s::Disconnected', threadName)
except Exception as E:
log.error('%s::%s', threadName, E)
time.sleep(10) # sleep for 10 seconds after exception
def sendImages(self, images):
"""
Sends image to the observer
@param images: dict() of binary data like {'camera1': b'....'}
"""
if not self.uid:
log.error('[%s] Cant send an image - self.uid is not defined!',
self.handlerId)
return
imagesList = []
for image in images:
image['content'] = base64.b64encode(image['content']).decode()
imagesList.append(image)
observerPacket = {
'uid': self.uid,
'time': datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f'),
'images': imagesList
}
result = self.store(observerPacket)
if result.isSuccess():
log.info('[%s] sendImages(): Images have been sent.',
self.handlerId)
else:
# ? Error messages should be converted into readable format
log.error('[%s] sendImages():\n %s',
self.handlerId, result.getErrorsList())
def threadReceiverOnMessage(self, body, message):
"""
Executes when there is a packet in signal queue
@param body: amqp message body
@param message: message instance
"""
threadName = 'ReceiverThread'
uid = 'unknown uid [!]'
try:
if isinstance(body, str):
body = json.loads(body)
if 'uid' in body:
uid = body['uid']
if 'time' not in body:
raise Exception('Incorrect packet structure')
if uid not in self._messages:
self._messages[uid] = deque()
# store message to the queue
self._messages[uid].append({
"message": message,
"body": body
})
log.debug('%s::Packet %s added %s', threadName, body['time'], uid)
self.sendMessage(uid, body)
except Exception as E:
log.error('%s::%s', threadName, E)
message.ack()
def handle(self):
try:
if HandlerClass:
log.debug('Protocol handler: %s', HandlerClass.__doc__)
self.__handler = HandlerClass(pipe.Manager(), self)
self.__handler.dispatch()
else:
log.error('No protocol handlers found!')
except Exception as E:
log.error("Dispatch error: %s", traceback.format_exc())
def sendPacketsViaBroker(self, packets):
"""
Sends data to the message broker (AMQP)
@param packets: list of packets
@return:
"""
try:
broker.send(packets)
except Exception as E:
#~print(E)
log.error(E)
def send(self, data):
"""
Sends data to a socket
@param data: data
"""
thread = self.getThread()
if thread:
sock = thread.request
sock.send(data)
else:
log.error("[%s] Handler thread is not found!", self.handlerId)
return self
def saveByUid(self, uid, data):
"""
Save protocol data into storage
@param uid: Device unqiue identifier
@param data: Protocol data
"""
log.debug('Storage::saveByUid(). %s', uid)
try:
with open(self.getStorageFileName(uid), 'ab') as f:
f.write(data)
except Exception as E:
log.error(E)
def store(self, packets):
"""
Sends a list of packets to store
@param packets: A list of packets
@return: Instance of lib.falcon.answer.FalconAnswer
"""
result = self.getStore().send(packets)
if result.isSuccess():
log.debug('[%s] store() ... OK', self.handlerId)
else:
errorsList = result.getErrorsList()
log.error('[%s] store():\n %s', self.handlerId, errorsList)
return result
def processCommands(self):
"""
Processing AMQP commands for current device
"""
try:
if not self._commandsFactory:
raise Exception("_commandsFactory is not defined!")
commands = broker.getCommands(self)
if commands:
log.debug("[%s] Received commands are: %s",
self.handlerId, commands)
self.processCommand(commands)
except Exception as E:
log.error('[%s] %s', self.handlerId, E)
def processProtocolPacket(self, protocolPacket):
"""
Process teltonika packet.
@type protocolPacket: packets.Packet
@param protocolPacket: Teltonika protocol packet
"""
if not self.__headPacketRawData:
self.__headPacketRawData = b''
if isinstance(protocolPacket, packets.PacketHead):
log.info('HeadPack is stored.')
self.__headPacketRawData = protocolPacket.rawData
self.uid = protocolPacket.deviceImei
if not self.uid:
return log.error('HeadPack is not found!')
# try to configure this tracker
if self.configure():
return
# sends the acknowledgment
self.sendAcknowledgement(protocolPacket)
if isinstance(protocolPacket, packets.PacketHead):
return
observerPackets = self.translate(protocolPacket)
if len(observerPackets) == 0:
log.info('Location packet not found. Exiting...')
return
log.info(observerPackets)
self.store(observerPackets)
def configure(self):
current_db = db.get(self.uid)
if not current_db.has('config'):
return False
data = current_db.get('config')
self.send(data)
log.debug('Configuration data sent = %s', data)
config = packets.TeltonikaConfiguration(data)
answer = b''
try:
log.debug('Waiting for the answer from device...')
answer = self.recv()
except Exception as E:
log.error(E)
current_db.remove('config')
return config.isCorrectAnswer(answer)
def loadByUid(self, uid):
"""
Returns saved protocol data by specified uid
@param uid: Device unqiue identifier
@return (str) Storage file data
"""
log.debug('Storage::loadByUid(). %s', uid)
data = b''
try:
storageFileName = self.getStorageFileName(uid)
if (os.path.isfile(storageFileName)):
with open(storageFileName, 'rb') as f:
data = f.read()
except Exception as E:
log.error(E)
return data
def threadSignalRequestOnMessage(self, body, message):
"""
Executes when there is a packet in signal queue
@param body: amqp message body
@param message: message instance
"""
threadName = 'SignalRequestThread'
uid = 'unknown uid [!]'
try:
if 'uid' in body:
uid = body['uid']
log.debug('%s:: > Signal for %s', threadName, uid)
if uid:
self._receiveManager.checkListeningForQueue(uid)
except Exception as E:
log.error('%s::%s', threadName, E)
message.ack()
def getEmails(self):
"""
Retrieves emails from IMAP connection
@return: List of email bodies
"""
list = []
host = conf.get("imap", "host")
port = conf.get("imap", "port")
username = conf.get("imap", "username")
password = conf.get("imap", "password")
filter = conf.get("imap", "filter")
if not host:
log.error('IMAP / No host specified! Exiting')
return list
log.info('IMAP / Connecting to %s:%s', host, port)
M = imaplib.IMAP4_SSL(host, port)
M.login(username, password)
log.debug("IMAP / Logged in as %s", username)
try:
M.select('INBOX', readonly=False) # select INBOX mailbox
res, data = M.uid('search', filter or '(unseen)')
nums = data[0].split()
if len(nums) > 0:
log.info('IMAP / There are %s new message(s)', len(nums))
else:
log.info('IMAP / There are no new messages')
for num in nums:
res, data = M.uid('fetch', num, '(RFC822)')
raw_email = data[0][1]
msg = email.message_from_bytes(raw_email)
list.append(msg)
finally:
try:
M.close()
except:
pass
M.logout()
return list
def delete(self, item, port, timestamp):
"""
Removes item from storage, puts in trash
@param item: Item we want to remove
@param port: Device port
@param timestamp: Start timestamp
"""
try:
uidName = item['name']
filename = os.path.join(conf.pathStorage, port, uidName)
newName = os.path.join(conf.pathTrash, timestamp, port, uidName)
log.info('Delete data for %s', uidName)
log.info('fileName = %s, newName = %s', filename, newName)
newDir = os.path.dirname(newName)
if not os.path.exists(newDir):
os.makedirs(newDir)
os.rename(filename, newName)
except Exception as E:
log.error(E)
def threadSignalResponseOnMessage(self, body, message):
"""
Executes when there is an answer in signal queue
@param body: amqp message body
@param message: message instance
"""
threadName = 'SignalResponseThread'
uid = 'unknown uid [!]'
try:
if isinstance(body, str):
body = json.loads(body)
if 'uid' in body:
uid = body['uid']
log.debug('%s:: < Signal for %s', threadName, uid)
if uid:
self._receiveManager.checkListeningForQueue(uid)
self._receiveManager.messageReceived(uid)
except Exception as E:
log.error('%s::%s', threadName, E)
message.ack()
def send(self, obj):
"""
Sending data to the controller receiving packets from the devices
"""
result = lib.falcon.FalconAnswer()
try:
packets = list()
if (isinstance(obj, list)):
# if multiple packets
packets = obj
elif (isinstance(obj, dict)):
# if one packet
packets.append(obj)
else:
return result
# Let's send packets to AMQP broker
self.sendPacketsViaBroker(packets)
except Exception as E:
result.error('500', ['Error sending packets: ' + str(E)])
log.error(E)
return result
def processCommand(self, command):
"""
Processing AMQP command
@param command: command
"""
if not command:
log.error("[%s] Empty command description!", self.handlerId)
return
if (not self.uid) and ('uid' in command):
self.uid = command['uid']
log.debug("[%s] Processing AMQP command: %s ", self.handlerId, command)
try:
if not self._commandsFactory:
raise Exception("_commandsFactory is not defined!")
commandName = command["command"]
commandInstance = self._commandsFactory.getInstance(command)
if commandInstance:
log.debug("[%s] Command class is %s",
self.handlerId, commandInstance.__class__)
self.sendCommand(commandInstance, command)
else:
broker.sendAmqpError(self, "Command is not supported")
log.error("[%s] No command with name %s",
self.handlerId, commandName)
except Exception as E:
log.error("[%s] Send command error is %s", self.handlerId, E)
def receiveImage(self, packet):
"""
Receives an image from tracker.
Sends it to the observer server, when totally received.
"""
if (packet == None) or (packet.body == None) or (len(packet.body) == 0):
log.error('Empty image packet. Transfer aborted!')
return
config = self.__imageReceivingConfig
partnum = packet.body[0]
if self.__imageReceivingConfig is None:
self.__imageReceivingConfig = {
'imageParts': {}
}
config = self.__imageReceivingConfig
log.info('Image transfer is started.')
else:
if len(packet.body) > 1:
log.debug('Image transfer in progress...')
log.debug('Size of chunk is %d bytes', len(packet.body) - 1)
else:
imageData = b''
imageParts = self.__imageReceivingConfig['imageParts']
for num in sorted(imageParts.keys()):
imageData += imageParts[num]
self.sendImages([{
'mime': 'image/jpeg',
'content': imageData
}])
self.__imageReceivingConfig = None
log.debug('Transfer complete.')
return
imageData = packet.body[1:]
config['imageParts'][partnum] = imageData
def processError(self, data):
"""
OK. Our pattern doesn't match the socket or config data.
The source of the problem can be in wrong report format.
"""
log.error("Unknown data format...")
def handle(self):
try:
self.handler.dispatch()
except Exception:
log.error("Dispatch error: %s", traceback.format_exc())
def send(self, packets, routing_key = None, exchangeName = None):
"""
Sends packets to the message broker
@param packets: list of dict
@param routing_key: str
@param exchangeName: str
"""
exchange = self._exchanges['mon.device']
if (exchangeName is not None) and (exchangeName in self._exchanges):
exchange = self._exchanges[exchangeName]
try:
with BrokerConnection(conf.amqpConnection) as conn:
log.debug('BROKER: Connect to %s' % conf.amqpConnection)
conn.connect()
connChannel = conn.channel()
queuesConfig = {}
# spike-nail START
timePrev = None
# spike-nail END
reUid = re.compile('[\w-]+')
for packet in packets:
uid = None if 'uid' not in packet else packet['uid']
# we should check uid for correctness
uidIsCorrect = uid is not None and reUid.match(uid)
timeCurr = None
if 'time' in packet:
timeCurr = packet['time']
if uidIsCorrect and uid in queuesConfig:
config = queuesConfig[uid]
else:
routingKey = routing_key
if not routing_key:
if not uidIsCorrect: continue # skip incorrect uid
routingKey = self.getRoutingKey(uid)
routingKey = conf.environment + '.' + routingKey
config = {
'routingKey': routingKey,
'queue': Queue(
routingKey,
exchange = exchange,
routing_key = routingKey
)
}
if uidIsCorrect:
queuesConfig[uid] = config
# spike-nail START
if timePrev and timeCurr:
fmtDate = "%Y-%m-%dT%H:%M:%S.%f"
t1 = datetime.strptime(timeCurr, fmtDate)
t2 = datetime.strptime(timePrev, fmtDate)
if t2 > t1:
tt = t1
t1 = t2
t2 = tt
if (t1 - t2).seconds < 10:
log.debug('Skip packet with time = ' + timeCurr)
continue # skip this packet if it's too close
# spike-nail END
with conn.Producer(channel = connChannel) as producer:
conn.ensure_connection()
producer.publish(
packet,
exchange = exchange,
routing_key = config['routingKey'],
declare = [config['queue']],
retry = True
)
if uid:
msg = 'Packet for "%s" is sent. ' % uid
if 'time' in packet:
msg += 'packet[\'time\'] = ' + timeCurr
# spike-nail START
timePrev = packet['time']
# spike-nail END
log.debug(msg)
else:
log.debug('Message is sent via message broker')
conn.release()
except Exception as E:
log.error('Error during packet send: %s', E)
log.debug('BROKER: Disconnected')
sock.settimeout(TIMEOUT_SECONDS)
try:
sock.connect((host, port))
try:
sock.send(data)
except Exception as E:
# In case of error during data sending
# try to send it again
sock = socket.socket(
socket.AF_INET,
socket.SOCK_STREAM
)
sock.connect((host, port))
sock.send(data)
try:
# if there is some data from server,
# let's receive it
while sock.recv(4096): pass
except:
pass
except Exception as E:
log.error(E)
finally:
sock.close()
except Exception as E:
log.error(E)
except Exception as E:
log.error(E)
# -*- coding: utf8 -*-
"""
@project Maprox <http://www.maprox.net>
@info Protocol handlers list
@copyright 2009-2013, Maprox LLC
"""
from kernel.logger import log
from kernel.config import conf
# Load modules
HandlerClass = None
handlerName = conf.get("settings", "handler")
handlerClassPath = "lib.handlers." + handlerName
try:
pkg = __import__(handlerClassPath, globals(), locals(), ["Handler"])
if hasattr(pkg, "Handler"):
HandlerClass = getattr(pkg, "Handler")
HandlerClass.initAmqpThread(handlerName)
log.info("Protocol is loaded: " + HandlerClass.__doc__)
else:
log.error("Class 'Handler' in not found in module %s", handlerClassPath)
except Exception as E:
log.error("Protocol '%s' loading error: %s", handlerClassPath, E)
