def threadReceiveHandler(self): threadName = 'ReceiverThread' log.debug('%s::started', threadName) while True: queues = [] for key in self._queues: queues.append(self._queues[key]) if not queues: time.sleep(2) # sleep for 2 seconds continue try: with BrokerConnection(conf.amqpConnection) as conn: conn.connect() conn.ensure_connection() log.debug('%s::Connected to %s', threadName, conf.amqpConnection) try: with conn.Consumer(queues, callbacks = [ self.threadReceiverOnMessage ]) as consumer: while True: conn.ensure_connection() try: conn.drain_events(timeout = 5) # 5 seconds except socket.timeout: pass self.appendNewQueues(consumer) except Exception as E: log.error('%s::%s', threadName, E) conn.release() log.debug('%s::Disconnected', threadName) except Exception as E: log.error('%s::%s', threadName, E) time.sleep(30) # sleep for 30 seconds after exception
def processData(self, data): """ Processing of data from socket / storage. Might be overridden in child classes @param data: Data from socket """ if self._packetsFactory: try: if self._buffer is None: self._buffer = b'' self._buffer += data protocolPackets = ( self._packetsFactory.getPacketsFromBuffer(self._buffer) ) for protocolPacket in protocolPackets: self.processProtocolPacket(protocolPacket) self._buffer = None except NeedMoreDataException as E: log.info('[%s] Need more data...', self.handlerId) return except Exception as E: log.error("[%s] processData error: %s", self.handlerId, E) log.debug('[%s] Checking handler commands', self.handlerId) if not self.needProcessCommands(): return self log.debug('[%s] Ok we can process commands!', self.handlerId) self.processCommands() return self
def getCommands(self, handler): """ Receives packets from the message broker. Runs until receives packet or timeout passes @param handler: AbstractHandler @return: received packets """ content = None try: with BrokerConnection(conf.amqpConnection) as conn: routing_key = conf.environment + '.mon.device.command.' + \ str(handler.uid) log.debug('[%s] Check commands queue %s', handler.handlerId, routing_key) command_queue = Queue( routing_key, exchange = self._exchanges['mon.device'], routing_key = routing_key) conn.connect() with conn.Consumer([command_queue], callbacks = [self.onCommand]): conn.ensure_connection() conn.drain_events(timeout = 1) command = self.getCommand(handler) if command: log.debug('[%s] We got command: %s', handler.handlerId, command) content = command else: log.debug('[%s] No commands found', handler.handlerId) conn.release() except Exception as E: log.error('[%s] %s', handler.handlerId, E) return content
def receiveImage(self, packet): """ Receives an image from tracker. Sends it to the observer server, when totally received. """ log.error('Image receiving...') log.info('[IS NOT IMPLEMENTED]')
def threadHandler(self): """ Thread handler """ commandRoutingKey = conf.environment + '.mon.device.command.' + \ self._protocolAlias commandQueue = Queue( commandRoutingKey, exchange = broker._exchanges['mon.device'], routing_key = commandRoutingKey ) while True: try: with BrokerConnection(conf.amqpConnection) as conn: conn.connect() conn.ensure_connection() log.debug('[%s] Connected to %s', self._protocolAlias, conf.amqpConnection) with conn.Consumer([commandQueue], callbacks = [self.onCommand]): while True: conn.ensure_connection() conn.drain_events() conn.release() except Exception as E: log.error('[%s] %s', self._protocolAlias, E) time.sleep(60) # sleep for 60 seconds after exception
def threadSignalRequestHandler(self): threadName = 'SignalRequestThread' signalQueueName = QUEUE_PREFIX + '.signal.request' signalRoutingKey = QUEUE_PREFIX + '.create.#' signalQueue = Queue( signalQueueName, exchange = broker._exchanges['mon.device'], routing_key = signalRoutingKey ) log.debug('%s::started', threadName) while True: try: with BrokerConnection(conf.amqpConnection) as conn: conn.connect() conn.ensure_connection() log.debug('%s::Connected to %s', threadName, conf.amqpConnection) try: with conn.Consumer([signalQueue], callbacks = [self.threadSignalRequestOnMessage]): while True: conn.ensure_connection() conn.drain_events() except Exception as E: log.error('%s::%s', threadName, E) conn.release() log.debug('%s::Disconnected', threadName) except Exception as E: log.error('%s::%s', threadName, E) time.sleep(10) # sleep for 10 seconds after exception
def sendImages(self, images): """ Sends image to the observer @param images: dict() of binary data like {'camera1': b'....'} """ if not self.uid: log.error('[%s] Cant send an image - self.uid is not defined!', self.handlerId) return imagesList = [] for image in images: image['content'] = base64.b64encode(image['content']).decode() imagesList.append(image) observerPacket = { 'uid': self.uid, 'time': datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f'), 'images': imagesList } result = self.store(observerPacket) if result.isSuccess(): log.info('[%s] sendImages(): Images have been sent.', self.handlerId) else: # ? Error messages should be converted into readable format log.error('[%s] sendImages():\n %s', self.handlerId, result.getErrorsList())
def threadReceiverOnMessage(self, body, message): """ Executes when there is a packet in signal queue @param body: amqp message body @param message: message instance """ threadName = 'ReceiverThread' uid = 'unknown uid [!]' try: if isinstance(body, str): body = json.loads(body) if 'uid' in body: uid = body['uid'] if 'time' not in body: raise Exception('Incorrect packet structure') if uid not in self._messages: self._messages[uid] = deque() # store message to the queue self._messages[uid].append({ "message": message, "body": body }) log.debug('%s::Packet %s added %s', threadName, body['time'], uid) self.sendMessage(uid, body) except Exception as E: log.error('%s::%s', threadName, E) message.ack()
def handle(self): try: if HandlerClass: log.debug('Protocol handler: %s', HandlerClass.__doc__) self.__handler = HandlerClass(pipe.Manager(), self) self.__handler.dispatch() else: log.error('No protocol handlers found!') except Exception as E: log.error("Dispatch error: %s", traceback.format_exc())
def sendPacketsViaBroker(self, packets): """ Sends data to the message broker (AMQP) @param packets: list of packets @return: """ try: broker.send(packets) except Exception as E: #~print(E) log.error(E)
def send(self, data): """ Sends data to a socket @param data: data """ thread = self.getThread() if thread: sock = thread.request sock.send(data) else: log.error("[%s] Handler thread is not found!", self.handlerId) return self
def saveByUid(self, uid, data): """ Save protocol data into storage @param uid: Device unqiue identifier @param data: Protocol data """ log.debug('Storage::saveByUid(). %s', uid) try: with open(self.getStorageFileName(uid), 'ab') as f: f.write(data) except Exception as E: log.error(E)
def store(self, packets): """ Sends a list of packets to store @param packets: A list of packets @return: Instance of lib.falcon.answer.FalconAnswer """ result = self.getStore().send(packets) if result.isSuccess(): log.debug('[%s] store() ... OK', self.handlerId) else: errorsList = result.getErrorsList() log.error('[%s] store():\n %s', self.handlerId, errorsList) return result
def processCommands(self): """ Processing AMQP commands for current device """ try: if not self._commandsFactory: raise Exception("_commandsFactory is not defined!") commands = broker.getCommands(self) if commands: log.debug("[%s] Received commands are: %s", self.handlerId, commands) self.processCommand(commands) except Exception as E: log.error('[%s] %s', self.handlerId, E)
def processProtocolPacket(self, protocolPacket): """ Process teltonika packet. @type protocolPacket: packets.Packet @param protocolPacket: Teltonika protocol packet """ if not self.__headPacketRawData: self.__headPacketRawData = b'' if isinstance(protocolPacket, packets.PacketHead): log.info('HeadPack is stored.') self.__headPacketRawData = protocolPacket.rawData self.uid = protocolPacket.deviceImei if not self.uid: return log.error('HeadPack is not found!') # try to configure this tracker if self.configure(): return # sends the acknowledgment self.sendAcknowledgement(protocolPacket) if isinstance(protocolPacket, packets.PacketHead): return observerPackets = self.translate(protocolPacket) if len(observerPackets) == 0: log.info('Location packet not found. Exiting...') return log.info(observerPackets) self.store(observerPackets)
def configure(self): current_db = db.get(self.uid) if not current_db.has('config'): return False data = current_db.get('config') self.send(data) log.debug('Configuration data sent = %s', data) config = packets.TeltonikaConfiguration(data) answer = b'' try: log.debug('Waiting for the answer from device...') answer = self.recv() except Exception as E: log.error(E) current_db.remove('config') return config.isCorrectAnswer(answer)
def loadByUid(self, uid): """ Returns saved protocol data by specified uid @param uid: Device unqiue identifier @return (str) Storage file data """ log.debug('Storage::loadByUid(). %s', uid) data = b'' try: storageFileName = self.getStorageFileName(uid) if (os.path.isfile(storageFileName)): with open(storageFileName, 'rb') as f: data = f.read() except Exception as E: log.error(E) return data
def threadSignalRequestOnMessage(self, body, message): """ Executes when there is a packet in signal queue @param body: amqp message body @param message: message instance """ threadName = 'SignalRequestThread' uid = 'unknown uid [!]' try: if 'uid' in body: uid = body['uid'] log.debug('%s:: > Signal for %s', threadName, uid) if uid: self._receiveManager.checkListeningForQueue(uid) except Exception as E: log.error('%s::%s', threadName, E) message.ack()
def getEmails(self): """ Retrieves emails from IMAP connection @return: List of email bodies """ list = [] host = conf.get("imap", "host") port = conf.get("imap", "port") username = conf.get("imap", "username") password = conf.get("imap", "password") filter = conf.get("imap", "filter") if not host: log.error('IMAP / No host specified! Exiting') return list log.info('IMAP / Connecting to %s:%s', host, port) M = imaplib.IMAP4_SSL(host, port) M.login(username, password) log.debug("IMAP / Logged in as %s", username) try: M.select('INBOX', readonly=False) # select INBOX mailbox res, data = M.uid('search', filter or '(unseen)') nums = data[0].split() if len(nums) > 0: log.info('IMAP / There are %s new message(s)', len(nums)) else: log.info('IMAP / There are no new messages') for num in nums: res, data = M.uid('fetch', num, '(RFC822)') raw_email = data[0][1] msg = email.message_from_bytes(raw_email) list.append(msg) finally: try: M.close() except: pass M.logout() return list
def delete(self, item, port, timestamp): """ Removes item from storage, puts in trash @param item: Item we want to remove @param port: Device port @param timestamp: Start timestamp """ try: uidName = item['name'] filename = os.path.join(conf.pathStorage, port, uidName) newName = os.path.join(conf.pathTrash, timestamp, port, uidName) log.info('Delete data for %s', uidName) log.info('fileName = %s, newName = %s', filename, newName) newDir = os.path.dirname(newName) if not os.path.exists(newDir): os.makedirs(newDir) os.rename(filename, newName) except Exception as E: log.error(E)
def threadSignalResponseOnMessage(self, body, message): """ Executes when there is an answer in signal queue @param body: amqp message body @param message: message instance """ threadName = 'SignalResponseThread' uid = 'unknown uid [!]' try: if isinstance(body, str): body = json.loads(body) if 'uid' in body: uid = body['uid'] log.debug('%s:: < Signal for %s', threadName, uid) if uid: self._receiveManager.checkListeningForQueue(uid) self._receiveManager.messageReceived(uid) except Exception as E: log.error('%s::%s', threadName, E) message.ack()
def send(self, obj): """ Sending data to the controller receiving packets from the devices """ result = lib.falcon.FalconAnswer() try: packets = list() if (isinstance(obj, list)): # if multiple packets packets = obj elif (isinstance(obj, dict)): # if one packet packets.append(obj) else: return result # Let's send packets to AMQP broker self.sendPacketsViaBroker(packets) except Exception as E: result.error('500', ['Error sending packets: ' + str(E)]) log.error(E) return result
def processCommand(self, command): """ Processing AMQP command @param command: command """ if not command: log.error("[%s] Empty command description!", self.handlerId) return if (not self.uid) and ('uid' in command): self.uid = command['uid'] log.debug("[%s] Processing AMQP command: %s ", self.handlerId, command) try: if not self._commandsFactory: raise Exception("_commandsFactory is not defined!") commandName = command["command"] commandInstance = self._commandsFactory.getInstance(command) if commandInstance: log.debug("[%s] Command class is %s", self.handlerId, commandInstance.__class__) self.sendCommand(commandInstance, command) else: broker.sendAmqpError(self, "Command is not supported") log.error("[%s] No command with name %s", self.handlerId, commandName) except Exception as E: log.error("[%s] Send command error is %s", self.handlerId, E)
def receiveImage(self, packet): """ Receives an image from tracker. Sends it to the observer server, when totally received. """ if (packet == None) or (packet.body == None) or (len(packet.body) == 0): log.error('Empty image packet. Transfer aborted!') return config = self.__imageReceivingConfig partnum = packet.body[0] if self.__imageReceivingConfig is None: self.__imageReceivingConfig = { 'imageParts': {} } config = self.__imageReceivingConfig log.info('Image transfer is started.') else: if len(packet.body) > 1: log.debug('Image transfer in progress...') log.debug('Size of chunk is %d bytes', len(packet.body) - 1) else: imageData = b'' imageParts = self.__imageReceivingConfig['imageParts'] for num in sorted(imageParts.keys()): imageData += imageParts[num] self.sendImages([{ 'mime': 'image/jpeg', 'content': imageData }]) self.__imageReceivingConfig = None log.debug('Transfer complete.') return imageData = packet.body[1:] config['imageParts'][partnum] = imageData
def processError(self, data): """ OK. Our pattern doesn't match the socket or config data. The source of the problem can be in wrong report format. """ log.error("Unknown data format...")
def handle(self): try: self.handler.dispatch() except Exception: log.error("Dispatch error: %s", traceback.format_exc())
def send(self, packets, routing_key = None, exchangeName = None): """ Sends packets to the message broker @param packets: list of dict @param routing_key: str @param exchangeName: str """ exchange = self._exchanges['mon.device'] if (exchangeName is not None) and (exchangeName in self._exchanges): exchange = self._exchanges[exchangeName] try: with BrokerConnection(conf.amqpConnection) as conn: log.debug('BROKER: Connect to %s' % conf.amqpConnection) conn.connect() connChannel = conn.channel() queuesConfig = {} # spike-nail START timePrev = None # spike-nail END reUid = re.compile('[\w-]+') for packet in packets: uid = None if 'uid' not in packet else packet['uid'] # we should check uid for correctness uidIsCorrect = uid is not None and reUid.match(uid) timeCurr = None if 'time' in packet: timeCurr = packet['time'] if uidIsCorrect and uid in queuesConfig: config = queuesConfig[uid] else: routingKey = routing_key if not routing_key: if not uidIsCorrect: continue # skip incorrect uid routingKey = self.getRoutingKey(uid) routingKey = conf.environment + '.' + routingKey config = { 'routingKey': routingKey, 'queue': Queue( routingKey, exchange = exchange, routing_key = routingKey ) } if uidIsCorrect: queuesConfig[uid] = config # spike-nail START if timePrev and timeCurr: fmtDate = "%Y-%m-%dT%H:%M:%S.%f" t1 = datetime.strptime(timeCurr, fmtDate) t2 = datetime.strptime(timePrev, fmtDate) if t2 > t1: tt = t1 t1 = t2 t2 = tt if (t1 - t2).seconds < 10: log.debug('Skip packet with time = ' + timeCurr) continue # skip this packet if it's too close # spike-nail END with conn.Producer(channel = connChannel) as producer: conn.ensure_connection() producer.publish( packet, exchange = exchange, routing_key = config['routingKey'], declare = [config['queue']], retry = True ) if uid: msg = 'Packet for "%s" is sent. ' % uid if 'time' in packet: msg += 'packet[\'time\'] = ' + timeCurr # spike-nail START timePrev = packet['time'] # spike-nail END log.debug(msg) else: log.debug('Message is sent via message broker') conn.release() except Exception as E: log.error('Error during packet send: %s', E) log.debug('BROKER: Disconnected')
sock.settimeout(TIMEOUT_SECONDS) try: sock.connect((host, port)) try: sock.send(data) except Exception as E: # In case of error during data sending # try to send it again sock = socket.socket( socket.AF_INET, socket.SOCK_STREAM ) sock.connect((host, port)) sock.send(data) try: # if there is some data from server, # let's receive it while sock.recv(4096): pass except: pass except Exception as E: log.error(E) finally: sock.close() except Exception as E: log.error(E) except Exception as E: log.error(E)
# -*- coding: utf8 -*- """ @project Maprox <http://www.maprox.net> @info Protocol handlers list @copyright 2009-2013, Maprox LLC """ from kernel.logger import log from kernel.config import conf # Load modules HandlerClass = None handlerName = conf.get("settings", "handler") handlerClassPath = "lib.handlers." + handlerName try: pkg = __import__(handlerClassPath, globals(), locals(), ["Handler"]) if hasattr(pkg, "Handler"): HandlerClass = getattr(pkg, "Handler") HandlerClass.initAmqpThread(handlerName) log.info("Protocol is loaded: " + HandlerClass.__doc__) else: log.error("Class 'Handler' in not found in module %s", handlerClassPath) except Exception as E: log.error("Protocol '%s' loading error: %s", handlerClassPath, E)