collector_obj.save()

        else:
            print('Datacollector IP and port must be provided if not provided a collector ID.')
            exit(0)

    collector = LoraServerIOCollector(
        data_collector_id=collector_obj.id,
        organization_id=collector_obj.organization_id,
        host=collector_obj.ip,
        port=int(collector_obj.port),
        ssl=None,
        user=None,
        password=None,
        topics=topics,
        last_seen=collector_obj.last_seen,
        connected=collector_obj.connected,
        verified=collector_obj.verified)

    collector.connect()

    while (True):
        time.sleep(5)
        try:
            commit()
            self.log.debug('Commit done!')
        except Exception as exc:
            self.log.error('Error at commit:' + str(exc))
            self.log.info('Rolling back the session')
            rollback()
示例#2
0
def processData():
    # Save the packet ids that have to be processed by the selected modules
    starting_rows = list()

    if analyze:
        analyzer_row = RowProcessed.find_one_by_analyzer("packet_analyzer")
        starting_rows.append(analyzer_row.last_row)

    if bruteforce:
        bruteforcer_row = RowProcessed.find_one_by_analyzer("bruteforcer")
        starting_rows.append(bruteforcer_row.last_row)

    # Get the lowest packet ID to be processed
    first_pending_id = starting_rows[0]
    for row in starting_rows:
        if row < first_pending_id:
            first_pending_id = row

    # Jump to the next to be procesed
    first_pending_id += 1

    # If the user provided the start id, do some checksstart_packet_id = None
    if options.from_id is not None:
        start_packet_id = options.from_id
        if start_packet_id > first_pending_id:
            print(
                "Warning! You are jumping over packets that weren't processed. Last packets ID processed: "
            )
            if bruteforce:
                print("Bruteforcer: %d." % (bruteforcer_row.last_row))
            if analyze:
                print("Analyzer: %d." % (analyzer_row.last_row))
        elif start_packet_id < first_pending_id:
            print(
                "Warning! You will process twice some packets and duplicate information in DB. Last packets ID processed: "
            )
            if bruteforce:
                print("Bruteforcer: %d." % (bruteforcer_row.last_row))
            if analyze:
                print("Analyzer: %d." % (analyzer_row.last_row))
    else:
        start_packet_id = first_pending_id

    # Start processing in batches
    keep_iterating = True
    while keep_iterating:
        session_packets = None

        # Select the quantity of packets to process according to PACKES_BATCH and the limit that the user may have provided
        if options.to_id is None:

            if (start_packet_id + 2 * PACKETS_BATCH) <= Packet.rows_quantity():
                session_packets = Packet.find_all_from(start_packet_id,
                                                       PACKETS_BATCH)
                start_packet_id += PACKETS_BATCH

            else:
                logging.debug("No more packets to process. Sleeping a while")
                time.sleep(20)
                continue

        else:
            if (start_packet_id + PACKETS_BATCH) <= options.to_id:

                if (start_packet_id + PACKETS_BATCH) <= Packet.rows_quantity():
                    session_packets = Packet.find_all_from(
                        start_packet_id, PACKETS_BATCH)
                    start_packet_id += PACKETS_BATCH

                else:
                    logging.debug(
                        "No more packets to process. Sleeping a while")
                    time.sleep(20)
                    continue

            else:
                session_packets = Packet.find_all_from(
                    start_packet_id, options.to_id - start_packet_id + 1)
                start_packet_id += (options.to_id % PACKETS_BATCH)
                keep_iterating = False

        if session_packets is not None:
            for packet in session_packets:
                logging.debug("Using packet: %d" % (packet.id))
                # Skip packets from /{dev_eui}/up topic?

                try:
                    # If the starting packet wasn't given, check if the packet wasn't processed by each analyzer (except for the parser, which doesn't modify the DB)
                    if options.from_id is None:
                        if bruteforce and bruteforcer_row.last_row < packet.id:
                            LafBruteforcer.bruteForce(packet)
                            bruteforcer_row.last_row = packet.id

                        if analyze and analyzer_row.last_row < packet.id:
                            LafPacketAnalysis.processPacket(packet)
                            analyzer_row.last_row = packet.id

                    # If the starting packet was given by the user, don't do any check
                    else:
                        if bruteforce:
                            LafBruteforcer.bruteForce(packet)
                            if bruteforcer_row.last_row < packet.id:
                                bruteforcer_row.last_row = packet.id

                        if analyze:
                            LafPacketAnalysis.processPacket(packet)

                            if analyzer_row.last_row < packet.id:
                                analyzer_row.last_row = packet.id

                except Exception as e:
                    logging.error(
                        "Error processing packet {0}. Exception: {1}".format(
                            packet.id, e))
                    rollback()

                if parsedata:
                    LafPrinter.printPacket(packet)

            # Commit objects in DB before starting with the next batch
            try:
                commit()
            except Exception as exc:
                logging.error(
                    "Error trying to commit after batch processing finish: {0}"
                    .format(exc))