Esempio n. 1
0
    def merge(self,
              data,
              peer_ip,
              c,
              size_bypass=False,
              wait=False,
              revert=False):
        """
        Checks and merge the tx list in out mempool
        :param data:
        :param peer_ip:
        :param c:
        :param size_bypass: if True, will merge whatever the mempool size is
        :param wait: if True, will wait until the main db_lock is free. if False, will just drop.
        :param revert: if True, we are reverting tx from digest_block, so main lock is on. Don't bother, process without lock.
        :return:
        """
        global REFUSE_OLDER_THAN
        # Easy cases of empty or invalid data
        if not data:
            return "Mempool from {} was empty".format(peer_ip)
        mempool_result = []
        if data == '*':
            raise ValueError("Connection lost")
        try:
            if self.peers_sent[peer_ip] > time.time(
            ) and peer_ip != '127.0.0.1':
                self.app_log.warning(
                    "Mempool ignoring merge from frozen {}".format(peer_ip))
                mempool_result.append(
                    "Mempool ignoring merge from frozen {}".format(peer_ip))
                return mempool_result
        except:
            # unknown peer
            pass
        if not essentials.is_sequence(data):
            if peer_ip != '127.0.0.1':
                with self.peers_lock:
                    self.peers_sent[peer_ip] = time.time() + 10 * 60
                self.app_log.warning(
                    "Freezing mempool from {} for 10 min - Bad TX format".
                    format(peer_ip))
            mempool_result.append("Bad TX Format")
            return mempool_result

        if not revert:
            while self.db_lock.locked():
                # prevent transactions which are just being digested from being added to mempool
                if not wait:
                    # not reverting, but not waiting, bye
                    # By default, we don't wait.
                    mempool_result.append("Locked ledger, dropping txs")
                    return mempool_result
                self.app_log.warning(
                    "Waiting for block digestion to finish before merging mempool"
                )
                time.sleep(1)
        # if reverting, don't bother with main lock, go on.

        # Let's really dig
        mempool_result.append(
            "Mempool merging started from {}".format(peer_ip))
        # Single time reference here for the whole merge.
        time_now = time.time()
        # calculate current mempool size before adding txs
        mempool_size = self.size()

        # TODO: we check main ledger db is not locked before beginning, but we don't lock? ok, see comment in node.py. since it's called from a lock, it would deadlock.
        # merge mempool
        # while self.lock.locked():
        #    time.sleep(1)
        with self.lock:
            try:
                block_list = data
                if not isinstance(
                        block_list[0], list
                ):  # convert to list of lists if only one tx and not handled
                    block_list = [block_list]

                for transaction in block_list:

                    if size_bypass or self.space_left_for_tx(
                            transaction, mempool_size):
                        # all transactions in the mempool need to be cycled to check for special cases,
                        # therefore no while/break loop here
                        mempool_timestamp = '%.2f' % (quantize_two(
                            transaction[0]))
                        mempool_timestamp_float = float(
                            transaction[0])  # limit Decimal where not needed
                        mempool_address = str(transaction[1])[:56]
                        mempool_recipient = str(transaction[2])[:56]
                        mempool_amount = '%.8f' % (quantize_eight(
                            transaction[3]))  # convert scientific notation
                        mempool_amount_float = float(transaction[3])
                        mempool_signature_enc = str(transaction[4])[:684]
                        mempool_public_key_hashed = str(transaction[5])[:1068]
                        if "b'" == mempool_public_key_hashed[:2]:
                            mempool_public_key_hashed = transaction[5][2:1070]
                        mempool_operation = str(transaction[6])[:30]
                        mempool_openfield = str(transaction[7])[:100000]

                        # Begin with the easy tests that do not require cpu or disk access
                        if mempool_amount_float < 0:
                            mempool_result.append(
                                "Mempool: Negative balance spend attempt")
                            continue
                        if not essentials.address_validate(mempool_address):
                            mempool_result.append(
                                "Mempool: Invalid address {}".format(
                                    mempool_address))
                            continue
                        if not essentials.address_validate(mempool_recipient):
                            mempool_result.append(
                                "Mempool: Invalid recipient {}".format(
                                    mempool_recipient))
                            continue
                        if mempool_timestamp_float > time_now:
                            mempool_result.append(
                                "Mempool: Future transaction rejected {}s".
                                format(mempool_timestamp_float - time_now))
                            continue
                        if mempool_timestamp_float < time_now - REFUSE_OLDER_THAN:
                            # don't accept old txs, mempool needs to be harsher than ledger
                            mempool_result.append(
                                "Mempool: Too old a transaction")
                            continue

                        # Then more cpu heavy tests
                        hashed_address = hashlib.sha224(
                            base64.b64decode(
                                mempool_public_key_hashed)).hexdigest()
                        if mempool_address != hashed_address:
                            mempool_result.append(
                                "Mempool: Attempt to spend from a wrong address {} instead of {}"
                                .format(mempool_address, hashed_address))
                            continue
                        # Crypto tests - more cpu hungry
                        try:
                            essentials.validate_pem(mempool_public_key_hashed)
                        except ValueError as e:
                            mempool_result.append(
                                "Mempool: Public key does not validate: {}".
                                format(e))
                        # recheck sig
                        try:
                            mempool_public_key = RSA.importKey(
                                base64.b64decode(mempool_public_key_hashed))
                            mempool_signature_dec = base64.b64decode(
                                mempool_signature_enc)
                            verifier = PKCS1_v1_5.new(mempool_public_key)
                            tx_signed = (mempool_timestamp, mempool_address,
                                         mempool_recipient, mempool_amount,
                                         mempool_operation, mempool_openfield)
                            my_hash = SHA.new(str(tx_signed).encode("utf-8"))
                            if not verifier.verify(my_hash,
                                                   mempool_signature_dec):
                                mempool_result.append(
                                    "Mempool: Wrong signature ({}) for data {} in mempool insert attempt"
                                    .format(mempool_signature_enc, tx_signed))
                                continue
                        except Exception as e:
                            mempool_result.append(
                                "Mempool: Unexpected error checking sig: {}".
                                format(e))
                            continue

                        # Only now, process the tests requiring db access
                        mempool_in = self.sig_check(mempool_signature_enc)

                        # Temp: get last block for HF reason
                        essentials.execute_param_c(
                            c,
                            "SELECT block_height FROM transactions WHERE 1 ORDER by block_height DESC limit ?",
                            (1, ), self.app_log)
                        last_block = c.fetchone()[0]
                        # reject transactions which are already in the ledger
                        # TODO: not clean, will need to have ledger as a module too.
                        essentials.execute_param_c(
                            c,
                            "SELECT timestamp FROM transactions WHERE signature = ?",
                            (mempool_signature_enc, ), self.app_log)
                        ledger_in = bool(c.fetchone())
                        # remove from mempool if it's in both ledger and mempool already
                        if mempool_in and ledger_in:
                            try:
                                # Do not lock, we already have the lock for the whole merge.
                                self.execute(SQL_DELETE_TX,
                                             (mempool_signature_enc, ))
                                self.commit()
                                mempool_result.append(
                                    "Mempool: Transaction deleted from our mempool"
                                )
                            except:  # experimental try and except
                                mempool_result.append(
                                    "Mempool: Transaction was not present in the pool anymore"
                                )
                            continue
                        if ledger_in:
                            mempool_result.append(
                                "That transaction is already in our ledger")
                            # Can be a syncing node. Do not request mempool from this peer until 10 min
                            if peer_ip != '127.0.0.1':
                                with self.peers_lock:
                                    self.peers_sent[peer_ip] = time.time(
                                    ) + 10 * 60
                                self.app_log.warning(
                                    "Freezing mempool from {} for 10 min.".
                                    format(peer_ip))
                            # Here, we point blank stop processing the batch from this host since it's outdated.
                            # Update: Do not, since it blocks further valid tx - case has been found in real use.
                            # return mempool_result
                            continue
                        # Already there, just ignore then
                        if mempool_in:
                            mempool_result.append(
                                "That transaction is already in our mempool")
                            continue

                        # Here we covered the basics, the current tx is conform and signed. Now let's check balance.

                        # verify balance
                        mempool_result.append(
                            "Mempool: Received address: {}".format(
                                mempool_address))
                        # include mempool fees
                        result = self.fetchall(
                            "SELECT amount, openfield, operation FROM transactions WHERE address = ?",
                            (mempool_address, ))
                        debit_mempool = 0
                        if result:
                            for x in result:
                                debit_tx = quantize_eight(x[0])
                                fee = quantize_eight(
                                    essentials.fee_calculate(
                                        x[1], x[2], last_block))
                                debit_mempool = quantize_eight(debit_mempool +
                                                               debit_tx + fee)

                        credit = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT amount FROM transactions WHERE recipient = ?",
                            (mempool_address, ), self.app_log):
                            credit = quantize_eight(credit) + quantize_eight(
                                entry[0])

                        debit_ledger = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT amount FROM transactions WHERE address = ?",
                            (mempool_address, ), self.app_log):
                            debit_ledger = quantize_eight(
                                debit_ledger) + quantize_eight(entry[0])
                        debit = debit_ledger + debit_mempool

                        fees = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT fee FROM transactions WHERE address = ?",
                            (mempool_address, ), self.app_log):
                            fees = quantize_eight(fees) + quantize_eight(
                                entry[0])

                        rewards = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT sum(reward) FROM transactions WHERE recipient = ?",
                            (mempool_address, ), self.app_log):
                            rewards = quantize_eight(rewards) + quantize_eight(
                                entry[0])

                        balance = quantize_eight(
                            credit - debit - fees + rewards -
                            quantize_eight(mempool_amount))
                        balance_pre = quantize_eight(credit - debit_ledger -
                                                     fees + rewards)

                        fee = essentials.fee_calculate(mempool_openfield,
                                                       mempool_operation,
                                                       last_block)

                        if quantize_eight(mempool_amount) > quantize_eight(
                                balance_pre):
                            mempool_result.append(
                                "Mempool: Sending more than owned")
                            continue
                        if quantize_eight(balance) - quantize_eight(fee) < 0:
                            mempool_result.append(
                                "Mempool: Cannot afford to pay fees")
                            continue

                        # Pfew! we can finally insert into mempool - all is str, type converted and enforced above
                        self.execute(
                            "INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?)",
                            (mempool_timestamp, mempool_address,
                             mempool_recipient, mempool_amount,
                             mempool_signature_enc, mempool_public_key_hashed,
                             mempool_operation, mempool_openfield))
                        mempool_result.append(
                            "Mempool updated with a received transaction from {}"
                            .format(peer_ip))
                        mempool_result.append("Success")
                        self.commit(
                        )  # Save (commit) the changes to mempool db

                        mempool_size += sys.getsizeof(
                            str(transaction)) / 1000000.0
                    else:
                        mempool_result.append(
                            "Local mempool is already full for this tx type, skipping merging"
                        )
                        # self.app_log.warning("Local mempool is already full for this tx type, skipping merging")
                # TEMP
                # print("Mempool insert", mempool_result)
                return mempool_result
                # TODO: Here maybe commit() on c to release the write lock?
            except Exception as e:
                self.app_log.warning("Mempool: Error processing: {} {}".format(
                    data, e))
                if self.config.debug_conf == 1:
                    raise
        return mempool_result
Esempio n. 2
0
    def merge(self,
              data,
              peer_ip,
              c,
              size_bypass=False,
              wait=False,
              revert=False):
        """
        Checks and merge the tx list in out mempool
        :param data:
        :param peer_ip:
        :param c:
        :param size_bypass: if True, will merge whatever the mempool size is
        :param wait: if True, will wait until the main db_lock is free. if False, will just drop.
        :param revert: if True, we are reverting tx from digest_block, so main lock is on. Don't bother, process without lock.
        :return:
        """
        if not data:
            return "Mempool from {} was empty".format(peer_ip)

        mempool_result = []

        if data == '*':
            raise ValueError("Connection lost")

        try:
            if self.peers_sent[peer_ip] > time.time():
                self.app_log.warning(
                    "Mempool ignoring merge from frozen {}".format(peer_ip))
                mempool_result.append(
                    "Mempool ignoring merge from frozen {}".format(peer_ip))
                return mempool_result
        except:
            # unknown peer
            pass

        if not essentials.is_sequence(data):
            with self.peers_lock:
                self.peers_sent[peer_ip] = time.time() + 10 * 60
            self.app_log.warning(
                "Freezing mempool from {} for 10 min - Bad TX format".format(
                    peer_ip))
            mempool_result.append("Bad TX Format")
            return mempool_result

        mempool_result.append(
            "Mempool merging started from {}".format(peer_ip))

        if not revert:
            while self.db_lock.locked(
            ):  # prevent transactions which are just being digested from being added to mempool
                if not wait:
                    # not reverting, but not waiting, bye
                    # By default, we don't wait.
                    mempool_result.append("Locked ledger, dropping txs")
                    return mempool_result
                self.app_log.warning(
                    "Waiting for block digestion to finish before merging mempool"
                )
                time.sleep(1)
        # if reverting, don't bother with main lock, go on.

        mempool_size = self.size(
        )  # caulculate current mempool size before adding txs

        # TODO: we check main ledger db is not locked before beginning, but we don't lock? ok, see comment in node.py. since it's called from a lock, it would deadlock.
        # merge mempool
        #while self.lock.locked():
        #    time.sleep(1)
        with self.lock:
            try:
                block_list = data

                if not isinstance(
                        block_list[0], list
                ):  # convert to list of lists if only one tx and not handled
                    block_list = [block_list]

                for transaction in block_list:  # set means unique, only accepts list of txs

                    if (mempool_size < 0.3 or size_bypass) or \
                            (len(str(transaction[7])) > 200 and mempool_size < 0.4) \
                            or (Decimal(transaction[3]) > Decimal(5) and mempool_size < 0.5) \
                            or (transaction[1] in self.config.mempool_allowed and mempool_size < 0.6):
                        # condition 1: size limit or bypass,
                        # condition 2: spend more than 25 coins,
                        # condition 3: have length of openfield larger than 200
                        # all transactions in the mempool need to be cycled to check for special cases,
                        # therefore no while/break loop here

                        mempool_timestamp = '%.2f' % (quantize_two(
                            transaction[0]))
                        mempool_address = str(transaction[1])[:56]
                        mempool_recipient = str(transaction[2])[:56]
                        mempool_amount = '%.8f' % (quantize_eight(
                            transaction[3]))  # convert scientific notation
                        mempool_signature_enc = str(transaction[4])[:684]
                        mempool_public_key_hashed = str(transaction[5])[:1068]
                        mempool_operation = str(transaction[6])[:10]
                        mempool_openfield = str(transaction[7])[:100000]

                        # convert readable key to instance
                        mempool_public_key = RSA.importKey(
                            base64.b64decode(mempool_public_key_hashed))
                        mempool_signature_dec = base64.b64decode(
                            mempool_signature_enc)

                        acceptable = True

                        try:
                            # TODO: sure it will throw an exception?
                            # condition 1)
                            dummy = self.fetchall(
                                "SELECT * FROM transactions WHERE signature = ?;",
                                (mempool_signature_enc, ))
                            if dummy:
                                # self.app_log.warning("That transaction is already in our mempool")
                                mempool_result.append(
                                    "That transaction is already in our mempool"
                                )
                                acceptable = False
                                mempool_in = True
                            else:
                                mempool_in = False
                        except:
                            #print('sigmempool NO ', mempool_signature_enc)
                            mempool_in = False

                        # reject transactions which are already in the ledger
                        # TODO: not clean, will need to have ledger as a module too.
                        # dup code atm.
                        essentials.execute_param_c(
                            c,
                            "SELECT * FROM transactions WHERE signature = ?;",
                            (mempool_signature_enc, ),
                            self.app_log)  # condition 2
                        try:
                            dummy = c.fetchall()[0]
                            #print('sigledger', mempool_signature_enc, dummy)
                            if dummy:
                                mempool_result.append(
                                    "That transaction is already in our ledger"
                                )
                                # self.app_log.warning("That transaction is already in our ledger")
                                # reject transactions which are already in the ledger
                                acceptable = False
                                ledger_in = True
                                # Can be a syncing node. Do not request mempool from this peer until 10 min
                                with self.peers_lock:
                                    self.peers_sent[peer_ip] = time.time(
                                    ) + 10 * 60
                                self.app_log.warning(
                                    "Freezing mempool from {} for 10 min.".
                                    format(peer_ip))
                                return mempool_result
                            else:
                                ledger_in = False
                        except:
                            #print('sigledger NO ', mempool_signature_enc)
                            ledger_in = False

                        # if mempool_operation != "1" and mempool_operation != "0":
                        #    mempool_result.append = ("Mempool: Wrong keep value {}".format(mempool_operation))
                        #    acceptable = 0

                        if mempool_address != hashlib.sha224(
                                base64.b64decode(
                                    mempool_public_key_hashed)).hexdigest():
                            mempool_result.append(
                                "Mempool: Attempt to spend from a wrong address"
                            )
                            # self.app_log.warning("Mempool: Attempt to spend from a wrong address")
                            acceptable = False

                        if not essentials.address_validate(
                                mempool_address
                        ) or not essentials.address_validate(
                                mempool_recipient):
                            mempool_result.append(
                                "Mempool: Not a valid address")
                            # self.app_log.warning("Mempool: Not a valid address")
                            acceptable = False

                        if quantize_eight(mempool_amount) < 0:
                            acceptable = False
                            mempool_result.append(
                                "Mempool: Negative balance spend attempt")
                            # self.app_log.warning("Mempool: Negative balance spend attempt")

                        if quantize_two(mempool_timestamp) > time.time(
                        ) + drift_limit:  # dont accept future txs
                            acceptable = False

                        # dont accept old txs, mempool needs to be harsher than ledger
                        if quantize_two(
                                mempool_timestamp) < time.time() - 82800:
                            acceptable = 0
                        # remove from mempool if it's in both ledger and mempool already
                        if mempool_in and ledger_in:
                            try:
                                # Do not lock, we already have the lock for the whole merge.
                                self.execute(SQL_DELETE_TX,
                                             (mempool_signature_enc, ))
                                self.commit()
                                mempool_result.append(
                                    "Mempool: Transaction deleted from our mempool"
                                )
                            except:  # experimental try and except
                                mempool_result.append(
                                    "Mempool: Transaction was not present in the pool anymore"
                                )
                                pass  # continue to mempool finished message

                                # verify signatures and balances

                        essentials.validate_pem(mempool_public_key_hashed)

                        # verify signature
                        verifier = PKCS1_v1_5.new(mempool_public_key)

                        my_hash = SHA.new(
                            str((mempool_timestamp, mempool_address,
                                 mempool_recipient, mempool_amount,
                                 mempool_operation,
                                 mempool_openfield)).encode("utf-8"))
                        if not verifier.verify(my_hash, mempool_signature_dec):
                            acceptable = False
                            mempool_result.append(
                                "Mempool: Wrong signature in mempool insert attempt: {}"
                                .format(transaction))
                            # self.app_log.warning("Mempool: Wrong signature in mempool insert attempt")

                        # verify signature
                        if acceptable:

                            # verify balance
                            # mempool_result.append("Mempool: Verifying balance")
                            mempool_result.append(
                                "Mempool: Received address: {}".format(
                                    mempool_address))

                            # include mempool fees
                            result = self.fetchall(
                                "SELECT amount, openfield FROM transactions WHERE address = ?;",
                                (mempool_address, ))
                            debit_mempool = 0

                            if result:
                                for x in result:
                                    debit_tx = quantize_eight(x[0])
                                    fee = quantize_eight(
                                        essentials.fee_calculate(x[1]))
                                    debit_mempool = quantize_eight(
                                        debit_mempool + debit_tx + fee)
                            else:
                                debit_mempool = 0

                            # include the new block
                            credit_ledger = Decimal("0")
                            for entry in essentials.execute_param_c(
                                    c,
                                    "SELECT amount FROM transactions WHERE recipient = ?;",
                                (mempool_address, ), self.app_log):
                                try:
                                    credit_ledger = quantize_eight(
                                        credit_ledger) + quantize_eight(
                                            entry[0])
                                    credit_ledger = 0 if credit_ledger is None else credit_ledger
                                except:
                                    credit_ledger = 0

                            credit = credit_ledger

                            debit_ledger = Decimal("0")
                            for entry in essentials.execute_param_c(
                                    c,
                                    "SELECT amount FROM transactions WHERE address = ?;",
                                (mempool_address, ), self.app_log):
                                try:
                                    debit_ledger = quantize_eight(
                                        debit_ledger) + quantize_eight(
                                            entry[0])
                                    debit_ledger = 0 if debit_ledger is None else debit_ledger
                                except:
                                    debit_ledger = 0

                            debit = debit_ledger + debit_mempool

                            fees = Decimal("0")
                            for entry in essentials.execute_param_c(
                                    c,
                                    "SELECT fee FROM transactions WHERE address = ?;",
                                (mempool_address, ), self.app_log):
                                try:
                                    fees = quantize_eight(
                                        fees) + quantize_eight(entry[0])
                                    fees = 0 if fees is None else fees
                                except:
                                    fees = 0

                            rewards = Decimal("0")
                            for entry in essentials.execute_param_c(
                                    c,
                                    "SELECT sum(reward) FROM transactions WHERE recipient = ?;",
                                (mempool_address, ), self.app_log):
                                try:
                                    rewards = quantize_eight(
                                        rewards) + quantize_eight(entry[0])
                                    rewards = 0 if rewards is None else rewards
                                except:
                                    rewards = 0

                            balance = quantize_eight(
                                credit - debit - fees + rewards -
                                quantize_eight(mempool_amount))
                            balance_pre = quantize_eight(credit_ledger -
                                                         debit_ledger - fees +
                                                         rewards)

                            fee = essentials.fee_calculate(mempool_openfield)

                            time_now = time.time()

                            if quantize_two(mempool_timestamp) > quantize_two(
                                    time_now) + drift_limit:
                                mempool_result.append(
                                    "Mempool: Future transaction not allowed, timestamp {} minutes in the future"
                                    .format(
                                        quantize_two(
                                            (quantize_two(mempool_timestamp) -
                                             quantize_two(time_now)) / 60)))
                                # self.app_log.warning("Mempool: Future transaction not allowed, timestamp {} minutes in the future.")

                            elif quantize_two(time_now) - 86400 > quantize_two(
                                    mempool_timestamp):
                                mempool_result.append(
                                    "Mempool: Transaction older than 24h not allowed."
                                )
                                # self.app_log.warning("Mempool: Transaction older than 24h not allowed.")

                            elif quantize_eight(
                                    mempool_amount) > quantize_eight(
                                        balance_pre):
                                mempool_result.append(
                                    "Mempool: Sending more than owned")
                                # self.app_log.warning("Mempool: Sending more than owned")
                            elif quantize_eight(balance) - quantize_eight(
                                    fee) < 0:
                                mempool_result.append(
                                    "Mempool: Cannot afford to pay fees")
                                # self.app_log.warning("Mempool: Cannot afford to pay fees")

                            # verify signatures and balances
                            else:
                                self.execute(
                                    "INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?)",
                                    (str(mempool_timestamp),
                                     str(mempool_address),
                                     str(mempool_recipient),
                                     str(mempool_amount),
                                     str(mempool_signature_enc),
                                     str(mempool_public_key_hashed),
                                     str(mempool_operation),
                                     str(mempool_openfield)))
                                mempool_result.append(
                                    "Mempool updated with a received transaction from {}"
                                    .format(peer_ip))
                                self.commit()  # Save (commit) the changes

                                mempool_size = mempool_size + sys.getsizeof(
                                    str(transaction)) / 1000000.0
                    else:
                        mempool_result.append(
                            "Local mempool is already full for this tx type, skipping merging"
                        )
                        # self.app_log.warning("Local mempool is already full for this tx type, skipping merging")
                        return mempool_result  # avoid spamming of the logs
                # TODO: Here maybe commit() on c to release the write lock?
            except Exception as e:
                self.app_log.warning("Mempool: Error processing: {} {}".format(
                    data, e))
                if self.config.debug_conf == 1:
                    raise
        try:
            return e, mempool_result
        except:
            return mempool_result
Esempio n. 3
0
    def merge(self,
              data: list,
              peer_ip: str,
              c,
              size_bypass: bool = False,
              wait: bool = False,
              revert: bool = False) -> list:
        """
        Checks and merge the tx list in out mempool
        :param data:
        :param peer_ip:
        :param c:
        :param size_bypass: if True, will merge whatever the mempool size is
        :param wait: if True, will wait until the main db_lock is free. if False, will just drop.
        :param revert: if True, we are reverting tx from digest_block, so main lock is on. Don't bother, process without lock.
        :return:
        """
        global REFUSE_OLDER_THAN
        # Easy cases of empty or invalid data
        if not data:
            return ["Mempool from {} was empty".format(peer_ip)]
        mempool_result = []
        if data == '*':
            raise ValueError("Connection lost")
        try:
            if self.peers_sent[peer_ip] > time.time(
            ) and peer_ip != '127.0.0.1':
                self.app_log.warning(
                    "Mempool ignoring merge from frozen {}".format(peer_ip))
                mempool_result.append(
                    "Mempool ignoring merge from frozen {}".format(peer_ip))
                return mempool_result
        except:
            # unknown peer
            pass
        if not essentials.is_sequence(data):
            if peer_ip != '127.0.0.1':
                with self.peers_lock:
                    self.peers_sent[peer_ip] = time.time() + 10 * 60
                self.app_log.warning(
                    "Freezing mempool from {} for 10 min - Bad TX format".
                    format(peer_ip))
            mempool_result.append("Bad TX Format")
            return mempool_result

        if not revert:
            while self.db_lock.locked():
                # prevent transactions which are just being digested from being added to mempool
                if not wait:
                    # not reverting, but not waiting, bye
                    # By default, we don't wait.
                    mempool_result.append("Locked ledger, dropping txs")
                    return mempool_result
                self.app_log.warning(
                    "Waiting for block digestion to finish before merging mempool"
                )
                time.sleep(1)
        # if reverting, don't bother with main lock, go on.

        # Let's really dig
        mempool_result.append(
            "Mempool merging started from {}".format(peer_ip))
        # Single time reference here for the whole merge.
        time_now = time.time()
        # calculate current mempool size before adding txs
        mempool_size = self.size()

        # TODO: we check main ledger db is not locked before beginning, but we don't lock? ok, see comment in node.py. since it's called from a lock, it would deadlock.
        # merge mempool
        # while self.lock.locked():
        #    time.sleep(1)
        with self.lock:
            try:
                block_list = data
                if not isinstance(
                        block_list[0], list
                ):  # convert to list of lists if only one tx and not handled
                    block_list = [block_list]

                for transaction in block_list:
                    if size_bypass or self.space_left_for_tx(
                            transaction, mempool_size):
                        # all transactions in the mempool need to be cycled to check for special cases,
                        # therefore no while/break loop here
                        try:
                            mempool_timestamp = '%.2f' % (quantize_two(
                                transaction[0]))
                            mempool_timestamp_float = float(
                                transaction[0]
                            )  # limit Decimal where not needed
                        except Exception as e:
                            mempool_result.append(
                                "Mempool: Invalid timestamp {}".format(
                                    transaction[0]))
                        if not essentials.address_validate(transaction[1]):
                            mempool_result.append(
                                "Mempool: Invalid address {}".format(
                                    transaction[1]))
                            continue
                        # We could now ignore the truncates here, I left them for explicit reminder of the various fields max lengths.
                        mempool_address = str(transaction[1])[:56]
                        if not essentials.address_validate(transaction[2]):
                            mempool_result.append(
                                "Mempool: Invalid recipient {}".format(
                                    transaction[2]))
                            continue
                        mempool_recipient = str(transaction[2])[:56]
                        try:
                            mempool_amount = '%.8f' % (quantize_eight(
                                transaction[3]))  # convert scientific notation
                            mempool_amount_float = float(transaction[3])
                        except Exception as e:
                            mempool_result.append(
                                "Mempool: Invalid amount {}".format(
                                    transaction[3]))
                            continue
                        if len(transaction[4]) > 684:
                            mempool_result.append(
                                "Mempool: Invalid signature len{}".format(
                                    len(transaction[4])))
                            continue
                        mempool_signature_enc = str(transaction[4])[:684]
                        if len(transaction[5]) > 1068:
                            mempool_result.append(
                                "Mempool: Invalid pubkey len{}".format(
                                    len(transaction[5])))
                            continue
                        mempool_public_key_b64encoded = str(
                            transaction[5])[:1068]
                        if "b'" == mempool_public_key_b64encoded[:2]:
                            # Binary content instead of str - leftover from legacy code?
                            mempool_public_key_b64encoded = transaction[5][
                                2:1070]
                        if len(transaction[6]) > 30:
                            mempool_result.append(
                                "Mempool: Invalid operation len{}".format(
                                    len(transaction[6])))
                            continue
                        mempool_operation = str(transaction[6])[:30]
                        if len(transaction[7]) > 100000:
                            mempool_result.append(
                                "Mempool: Invalid openfield len{}".format(
                                    len(transaction[7])))
                            continue
                        mempool_openfield = str(transaction[7])[:100000]

                        if len(mempool_openfield) <= 4:
                            # no or short message for a mandatory message
                            if mempool_recipient in self.config.mandatory_message.keys(
                            ):
                                mempool_result.append(
                                    "Mempool: Missing message - {}".format(
                                        self.config.
                                        mandatory_message[mempool_recipient]))
                                continue

                        # Begin with the easy tests that do not require cpu or disk access
                        if mempool_amount_float < 0:
                            mempool_result.append(
                                "Mempool: Negative balance spend attempt")
                            continue
                        if mempool_timestamp_float > time_now:
                            mempool_result.append(
                                "Mempool: Future transaction rejected {}s".
                                format(mempool_timestamp_float - time_now))
                            continue
                        if mempool_timestamp_float < time_now - REFUSE_OLDER_THAN:
                            # don't accept old txs, mempool needs to be harsher than ledger
                            mempool_result.append(
                                "Mempool: Too old a transaction")
                            continue

                        # Then more cpu heavy tests
                        buffer = str((mempool_timestamp, mempool_address,
                                      mempool_recipient, mempool_amount,
                                      mempool_operation,
                                      mempool_openfield)).encode("utf-8")

                        #  Will raise if error
                        try:
                            SignerFactory.verify_bis_signature(
                                mempool_signature_enc,
                                mempool_public_key_b64encoded, buffer,
                                mempool_address)
                        except Exception as e:
                            mempool_result.append(
                                f"Mempool: Signature did not match for address ({e})"
                            )
                            continue

                        # Only now, process the tests requiring db access
                        mempool_in = self.sig_check(mempool_signature_enc)

                        # Temp: get last block for HF reason
                        essentials.execute_param_c(
                            c,
                            "SELECT block_height FROM transactions WHERE 1 ORDER by block_height DESC limit ?",
                            (1, ), self.app_log)
                        last_block = c.fetchone()[0]
                        # reject transactions which are already in the ledger
                        # TODO: not clean, will need to have ledger as a module too.
                        # TODO: need better txid index, this is very sloooooooow
                        if self.config.old_sqlite:
                            essentials.execute_param_c(
                                c,
                                "SELECT timestamp FROM transactions WHERE signature = ?1",
                                (mempool_signature_enc, ), self.app_log)
                        else:
                            essentials.execute_param_c(
                                c,
                                "SELECT timestamp FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) AND signature = ?1",
                                (mempool_signature_enc, ), self.app_log)
                        ledger_in = bool(c.fetchone())
                        # remove from mempool if it's in both ledger and mempool already
                        if mempool_in and ledger_in:
                            try:
                                # Do not lock, we already have the lock for the whole merge.
                                if self.config.old_sqlite:
                                    self.execute(SQL_DELETE_TX_OLD,
                                                 (mempool_signature_enc, ))
                                else:
                                    self.execute(SQL_DELETE_TX,
                                                 (mempool_signature_enc, ))
                                self.commit()
                                mempool_result.append(
                                    "Mempool: Transaction deleted from our mempool"
                                )
                            except:  # experimental try and except
                                mempool_result.append(
                                    "Mempool: Transaction was not present in the pool anymore"
                                )
                            continue
                        if ledger_in:
                            mempool_result.append(
                                "That transaction is already in our ledger")
                            # Can be a syncing node. Do not request mempool from this peer until FREEZE_MIN min
                            # ledger_in is the ts of the tx in ledger. if it's recent, maybe the peer is just one block late.
                            # give him 15 minute margin.
                            if (peer_ip != '127.0.0.1') and (
                                    ledger_in < time_now - 60 * 15):
                                with self.peers_lock:
                                    self.peers_sent[peer_ip] = time.time(
                                    ) + FREEZE_MIN * 60
                                self.app_log.warning(
                                    "Freezing mempool from {} for {} min.".
                                    format(peer_ip, FREEZE_MIN))
                            # Here, we point blank stop processing the batch from this host since it's outdated.
                            # Update: Do not, since it blocks further valid tx - case has been found in real use.
                            # return mempool_result
                            continue
                        # Already there, just ignore then
                        if mempool_in:
                            mempool_result.append(
                                "That transaction is already in our mempool")
                            continue

                        # Here we covered the basics, the current tx is conform and signed. Now let's check balance.

                        # verify balance
                        mempool_result.append(
                            "Mempool: Received address: {}".format(
                                mempool_address))
                        # include mempool fees
                        result = self.fetchall(
                            "SELECT amount, openfield, operation FROM transactions WHERE address = ?",
                            (mempool_address, ))
                        debit_mempool = 0
                        if result:
                            for x in result:
                                debit_tx = quantize_eight(x[0])
                                fee = quantize_eight(
                                    essentials.fee_calculate(
                                        x[1], x[2], last_block))
                                debit_mempool = quantize_eight(debit_mempool +
                                                               debit_tx + fee)

                        credit = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT amount FROM transactions WHERE recipient = ?",
                            (mempool_address, ), self.app_log):
                            credit = quantize_eight(credit) + quantize_eight(
                                entry[0])

                        debit_ledger = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT amount FROM transactions WHERE address = ?",
                            (mempool_address, ), self.app_log):
                            debit_ledger = quantize_eight(
                                debit_ledger) + quantize_eight(entry[0])
                        debit = debit_ledger + debit_mempool

                        fees = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT fee FROM transactions WHERE address = ?",
                            (mempool_address, ), self.app_log):
                            fees = quantize_eight(fees) + quantize_eight(
                                entry[0])

                        rewards = 0
                        for entry in essentials.execute_param_c(
                                c,
                                "SELECT sum(reward) FROM transactions WHERE recipient = ?",
                            (mempool_address, ), self.app_log):
                            rewards = quantize_eight(rewards) + quantize_eight(
                                entry[0])
                            # error conversion from NoneType to Decimal is not supported

                        balance = quantize_eight(
                            credit - debit - fees + rewards -
                            quantize_eight(mempool_amount))
                        balance_pre = quantize_eight(credit - debit_ledger -
                                                     fees + rewards)

                        fee = essentials.fee_calculate(mempool_openfield,
                                                       mempool_operation,
                                                       last_block)

                        # print("Balance", balance, fee)

                        if quantize_eight(mempool_amount) > quantize_eight(
                                balance_pre):
                            # mp amount is already included in "balance" var! also, that tx might already be in the mempool
                            mempool_result.append(
                                "Mempool: Sending more than owned")
                            continue
                        if quantize_eight(balance) - quantize_eight(fee) < 0:
                            mempool_result.append(
                                "Mempool: Cannot afford to pay fees")
                            continue

                        # Pfew! we can finally insert into mempool - all is str, type converted and enforced above
                        self.execute(
                            "INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?,?)",
                            (mempool_timestamp, mempool_address,
                             mempool_recipient, mempool_amount,
                             mempool_signature_enc,
                             mempool_public_key_b64encoded, mempool_operation,
                             mempool_openfield, int(time_now)))
                        mempool_result.append(
                            "Mempool updated with a received transaction from {}"
                            .format(peer_ip))
                        mempool_result.append(
                            "Success"
                        )  # WARNING: Do not change string or case ever!
                        self.commit(
                        )  # Save (commit) the changes to mempool db

                        mempool_size += sys.getsizeof(
                            str(transaction)) / 1000000.0
                    else:
                        mempool_result.append(
                            "Local mempool is already full for this tx type, skipping merging"
                        )
                        # self.app_log.warning("Local mempool is already full for this tx type, skipping merging")
                # TEMP
                # print("Mempool insert", mempool_result)
                return mempool_result
                # TODO: Here maybe commit() on c to release the write lock?
            except Exception as e:
                self.app_log.warning("Mempool: Error processing: {} {}".format(
                    data, e))
                if self.config.debug:
                    exc_type, exc_obj, exc_tb = sys.exc_info()
                    fname = os.path.split(
                        exc_tb.tb_frame.f_code.co_filename)[1]
                    self.app_log.warning("{} {} {}".format(
                        exc_type, fname, exc_tb.tb_lineno))
                    mempool_result.append("Exception: {}".format(str(e)))
                    # if left there, means debug can *not* be used in production, or exception is not sent back to the client.
                    raise
        return mempool_result