def attempt_single_batch(self, payment_records, op_counter, dry_run=None):
if not op_counter.get():
status, counter = self.clnt_mngr.request_url(self.comm_counter)
if status != 200:
raise Exception(
"Received response code {} for request '{}'".format(
status, self.comm_counter))
counter = int(counter)
self.base_counter = int(counter)
op_counter.set(self.base_counter)
_, head = self.clnt_mngr.request_url(self.comm_head)
branch = head["hash"]
chain_id = head["chain_id"]
protocol = head["metadata"]["protocol"]
logger.debug("head: branch {} counter {} protocol {}".format(
branch, op_counter.get(), protocol))
content_list = []
for payment_item in payment_records:
storage = self.storage_limit
pymnt_amnt = payment_item.amount # expects in micro tezos
# TRD extension for non scriptless contract accounts
gas_limit, tx_fee = self.gas_limit, self.default_fee
if payment_item.paymentaddress.startswith('KT'):
simulation_status, simulation_results = self.simulate_single_operation(
payment_item, pymnt_amnt, branch, chain_id)
if simulation_status == PaymentStatus.FAIL:
payment_item.paid = PaymentStatus.FAIL
continue
gas_limit, tx_fee, storage = simulation_results
burn_fee = COST_PER_BYTE * storage
total_fee = tx_fee + burn_fee
# Bound the total (baker and burn) fee by the reward amount in case of KT1 accounts
if total_fee > FEE_LIMIT_CONTRACTS:
logger.debug(
"Payment to {} script requires higher fees than reward amount. Skipping."
.format(payment_item.paymentaddress))
payment_item.paid = PaymentStatus.FAIL
continue
if payment_item.needs_activation:
storage += RA_STORAGE
if self.delegator_pays_ra_fee:
pymnt_amnt = max(pymnt_amnt - RA_BURN_FEE,
0) # ensure not less than 0
if self.delegator_pays_xfer_fee:
pymnt_amnt = max(pymnt_amnt - tx_fee,
0) # ensure not less than 0
# if pymnt_amnt becomes 0, don't pay
if pymnt_amnt == 0:
logger.debug(
"Payment to {} became 0 after deducting fees. Skipping.".
format(payment_item.paymentaddress))
continue
op_counter.inc()
content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.paymentaddress) \
.replace("%AMOUNT%", str(pymnt_amnt)).replace("%COUNTER%", str(op_counter.get())) \
.replace("%fee%", str(tx_fee)).replace("%gas_limit%", str(gas_limit)).replace(
"%storage_limit%", str(storage))
content_list.append(content)
verbose_logger.info("Payment content: {}".format(content))
contents_string = ",".join(content_list)
# run the operations
logger.debug("Running {} operations".format(len(content_list)))
runops_json = RUNOPS_JSON.replace('%BRANCH%', branch).replace(
"%CONTENT%", contents_string)
runops_json = JSON_WRAP.replace("%JSON%", runops_json).replace(
"%chain_id%", chain_id)
status, run_ops_parsed = self.clnt_mngr.request_url_post(
self.comm_runops, runops_json)
if not (status == 200):
logger.error("Error in run_operation")
return PaymentStatus.FAIL, ""
# Check each contents object for failure
for op in run_ops_parsed["contents"]:
# https://docs.python.org/3/glossary.html#term-eafp
try:
op_status = op["metadata"]["operation_result"]["status"]
if op_status == "failed":
op_error = op["metadata"]["operation_result"]["errors"][0][
"id"]
logger.error(
"Error while validating operation - Status: {}, Message: {}"
.format(op_status, op_error))
return PaymentStatus.FAIL, ""
except KeyError:
logger.debug(
"Unable to find metadata->operation_result->{status,errors} in run_ops response"
)
pass
# forge the operations
logger.debug("Forging {} operations".format(len(content_list)))
forge_json = FORGE_JSON.replace('%BRANCH%', branch).replace(
"%CONTENT%", contents_string)
# if verbose: print("--> forge_command_str is |{}|".format(forge_command_str))
status, bytes = self.clnt_mngr.request_url_post(
self.comm_forge, forge_json)
if not (status == 200):
logger.error("Error in forge operation")
return PaymentStatus.FAIL, ""
signed_bytes = self.clnt_mngr.sign(bytes, self.manager)
# pre-apply operations
logger.debug("Preapplying the operations")
preapply_json = PREAPPLY_JSON.replace('%BRANCH%', branch).replace(
"%CONTENT%",
contents_string).replace("%PROTOCOL%",
protocol).replace("%SIGNATURE%",
signed_bytes)
# if verbose: print("--> preapply_command_str is |{}|".format(preapply_command_str))
status, preapply_command_response = self.clnt_mngr.request_url_post(
self.comm_preapply, preapply_json)
if not (status == 200):
logger.error("Error in preapply operation")
return PaymentStatus.FAIL, ""
# if dry_run, skip injection
if dry_run:
return PaymentStatus.DONE, ""
# inject the operations
logger.debug("Injecting {} operations".format(len(content_list)))
decoded = base58.b58decode(signed_bytes).hex()
if signed_bytes.startswith("edsig"): # edsig signature
decoded_edsig_signature = decoded[
10:][:-8] # first 5 bytes edsig, last 4 bytes checksum
decoded_signature = decoded_edsig_signature
elif signed_bytes.startswith("sig"): # generic signature
decoded_sig_signature = decoded[
6:][:-8] # first 3 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
elif signed_bytes.startswith("p2sig"):
decoded_sig_signature = decoded[
8:][:-8] # first 4 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
else:
raise Exception("Signature '{}' is not in expected format".format(
signed_bytes))
if len(decoded_signature) != 128: # must be 64 bytes
# raise Exception("Signature length must be 128 but it is {}. Signature is '{}'".format(len(signed_bytes), signed_bytes))
logger.warn(
"Signature length must be 128 but it is {}. Signature is '{}'".
format(len(signed_bytes), signed_bytes))
# return False, ""
signed_operation_bytes = bytes + decoded_signature
_, head = self.clnt_mngr.request_url(self.comm_head)
last_level_before_injection = head['header']['level']
status, operation_hash = self.clnt_mngr.request_url_post(
self.comm_inject, json.dumps(signed_operation_bytes))
if not (status == 200):
logger.error("Error in inject operation")
return PaymentStatus.FAIL, ""
logger.info("Operation hash is {}".format(operation_hash))
# wait for inclusion
timeout = MAX_BLOCKS_TO_CHECK_AFTER_INJECTION * MAX_NUM_TRIALS_PER_BLOCK * self.network_config[
'BLOCK_TIME_IN_SEC'] // 60
logger.info(
"Waiting for operation {} to be included... Please do not interrupt the process!!! (Timeout is around {} minutes)"
.format(operation_hash, timeout))
for i in range(
last_level_before_injection + 1, last_level_before_injection +
1 + MAX_BLOCKS_TO_CHECK_AFTER_INJECTION):
cmd = self.comm_wait.replace("%BLOCK_HASH%", str(i))
status = -1
list_op_hash = []
trial_i = 0
while not (status == 200) and (trial_i < MAX_NUM_TRIALS_PER_BLOCK):
sleep(self.network_config['BLOCK_TIME_IN_SEC'])
status, list_op_hash = self.clnt_mngr.request_url(cmd)
if not (status == 200):
logger.warning(
"Level {} could not be queried about operation hashes".
format(i))
break
for op_hashes in list_op_hash:
if operation_hash in op_hashes:
logger.info(
"Operation {} is included".format(operation_hash))
return PaymentStatus.PAID, operation_hash
logger.debug("Operation {} is not included at level {}".format(
operation_hash, i))
logger.warning(
"Operation {} wait is timed out. Not sure about the result!".
format(operation_hash))
return PaymentStatus.INJECTED, operation_hash
def attempt_single_batch(self, payment_records, op_counter, dry_run=None):
if not op_counter.get():
_, response = self.wllt_clnt_mngr.send_request(self.comm_counter)
counter = parse_json_response(response)
counter = int(counter)
op_counter.set(counter)
_, response = self.wllt_clnt_mngr.send_request(self.comm_head,
verbose_override=False)
head = parse_json_response(response)
branch = head["hash"]
chain_id = head["chain_id"]
protocol = head["metadata"]["protocol"]
logger.debug("head: branch {} counter {} protocol {}".format(
branch, op_counter.get(), protocol))
content_list = []
for payment_item in payment_records:
storage = self.storage_limit
pymnt_amnt = payment_item.amount # expects in micro tezos
if payment_item.needs_activation:
storage += RA_STORAGE
if self.delegator_pays_ra_fee:
pymnt_amnt = max(pymnt_amnt - RA_BURN_FEE,
0) # ensure not less than 0
if self.delegator_pays_xfer_fee:
pymnt_amnt = max(pymnt_amnt - self.default_fee,
0) # ensure not less than 0
# if pymnt_amnt becomes 0, don't pay
if pymnt_amnt == 0:
logger.debug(
"Payment to {} became 0 after deducting fees. Skipping.".
format(payment_item.paymentaddress))
continue
op_counter.inc()
content = CONTENT.replace("%SOURCE%", self.source).replace("%DESTINATION%", payment_item.paymentaddress) \
.replace("%AMOUNT%", str(pymnt_amnt)).replace("%COUNTER%", str(op_counter.get())) \
.replace("%fee%", str(self.default_fee)).replace("%gas_limit%", self.gas_limit).replace(
"%storage_limit%", str(storage))
content_list.append(content)
verbose_logger.info("Payment content: {}".format(content))
contents_string = ",".join(content_list)
# run the operations
logger.debug("Running {} operations".format(len(content_list)))
runops_json = RUNOPS_JSON.replace('%BRANCH%', branch).replace(
"%CONTENT%", contents_string)
runops_json = JSON_WRAP.replace("%JSON%", runops_json).replace(
"%chain_id%", chain_id)
runops_command_str = self.comm_runops.replace("%JSON%", runops_json)
result, runops_command_response = self.wllt_clnt_mngr.send_request(
runops_command_str)
if not result:
logger.error("Error in run_operation")
logger.debug("Error in run_operation, request ->{}<-".format(
runops_command_str))
logger.debug("---")
logger.debug("Error in run_operation, response ->{}<-".format(
runops_command_response))
return PaymentStatus.FAIL, ""
# Parse result of run_operation and check for potential failures
run_ops_parsed = parse_json_response(runops_command_response)
# Check each contents object for failure
for op in run_ops_parsed["contents"]:
# https://docs.python.org/3/glossary.html#term-eafp
try:
op_status = op["metadata"]["operation_result"]["status"]
if op_status == "failed":
op_error = op["metadata"]["operation_result"]["errors"][0][
"id"]
logger.error(
"Error while validating operation - Status: {}, Message: {}"
.format(op_status, op_error))
return PaymentStatus.FAIL, ""
except KeyError:
logger.debug(
"Unable to find metadata->operation_result->{status,errors} in run_ops response"
)
pass
# forge the operations
logger.debug("Forging {} operations".format(len(content_list)))
forge_json = FORGE_JSON.replace('%BRANCH%', branch).replace(
"%CONTENT%", contents_string)
forge_command_str = self.comm_forge.replace("%JSON%", forge_json)
result, forge_command_response = self.wllt_clnt_mngr.send_request(
forge_command_str)
if not result:
logger.error("Error in forge operation")
logger.debug(
"Error in forge, request '{}'".format(forge_command_str))
logger.debug("---")
logger.debug(
"Error in forge, response '{}'".format(forge_command_response))
return PaymentStatus.FAIL, ""
# sign the operations
bytes = parse_json_response(forge_command_response)
signed_bytes = self.wllt_clnt_mngr.sign(bytes,
self.manager_alias,
verbose_override=True)
# pre-apply operations
logger.debug("Preapplying the operations")
preapply_json = PREAPPLY_JSON.replace('%BRANCH%', branch).replace(
"%CONTENT%",
contents_string).replace("%PROTOCOL%",
protocol).replace("%SIGNATURE%",
signed_bytes)
preapply_command_str = self.comm_preapply.replace(
"%JSON%", preapply_json)
result, preapply_command_response = self.wllt_clnt_mngr.send_request(
preapply_command_str)
if not result:
logger.error("Error in preapply operation")
logger.debug(
"Error in preapply, request '{}'".format(preapply_command_str))
logger.debug("---")
logger.debug("Error in preapply, response '{}'".format(
preapply_command_response))
return PaymentStatus.FAIL, ""
# not necessary
# preapplied = parse_response(preapply_command_response)
# if dry_run, skip injection
if dry_run:
return PaymentStatus.DONE, ""
# inject the operations
logger.debug("Injecting {} operations".format(len(content_list)))
decoded = base58.b58decode(signed_bytes).hex()
if signed_bytes.startswith("edsig"): # edsig signature
decoded_edsig_signature = decoded[
10:][:-8] # first 5 bytes edsig, last 4 bytes checksum
decoded_signature = decoded_edsig_signature
elif signed_bytes.startswith("sig"): # generic signature
decoded_sig_signature = decoded[
6:][:-8] # first 3 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
elif signed_bytes.startswith("p2sig"):
decoded_sig_signature = decoded[
8:][:-8] # first 4 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
else:
raise Exception("Signature '{}' is not in expected format".format(
signed_bytes))
if len(decoded_signature) != 128: # must be 64 bytes
# raise Exception("Signature length must be 128 but it is {}. Signature is '{}'".format(len(signed_bytes), signed_bytes))
logger.warn(
"Signature length must be 128 but it is {}. Signature is '{}'".
format(len(signed_bytes), signed_bytes))
# return False, ""
signed_operation_bytes = bytes + decoded_signature
inject_command_str = self.comm_inject.replace("%OPERATION_HASH%",
signed_operation_bytes)
result, inject_command_response = self.wllt_clnt_mngr.send_request(
inject_command_str)
if not result:
logger.error("Error in inject operation")
logger.debug(
"Error in inject, response '{}'".format(inject_command_str))
logger.debug("---")
logger.debug("Error in inject, response '{}'".format(
inject_command_response))
return PaymentStatus.FAIL, ""
operation_hash = parse_json_response(inject_command_response)
logger.info("Operation hash is {}".format(operation_hash))
# wait for inclusion
logger.info(
"Waiting for operation {} to be included. Please be patient until the block has {} confirmation(s)"
.format(operation_hash, CONFIRMATIONS))
try:
cmd = self.comm_wait.replace("%OPERATION%", operation_hash)
self.wllt_clnt_mngr.send_request(
cmd, timeout=self.get_confirmation_timeout())
logger.info("Operation {} is included".format(operation_hash))
except TimeoutExpired:
logger.warn(
"Operation {} wait is timed out. Not sure about the result!".
format(operation_hash))
return PaymentStatus.INJECTED, operation_hash
return PaymentStatus.PAID, operation_hash
def attempt_single_batch(self, payment_items, op_counter, dry_run=None):
if not op_counter.get():
status, counter = self.clnt_mngr.request_url(self.comm_counter)
if status != HTTPStatus.OK:
raise Exception(
"Received response code {} for request '{}'".format(
status, self.comm_counter))
counter = int(counter)
self.base_counter = int(counter)
op_counter.set(self.base_counter)
_, head = self.clnt_mngr.request_url(self.comm_payment_head)
branch = head["hash"]
chain_id = head["chain_id"]
protocol = head["metadata"]["protocol"]
logger.debug("head: branch {} counter {} protocol {}".format(
branch, op_counter.get(), protocol))
content_list = []
total_gas = total_tx_fees = total_burn_fees = 0
for payment_item in payment_items:
pymnt_amnt = payment_item.adjusted_amount # expected in micro tez
# Get initial default values for storage, gas and fees
# These default values are used for non-empty tz1 accounts transactions
storage_limit, gas_limit, tx_fee, burn_fee = (
self.storage_limit,
self.gas_limit,
self.default_fee,
0,
)
# TRD extension for non scriptless contract accounts
if payment_item.paymentaddress.startswith("KT"):
simulation_status, simulation_results = self.simulate_single_operation(
payment_item, pymnt_amnt, branch, chain_id)
if simulation_status == PaymentStatus.FAIL:
logger.info(
"Payment to {} script could not be processed. Possible reason: liquidated contract. Skipping. Think about redirecting the payout to the owner address using the maps rules. Please refer to the TRD documentation or to one of the TRD maintainers."
.format(payment_item.paymentaddress))
payment_item.paid = PaymentStatus.AVOIDED
payment_item.desc += "Investigate on https://tzkt.io - Liquidated oven or no default entry point. Use rules map for payment redirect. "
continue
gas_limit, tx_fee, storage_limit = simulation_results
burn_fee = COST_PER_BYTE * storage_limit
if KT1_FEE_SAFETY_CHECK:
total_fee = tx_fee + burn_fee
if total_fee > FEE_LIMIT_CONTRACTS:
logger.info(
"Payment to {:s} script requires higher fees than allowed maximum. Skipping. Needed fee: {:<,d} mutez, max fee: {:<,d} mutez. Either configure a higher fee or redirect to the owner address using the maps rules. Refer to the TRD documentation."
.format(
payment_item.paymentaddress,
total_fee,
FEE_LIMIT_CONTRACTS,
))
payment_item.paid = PaymentStatus.AVOIDED
payment_item.desc += "Kt safety check: Transaction fees higher then allowed maximum: {:<,d} mutez. ".format(
FEE_LIMIT_CONTRACTS, )
continue
if (pymnt_amnt - total_fee) < ZERO_THRESHOLD:
logger.info(
"Payment to {:s} requires fees of {:<,d} mutez higher than payment amount of {:<,d} mutez. "
"Payment avoided due KT1_FEE_SAFETY_CHECK set to True."
.format(
payment_item.paymentaddress,
total_fee,
pymnt_amnt,
))
payment_item.paid = PaymentStatus.AVOIDED
payment_item.desc += "Kt safety check: Burn + transaction fees higher then payment amount. "
continue
else:
# An implicit tz1 account
if payment_item.needs_activation:
storage_limit += RA_STORAGE
# TODO: Check what value is actually correct here
# burn_fee = COST_PER_BYTE * RA_STORAGE
burn_fee = RA_BURN_FEE
if burn_fee > 0:
# Subtract burn fee from the payment amount
orig_pymnt_amnt = pymnt_amnt
if self.delegator_pays_ra_fee:
pymnt_amnt = max(pymnt_amnt - burn_fee, 0)
payment_item.delegator_transaction_fee += burn_fee
else:
payment_item.delegate_transaction_fee += burn_fee
logger.info(
"Payment to {} requires {:<,d} gas * {:.2f} mutez-per-gas + {:<,d} mutez burn fee; "
"Payment reduced from {:<,d} mutez to {:<,d} mutez".format(
payment_item.paymentaddress,
gas_limit,
MUTEZ_PER_GAS_UNIT,
burn_fee,
orig_pymnt_amnt,
pymnt_amnt,
))
# Subtract transaction's fee from the payment amount if delegator has to pay for it
if self.delegator_pays_xfer_fee:
pymnt_amnt = max(pymnt_amnt - tx_fee, 0)
payment_item.delegator_transaction_fee += tx_fee
else:
payment_item.delegate_transaction_fee += tx_fee
# Resume main logic
# if pymnt_amnt becomes < ZERO_THRESHOLD, don't pay
if pymnt_amnt < ZERO_THRESHOLD:
payment_item.paid = PaymentStatus.DONE
payment_item.delegator_transaction_fee = 0
payment_item.delegate_transaction_fee = 0
payment_item.desc += (
"Payment amount < ZERO_THRESHOLD after substracting fees. "
)
logger.info(
"Payment to {:s} became < {:<,d} mutez after deducting fees. Skipping."
.format(payment_item.paymentaddress, ZERO_THRESHOLD))
continue
else:
logger.debug(
"Payment to {:s} became {:<,d} mutez after deducting fees."
.format(payment_item.paymentaddress, pymnt_amnt))
op_counter.inc()
total_gas += int(gas_limit)
total_burn_fees += int(burn_fee)
total_tx_fees += int(tx_fee)
content = (CONTENT.replace("%SOURCE%", self.source).replace(
"%DESTINATION%", payment_item.paymentaddress).replace(
"%AMOUNT%", str(pymnt_amnt)).replace(
"%COUNTER%", str(op_counter.get())).replace(
"%fee%",
str(tx_fee)).replace("%gas_limit%",
str(gas_limit)).replace(
"%storage_limit%",
str(storage_limit)))
content_list.append(content)
verbose_logger.info("Payment content: {}".format(content))
if len(content_list) == 0:
return PaymentStatus.DONE, None, ""
contents_string = ",".join(content_list)
# run the operations for simulation results
logger.debug("Running {} operations".format(len(content_list)))
runops_json = RUNOPS_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%", contents_string)
runops_json = JSON_WRAP.replace("%JSON%", runops_json).replace(
"%chain_id%", chain_id)
status, run_ops_parsed = self.clnt_mngr.request_url_post(
self.comm_runops, runops_json)
if status != HTTPStatus.OK:
error_message = "Error in run_operation"
logger.error(error_message)
return PaymentStatus.FAIL, None, error_message
# Check each contents object for failure
for op in run_ops_parsed["contents"]:
# https://docs.python.org/3/glossary.html#term-eafp
try:
op_status = op["metadata"]["operation_result"]["status"]
if op_status == "failed":
op_error = op["metadata"]["operation_result"]["errors"][0][
"id"]
error_message = "Error while validating operation - Status: {}, Message: {}".format(
op_status, op_error)
logger.error(error_message)
return PaymentStatus.FAIL, None, error_message
except KeyError:
logger.debug(
"Unable to find metadata->operation_result->{status,errors} in run_ops response"
)
# forge the operations
logger.debug("Forging {} operations".format(len(content_list)))
forge_json = FORGE_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%", contents_string)
status, bytes = self.clnt_mngr.request_url_post(
self.comm_forge, forge_json)
if status != HTTPStatus.OK:
error_message = "Error in forge operation"
logger.error(error_message)
return PaymentStatus.FAIL, None, error_message
# Re-compute minimal required fee by the batch transaction and re-adjust the fee if necessary
size = SIGNATURE_BYTES_SIZE + len(bytes) / 2
required_fee = math.ceil(MINIMUM_FEE_MUTEZ +
MUTEZ_PER_GAS_UNIT * total_gas +
MUTEZ_PER_BYTE * size)
logger.info(
f"minimal required fee is {required_fee}, current used fee is {total_tx_fees}"
)
# TODO: This should be a function to be more modular as it is called twice
# If all fees are computed correctly above, the condition of this loop should not be True
# It is still recommended to leave this block here in order to double-check that all fee calculations
# were verified and that in the worst case any tiny differences in fee computations are adjusted
while total_tx_fees < required_fee:
# The difference in fees will be added to the fee of the first transaction
# This works because the Tezos blockchain is interested in the sum of all fees in a batch transaction
# and not in the individual fees of each transaction
difference_fees = math.ceil(required_fee - total_tx_fees)
first_tx = json.loads(content_list[0])
first_tx["fee"] = str(int(first_tx["fee"]) + difference_fees)
# We do not want to adjust the content (payment amount) anymore and let the delegate pay this fee
# TODO: Log info in description?
payment_items[0].delegate_transaction_fee += difference_fees
# Re-adjust the contents according to the new fee
total_tx_fees = required_fee
content_list[0] = json.dumps(first_tx)
contents_string = ",".join(content_list)
forge_json = FORGE_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%", contents_string)
status, bytes = self.clnt_mngr.request_url_post(
self.comm_forge, forge_json)
if status != HTTPStatus.OK:
error_message = "Error in forge operation"
logger.error(error_message)
return PaymentStatus.FAIL, None, error_message
# Compute the new required fee. It is possible that the size of the transaction in bytes is now higher
# because of the increase in the fee of the first transaction
size = SIGNATURE_BYTES_SIZE + len(bytes) / 2
required_fee = math.ceil(MINIMUM_FEE_MUTEZ +
MUTEZ_PER_GAS_UNIT * total_gas +
MUTEZ_PER_BYTE * size)
logger.info(
f"minimal required fee is {required_fee}, current used fee is {total_tx_fees}"
)
# Sign the batch transaction
signed_bytes = self.clnt_mngr.sign(bytes, self.manager)
# pre-apply operations
logger.debug("Preapplying the operations")
preapply_json = (PREAPPLY_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%",
contents_string).replace("%PROTOCOL%",
protocol).replace("%SIGNATURE%",
signed_bytes))
# if verbose: print("--> preapply_command_str is |{}|".format(preapply_command_str))
status, preapply_command_response = self.clnt_mngr.request_url_post(
self.comm_preapply, preapply_json)
if status != HTTPStatus.OK:
error_message = "Error in preapply operation"
logger.error(error_message)
return PaymentStatus.FAIL, None, error_message
# if dry_run, skip injection
if dry_run:
return PaymentStatus.DONE, None, ""
# inject the operations
logger.debug("Injecting {} operations".format(len(content_list)))
decoded = base58.b58decode(signed_bytes).hex()
if signed_bytes.startswith("edsig"): # edsig signature
decoded_edsig_signature = decoded[
10:][:-8] # first 5 bytes edsig, last 4 bytes checksum
decoded_signature = decoded_edsig_signature
elif signed_bytes.startswith("sig"): # generic signature
decoded_sig_signature = decoded[
6:][:-8] # first 3 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
elif signed_bytes.startswith("spsig"):
decoded_sig_signature = decoded[
10:][:-8] # first 5 bytes spsig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
elif signed_bytes.startswith("p2sig"):
decoded_sig_signature = decoded[
8:][:-8] # first 4 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
else:
raise Exception("Signature '{}' is not in expected format".format(
signed_bytes))
if len(decoded_signature) != 128: # must be 64 bytes
logger.warn(
"Signature length must be 128 but it is {}. Signature is '{}'".
format(len(signed_bytes), signed_bytes))
signed_operation_bytes = bytes + decoded_signature
_, head = self.clnt_mngr.request_url(self.comm_head)
last_level_before_injection = head["header"]["level"]
status, operation_hash = self.clnt_mngr.request_url_post(
self.comm_inject, json.dumps(signed_operation_bytes))
if status != HTTPStatus.OK:
error_message = "Error in inject operation"
logger.error(error_message)
return PaymentStatus.FAIL, None, error_message
logger.info("Operation hash is {}".format(operation_hash))
# wait for inclusion
timeout = (MAX_BLOCKS_TO_CHECK_AFTER_INJECTION *
MAX_NUM_TRIALS_PER_BLOCK *
self.network_config["MINIMAL_BLOCK_DELAY"])
logger.info(
"Waiting for operation {} to be included... Please do not interrupt the process! (Timeout is around {} minutes)"
.format(operation_hash, timeout))
for i in range(
last_level_before_injection + 1,
last_level_before_injection + 1 +
MAX_BLOCKS_TO_CHECK_AFTER_INJECTION,
):
cmd = self.comm_wait.replace("%BLOCK_HASH%", str(i))
status = -1
list_op_hash = []
trial_i = 0
while status != HTTPStatus.OK and (trial_i <
MAX_NUM_TRIALS_PER_BLOCK):
sleep(self.network_config["MINIMAL_BLOCK_DELAY"])
status, list_op_hash = self.clnt_mngr.request_url(cmd)
trial_i += 1
if status != HTTPStatus.OK:
logger.warning(
"Level {} could not be queried about operation hashes".
format(i))
break
for op_hashes in list_op_hash:
if operation_hash in op_hashes:
logger.info(
"Operation {} is included".format(operation_hash))
return PaymentStatus.PAID, operation_hash, ""
logger.debug("Operation {} is not included at level {}".format(
operation_hash, i))
error_message = (
"Investigate on https://tzkt.io - Operation {} wait is timed out.".
format(operation_hash))
logger.warning(error_message)
return PaymentStatus.INJECTED, operation_hash, error_message
def attempt_single_batch(self, payment_records, op_counter, dry_run=None):
if not op_counter.get():
status, counter = self.clnt_mngr.request_url(self.comm_counter)
if status != HTTPStatus.OK:
raise Exception(
"Received response code {} for request '{}'".format(
status, self.comm_counter
)
)
counter = int(counter)
self.base_counter = int(counter)
op_counter.set(self.base_counter)
_, head = self.clnt_mngr.request_url(self.comm_payment_head)
branch = head["hash"]
chain_id = head["chain_id"]
protocol = head["metadata"]["protocol"]
logger.debug(
"head: branch {} counter {} protocol {}".format(
branch, op_counter.get(), protocol
)
)
content_list = []
total_gas = total_fees = 0
for payment_item in payment_records:
pymnt_amnt = payment_item.amount # expected in micro tez
storage_limit, gas_limit, tx_fee = (
self.storage_limit,
self.gas_limit,
self.default_fee,
)
# TRD extension for non scriptless contract accounts
if payment_item.paymentaddress.startswith("KT"):
simulation_status, simulation_results = self.simulate_single_operation(
payment_item, pymnt_amnt, branch, chain_id
)
if simulation_status == PaymentStatus.FAIL:
logger.info(
"Payment to {} script could not be processed. Possible reason: liquidated contract. Skipping. Think about redirecting the payout to the owner address using the maps rules. Please refer to the TRD documentation or to one of the TRD maintainers.".format(
payment_item.paymentaddress
)
)
payment_item.paid = PaymentStatus.AVOIDED
continue
gas_limit, tx_fee, storage_limit = simulation_results
burn_fee = COST_PER_BYTE * storage_limit
total_fee = tx_fee + burn_fee
if KT1_FEE_SAFETY_CHECK:
if total_fee > FEE_LIMIT_CONTRACTS:
logger.info(
"Payment to {:s} script requires higher fees than reward amount. Skipping. Needed fee: {:10.6f} XTZ, max fee: {:10.6f} XTZ. Either configure a higher fee or redirect to the owner address using the maps rules. Refer to the TRD documentation.".format(
payment_item.paymentaddress,
total_fee / MUTEZ,
FEE_LIMIT_CONTRACTS / MUTEZ,
)
)
payment_item.paid = PaymentStatus.AVOIDED
continue
if total_fee > pymnt_amnt:
logger.info(
"Payment to {:s} requires fees of {:10.6f} higher than payment amount of {:10.6f}."
"Payment avoided due KT1_FEE_SAFETY_CHECK set to True.".format(
payment_item.paymentaddress,
total_fee / MUTEZ,
pymnt_amnt / MUTEZ,
)
)
payment_item.paid = PaymentStatus.AVOIDED
continue
# Subtract burn fee from the payment amount
orig_pymnt_amnt = pymnt_amnt
pymnt_amnt = max(pymnt_amnt - burn_fee, 0) # ensure not less than 0
logger.info(
"Payment to {} script requires {:.0f} gas * {:.2f} mutez-per-gas + {:10.6f} burn fee; Payment reduced from {:10.6f} to {:10.6f}".format(
payment_item.paymentaddress,
gas_limit,
MUTEZ_PER_GAS_UNIT,
burn_fee / MUTEZ,
orig_pymnt_amnt / MUTEZ,
pymnt_amnt / MUTEZ,
)
)
else:
# An implicit tz1 account
if payment_item.needs_activation:
storage_limit += RA_STORAGE
if self.delegator_pays_ra_fee:
# Subtract reactivation fee from the payment amount
orig_pymnt_amnt = pymnt_amnt
pymnt_amnt = max(
pymnt_amnt - RA_BURN_FEE, 0
) # ensure not less than 0
logger.info(
"Payment to {:s} reduced from {:>10.6f} to {:>10.6f} due to reactivation fee".format(
payment_item.address,
orig_pymnt_amnt / MUTEZ,
pymnt_amnt / MUTEZ,
)
)
# Subtract transaction's fee from the payment amount if delegator has to pay for it
if self.delegator_pays_xfer_fee:
pymnt_amnt = max(pymnt_amnt - tx_fee, 0) # ensure not less than 0
# Resume main logic
# if pymnt_amnt becomes 0, don't pay
if pymnt_amnt == 0:
payment_item.paid = PaymentStatus.DONE
logger.info(
"Payment to {:s} became 0 after deducting fees. Skipping.".format(
payment_item.paymentaddress
)
)
continue
else:
logger.debug(
"Payment to {:s} became {:10.6f} after deducting fees.".format(
payment_item.paymentaddress, pymnt_amnt / MUTEZ
)
)
op_counter.inc()
total_gas += int(gas_limit)
total_fees += int(tx_fee)
content = (
CONTENT.replace("%SOURCE%", self.source)
.replace("%DESTINATION%", payment_item.paymentaddress)
.replace("%AMOUNT%", str(pymnt_amnt))
.replace("%COUNTER%", str(op_counter.get()))
.replace("%fee%", str(tx_fee))
.replace("%gas_limit%", str(gas_limit))
.replace("%storage_limit%", str(storage_limit))
)
content_list.append(content)
verbose_logger.info("Payment content: {}".format(content))
if len(content_list) == 0:
return PaymentStatus.DONE, ""
contents_string = ",".join(content_list)
# run the operations
logger.debug("Running {} operations".format(len(content_list)))
runops_json = RUNOPS_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%", contents_string
)
runops_json = JSON_WRAP.replace("%JSON%", runops_json).replace(
"%chain_id%", chain_id
)
status, run_ops_parsed = self.clnt_mngr.request_url_post(
self.comm_runops, runops_json
)
if status != HTTPStatus.OK:
logger.error("Error in run_operation")
return PaymentStatus.FAIL, ""
# Check each contents object for failure
for op in run_ops_parsed["contents"]:
# https://docs.python.org/3/glossary.html#term-eafp
try:
op_status = op["metadata"]["operation_result"]["status"]
if op_status == "failed":
op_error = op["metadata"]["operation_result"]["errors"][0]["id"]
logger.error(
"Error while validating operation - Status: {}, Message: {}".format(
op_status, op_error
)
)
return PaymentStatus.AVOIDED, ""
except KeyError:
logger.debug(
"Unable to find metadata->operation_result->{status,errors} in run_ops response"
)
# forge the operations
logger.debug("Forging {} operations".format(len(content_list)))
forge_json = FORGE_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%", contents_string
)
# if verbose: print("--> forge_command_str is |{}|".format(forge_command_str))
status, bytes = self.clnt_mngr.request_url_post(self.comm_forge, forge_json)
if status != HTTPStatus.OK:
logger.error("Error in forge operation")
return PaymentStatus.FAIL, ""
size = SIGNATURE_BYTES_SIZE + len(bytes) / 2
required_fee = math.ceil(
MINIMUM_FEE_MUTEZ + MUTEZ_PER_GAS_UNIT * total_gas + MUTEZ_PER_BYTE * size
)
logger.info(
f"minimal required fee is {required_fee}, current used fee is {total_fees}"
)
while total_fees < required_fee:
difference_fees = int(math.ceil(required_fee - total_fees))
first_tx = json.loads(content_list[0])
first_tx["fee"] = str(int(int(first_tx["fee"]) + difference_fees))
total_fees = required_fee
content_list[0] = json.dumps(first_tx)
contents_string = ",".join(content_list)
forge_json = FORGE_JSON.replace("%BRANCH%", branch).replace(
"%CONTENT%", contents_string
)
status, bytes = self.clnt_mngr.request_url_post(self.comm_forge, forge_json)
if status != HTTPStatus.OK:
logger.error("Error in forge operation")
return PaymentStatus.FAIL, ""
size = SIGNATURE_BYTES_SIZE + len(bytes) / 2
required_fee = math.ceil(
MINIMUM_FEE_MUTEZ
+ MUTEZ_PER_GAS_UNIT * total_gas
+ MUTEZ_PER_BYTE * size
)
logger.info(
f"minimal required fee is {required_fee}, current used fee is {total_fees}"
)
signed_bytes = self.clnt_mngr.sign(bytes, self.manager)
# pre-apply operations
logger.debug("Preapplying the operations")
preapply_json = (
PREAPPLY_JSON.replace("%BRANCH%", branch)
.replace("%CONTENT%", contents_string)
.replace("%PROTOCOL%", protocol)
.replace("%SIGNATURE%", signed_bytes)
)
# if verbose: print("--> preapply_command_str is |{}|".format(preapply_command_str))
status, preapply_command_response = self.clnt_mngr.request_url_post(
self.comm_preapply, preapply_json
)
if status != HTTPStatus.OK:
logger.error("Error in preapply operation")
return PaymentStatus.FAIL, ""
# if dry_run, skip injection
if dry_run:
return PaymentStatus.DONE, ""
# inject the operations
logger.debug("Injecting {} operations".format(len(content_list)))
decoded = base58.b58decode(signed_bytes).hex()
if signed_bytes.startswith("edsig"): # edsig signature
decoded_edsig_signature = decoded[10:][
:-8
] # first 5 bytes edsig, last 4 bytes checksum
decoded_signature = decoded_edsig_signature
elif signed_bytes.startswith("sig"): # generic signature
decoded_sig_signature = decoded[6:][
:-8
] # first 3 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
elif signed_bytes.startswith("p2sig"):
decoded_sig_signature = decoded[8:][
:-8
] # first 4 bytes sig, last 4 bytes checksum
decoded_signature = decoded_sig_signature
else:
raise Exception(
"Signature '{}' is not in expected format".format(signed_bytes)
)
if len(decoded_signature) != 128: # must be 64 bytes
# raise Exception("Signature length must be 128 but it is {}. Signature is '{}'".format(len(signed_bytes), signed_bytes))
logger.warn(
"Signature length must be 128 but it is {}. Signature is '{}'".format(
len(signed_bytes), signed_bytes
)
)
# return False, ""
signed_operation_bytes = bytes + decoded_signature
_, head = self.clnt_mngr.request_url(self.comm_head)
last_level_before_injection = head["header"]["level"]
status, operation_hash = self.clnt_mngr.request_url_post(
self.comm_inject, json.dumps(signed_operation_bytes)
)
if status != HTTPStatus.OK:
logger.error("Error in inject operation")
return PaymentStatus.FAIL, ""
logger.info("Operation hash is {}".format(operation_hash))
# wait for inclusion
timeout = (
MAX_BLOCKS_TO_CHECK_AFTER_INJECTION
* MAX_NUM_TRIALS_PER_BLOCK
* self.network_config["MINIMAL_BLOCK_DELAY"]
)
logger.info(
"Waiting for operation {} to be included... Please do not interrupt the process! (Timeout is around {} minutes)".format(
operation_hash, timeout
)
)
for i in range(
last_level_before_injection + 1,
last_level_before_injection + 1 + MAX_BLOCKS_TO_CHECK_AFTER_INJECTION,
):
cmd = self.comm_wait.replace("%BLOCK_HASH%", str(i))
status = -1
list_op_hash = []
trial_i = 0
while status != HTTPStatus.OK and (trial_i < MAX_NUM_TRIALS_PER_BLOCK):
sleep(self.network_config["MINIMAL_BLOCK_DELAY"])
status, list_op_hash = self.clnt_mngr.request_url(cmd)
trial_i += 1
if status != HTTPStatus.OK:
logger.warning(
"Level {} could not be queried about operation hashes".format(i)
)
break
for op_hashes in list_op_hash:
if operation_hash in op_hashes:
logger.info("Operation {} is included".format(operation_hash))
return PaymentStatus.PAID, operation_hash
logger.debug(
"Operation {} is not included at level {}".format(operation_hash, i)
)
logger.warning(
"Operation {} wait is timed out. Not sure about the result!".format(
operation_hash
)
)
return PaymentStatus.INJECTED, operation_hash