def transaction_already_sent(self, did, did_request, memo):
rows = Didtx.objects(did=did, status="Processing")
if rows:
return rows[0]
rows = Didtx.objects(did=did, status="Pending")
if rows:
row = rows[0]
row.memo = memo
row.didRequest = did_request
row.save()
return row
return None
def transaction_already_sent(self, did, did_request, memo):
rows = Didtx.objects(did=did)
if rows:
for row in rows:
# Only check transactions that are in Pending state
if row.status == config.SERVICE_STATUS_PENDING:
# Check if header is the same(whether create or update operation)
if row.didRequest["header"] == did_request["header"]:
# Check if payload is the same(the info to be published)
if row.didRequest["payload"] == did_request["payload"]:
# Check if memo is the same. If not, just update the row with the new memo passed
if row.memo != memo:
row.memo = memo
row.save()
else:
# If payload is not the same, update the row with new didRequest
row.didRequest = did_request
row.save()
else:
# If header is not the same, update the row with new didRequest
row.didRequest = did_request
row.save()
return row
# If another transaction for this DID is already Processing, return it because we
# don't want to create a new request without that first being processed successfully
elif row.status == config.SERVICE_STATUS_PROCESSING:
return row
return None
def on_get(self, req, res, did):
LOG.info(f'Enter /v2/didtx/recent/did/{did}')
rows = Didtx.objects(did=did.replace("did:elastos:", "").split("#")
[0]).order_by('-modified')[:5]
if rows:
obj = [each.as_dict() for each in rows]
self.on_success(res, obj)
else:
LOG.info(f"Error /v2/didtx/recent/did/{did}")
raise NotFoundError()
def on_get(self, req, res, confirmation_id):
LOG.info(f'Enter /v2/didtx/confirmation_id/{confirmation_id}')
try:
rows = Didtx.objects(id=confirmation_id)
if rows:
row = [each.as_dict() for each in rows][0]
self.on_success(res, row)
else:
LOG.info(f"Error /v2/didtx/id/{confirmation_id}")
raise NotFoundError()
except Exception as e:
LOG.info(f"Error /v2/didtx/id/{confirmation_id}: {str(e)}")
raise NotFoundError()
def on_post(self, req, res):
LOG.info(f'Enter /v2/didtx/create')
data = req.media
did_request = data["didRequest"]
memo = data["memo"]
did_request_payload = did_request["payload"]
did_request_payload = did_request_payload + "=" * divmod(
len(did_request_payload), 4)[1]
did_request_payload = json.loads(
base64.urlsafe_b64decode(did_request_payload))
did_request_did = did_request_payload["id"].replace(
"did:elastos:", "").split("#")[0]
try:
caller_did = data["did"].replace("did:elastos:", "").split("#")[0]
# Verify whether the DID who's making the call, is valid
did_sidechain_rpc = DidSidechainRpcV2()
did_resolver_result = did_sidechain_rpc.resolve_did(caller_did)
if not did_resolver_result:
err_message = f"Invalid DID: {caller_did}"
LOG.info(f"Error /v2/didtx/create: {err_message}")
raise UserNotExistsError(description=err_message)
except:
LOG.info(
f"Info /v2/didtx/create: Defaulting to DID found inside didRequest payload"
)
caller_did = did_request_did
# First verify whether this is a valid payload
did_publish = Web3DidAdapter()
tx, err_message = did_publish.create_transaction(
config.WALLETSV2[0], 1, did_request)
if err_message:
err_message = f"Could not generate a valid transaction out of the given didRequest. Error Message: {err_message}"
LOG.info(f"Error /v2/didtx/create: {err_message}")
raise InvalidParameterError(description=err_message)
# Check the number of times this did has used the "did_publish" service
count = self.retrieve_service_count(caller_did,
config.SERVICE_DIDPUBLISH)
count_did_request_did = self.retrieve_service_count(
did_request_did, config.SERVICE_DIDPUBLISH)
if count_did_request_did > count:
count = count_did_request_did
result = {}
# Check if the row already exists with the same didRequest
transaction_sent = self.transaction_already_sent(
caller_did, did_request, memo)
if transaction_sent:
result["duplicate"] = True
result["service_count"] = count
result["confirmation_id"] = str(transaction_sent.id)
else:
result["service_count"] = self.retrieve_service_count(
caller_did, config.SERVICE_DIDPUBLISH)
result["duplicate"] = False
# If less than limit, increment and allow, otherwise, not allowed as max limit is reached
if count < config.SERVICE_DIDPUBLISH_DAILY_LIMIT:
row = Didtx(did=caller_did,
requestFrom=data["requestFrom"],
didRequestDid=did_request_did,
didRequest=did_request,
memo=memo,
version="2",
status=config.SERVICE_STATUS_PENDING)
row.save()
result["confirmation_id"] = str(row.id)
self.add_service_count_record(caller_did,
config.SERVICE_DIDPUBLISH)
self.add_service_count_record(did_request_did,
config.SERVICE_DIDPUBLISH)
else:
LOG.info(
f"Error /v2/didtx/create: Daily limit reached for this DID"
)
raise DailyLimitReachedError()
self.on_success(res, result)
def cron_send_tx_to_did_sidechain():
LOG.info('Started cron job: send_tx_to_did_sidechain')
# Verify the DID sidechain is reachable
response = did_sidechain_rpc.get_block_count()
if not (response and response["result"]):
LOG.info("DID sidechain is currently not reachable...")
return
current_height = response["result"] - 1
# Retrieve the current height from the database
rows = Didstate.objects()
if rows:
row = rows[0]
# Verify whether a new block has been added since last time
if current_height > row.currentHeight:
row.currentHeight = current_height
row.save()
else:
LOG.info(
"There hasn't been any new block since last cron job was run..."
)
return
else:
row = Didstate(currentHeight=current_height)
row.save()
pending_transactions = []
current_time = datetime.utcnow().strftime("%a, %b %d, %Y @ %I:%M:%S %p")
slack_blocks = [{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ""
}
}, {
"type": "divider"
}, {
"type": "section",
"text": {
"type": "mrkdwn",
"text": ""
}
}, {
"type": "divider"
}]
try:
# Create raw transactions
rows_pending = Didtx.objects(status=config.SERVICE_STATUS_PENDING,
version="1")
for row in rows_pending:
time_since_created = datetime.utcnow() - row.created
if (time_since_created.total_seconds() / 60.0) > 60:
LOG.info(
f"The id '{row.id}' with DID '{row.did}' has been in Pending state for the last hour. Changing "
f"its state to Cancelled")
row.status = config.SERVICE_STATUS_CANCELLED
row.extraInfo = {
"reason": "Was in pending state for more than 1 hour"
}
row.save()
continue
tx = did_publish.create_raw_transaction(row.did, row.didRequest)
if not tx:
continue
tx_decoded = binascii.hexlify(tx).decode(encoding="utf-8")
pending_transactions.append(tx_decoded)
if pending_transactions:
LOG.info("Pending: Found transactions. Sending " +
str(len(pending_transactions)) + " transactions to DID "
"sidechain now...")
# Send transaction to DID sidechain
response = did_sidechain_rpc.send_raw_transaction(
pending_transactions)
if not response:
return
tx_id = response["result"]
for row in rows_pending:
# If for whatever reason, the transactions fail, put them in quarantine and come back to it later
if tx_id:
row.status = config.SERVICE_STATUS_PROCESSING
row.blockchainTxId = tx_id
LOG.info(
"Pending: Successfully sent transaction from wallet: "
+ did_publish.wallets[did_publish.current_wallet_index]
["address"] + " to the blockchain for id: " +
str(row.id) + " DID: " + row.did + " tx_id: " + tx_id)
else:
row.extraInfo = response["error"]
row.status = config.SERVICE_STATUS_QUARANTINE
LOG.info(
"Pending: Error sending transaction from wallet: " +
did_publish.wallets[did_publish.current_wallet_index]
["address"] + " for id: " + str(row.id) + " DID:" +
row.did + " Error: " + str(row.extraInfo))
slack_blocks[0]["text"][
"text"] = f"The following transaction was sent to quarantine at {current_time}"
slack_blocks[2]["text"]["text"] = f"Wallet used: {did_publish.wallets[did_publish.current_wallet_index]['address']}\n" \
f"Transaction ID: {str(row.id)}\n" \
f"DID: {row.did}\n" \
f"Error: {str(row.extraInfo)}"
send_slack_notification(slack_blocks)
row.save()
# Get info about all the transactions and save them to the database
rows_processing = Didtx.objects(
status=config.SERVICE_STATUS_PROCESSING, version="1")
for row in rows_processing:
blockchain_tx = did_sidechain_rpc.get_raw_transaction(
row.blockchainTxId)
row.blockchainTx = blockchain_tx
LOG.info("Processing: Blockchain transaction info from wallet: " +
did_publish.wallets[
did_publish.current_wallet_index]["address"] +
" for id: " + str(row.id) + " DID:" + row.did)
if blockchain_tx["result"]:
confirmations = blockchain_tx["result"]["confirmations"]
if confirmations > 2 and row.status != config.SERVICE_STATUS_COMPLETED:
row.status = config.SERVICE_STATUS_COMPLETED
row.blockchainTx["result"]["confirmations"] = "2+"
row.save()
# Try to process quarantined transactions one at a time
rows_quarantined = Didtx.objects(
status=config.SERVICE_STATUS_QUARANTINE, version="1")
binary_split_resend(rows_quarantined)
except Exception as err:
message = "Error: " + str(err) + "\n"
exc_type, exc_obj, exc_tb = sys.exc_info()
message += "Unexpected error: " + str(exc_type) + "\n"
message += ' File "' + exc_tb.tb_frame.f_code.co_filename + '", line ' + str(
exc_tb.tb_lineno) + "\n"
slack_blocks[0]["text"][
"text"] = f"Error while sending tx to the blockchain at {current_time}"
slack_blocks[2]["text"]["text"] = f"Wallet used: {did_publish.wallets[did_publish.current_wallet_index]['address']}\n" \
f"Error: {message}"
send_slack_notification(slack_blocks)
LOG.info(f"Error while running cron job: {message}")
LOG.info('Completed cron job: send_tx_to_did_sidechain')
def cron_send_daily_stats_v2():
LOG.info('Started cron job: cron_send_daily_stats_v2')
to_email = config.EMAIL["SENDER"]
subject = "Assist Backend Daily Stats"
if config.NETWORK == "testnet":
subject = "Assist Backend Daily Stats - Testnet"
current_time = datetime.utcnow().strftime("%a, %b %d, %Y @ %I:%M:%S %p")
slack_blocks = [{
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"These are the daily stats for Assist for {current_time}"
}
}, {
"type": "divider"
}]
wallets_stats = "<table><tr><th>Address</th><th>Balance</th><th>Type</th></tr>"
# Used for testing purposes
test_address = "0x365b70f14e10b02bef7e463eca6aa3e75ca3cdb1"
test_balance = "{:.4f}".format(did_sidechain_rpc.get_balance(test_address))
wallets_stats += f"<tr><td>{test_address}</td><td>{test_balance}</td><td>Testing</td></tr>"
slack_blocks.append({
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
f"*Wallets and Current Balances*\n {test_address} | {test_balance} | Testing\n"
}
})
for wallet in config.WALLETSV2:
address = json.loads(wallet)["address"]
balance = "{:.4f}".format(
did_sidechain_rpc.get_balance(f"0x{address}"))
wallets_stats += f"<tr><td>0x{address}</td><td>{balance}</td><td>Production</td></tr>"
slack_blocks[2]["text"][
"text"] += f"{address} | {balance} | Production\n"
wallets_stats += "</table>"
service_stats = "<table><tr><th>Service</th><th>Users</th><th>Today</th><th>All time</th></tr>"
slack_blocks.append({
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*Service Stats*\n"
}
})
for service, stats in get_service_count().items():
service_stats += f"<tr><td>{service}</td><td>{stats['users']}</td><td>{stats['today']}</td><td>{stats['total']}</td></tr>"
slack_blocks[3]["text"][
"text"] += f"{service} | {stats['users']} total users | {stats['today']} tx today | {stats['total']} tx total\n"
service_stats += "</table>"
didtx_stats = "<table><tr><th>Application</th><th>Today</th><th>All time</th></tr>"
slack_blocks.append({
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*DID Transactions*\n"
}
})
didtx_by_app = get_didtx_count()
for app in didtx_by_app["total"].keys():
didtx_stats += f"<tr><td>{app}</td><td>{didtx_by_app['today'].get(app, 0)}</td><td>{didtx_by_app['total'].get(app, 0)}</td></tr>"
slack_blocks[4]["text"][
"text"] += f"{app} | {didtx_by_app['today'].get(app, 0)} tx today | {didtx_by_app['total'].get(app, 0)} tx total\n"
didtx_stats += "</table>"
quarantined_transactions = "<table><tr><th>Transaction ID</th><th>DID</th><th>From</th><th>Extra " \
"Info</th><th>Created</th></tr>"
slack_blocks.append({
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"*Quarantined Transactions*\n"
}
})
for transaction in Didtx.objects(status=config.SERVICE_STATUS_QUARANTINE,
version='2'):
id = transaction.id
did = transaction.did
request_from = transaction.requestFrom
created = transaction.created
extra_info = json.dumps(transaction.extraInfo)
quarantined_transactions += f"<tr><td>{id}</td><td>{did}</td><td>{request_from}</td><td>{extra_info}</td><td>{created}</td></tr>"
slack_blocks[5]["text"][
"text"] += f"{id} | {did} | {request_from} | {extra_info} | {created}\n"
quarantined_transactions += "</table>"
slack_blocks.append({"type": "divider"})
content_html = f"""
<html>
<head>
<style>
table {{
font-family: arial, sans-serif;
border-collapse: collapse;
width: 100%;
}}
td, th {{
border: 1px solid #dddddd;
text-align: left;
padding: 8px;
}}
tr:nth-child(even) {{
background-color: #dddddd;
}}
</style>
</head>
<body>
<h2>These are the daily stats for Assist for {current_time}</h1>
<h3>Wallets and Current Balances</h3>
{wallets_stats}
<h3>Service Stats</h3>
{service_stats}
<h3>DID Transactions</h3>
{didtx_stats}
<h3>Quarantined Transactions</h3>
{quarantined_transactions}
</body>
</html>
"""
if config.PRODUCTION:
send_email(to_email, subject, content_html)
send_slack_notification(slack_blocks)
cron_reset_didpublish_daily_limit()
LOG.info('Completed cron job: cron_send_daily_stats')
def cron_send_tx_to_did_sidechain_v2():
current_time = datetime.utcnow().strftime("%a, %b %d, %Y @ %I:%M:%S %p")
LOG.info('Started cron job: cron_send_tx_to_did_sidechain_v2')
try:
# Verify the DID sidechain is reachable
response = did_sidechain_rpc.get_block_count()
if not response:
LOG.info("DID sidechain is currently not reachable...")
return
rows_pending = Didtx.objects(status=config.SERVICE_STATUS_PENDING,
version='2')
rows_processing = Didtx.objects(
status=config.SERVICE_STATUS_PROCESSING, version='2')
LOG.info(f"rows pending {len(rows_pending)}")
current_height = response - 1
# Retrieve the current height from the database
rows = Didstate.objects()
if rows:
row = rows[0]
# Verify whether a new block has been added since last time
if current_height > row.currentHeightv2:
if len(rows_pending) == 0 and len(rows_processing) == 0:
LOG.info('Completed cron job: send_tx_to_did_sidechain_v2')
return
row.currentHeightv2 = current_height
row.save()
else:
LOG.info(
"There hasn't been any new block since last cron job was run..."
)
return
else:
row = Didstate(currentHeight=0, currentHeightv2=0)
row.save()
return
slack_blocks = [{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ""
}
}, {
"type": "divider"
}, {
"type": "section",
"text": {
"type": "mrkdwn",
"text": ""
}
}, {
"type": "divider"
}]
# Create raw transactions
if len(rows_pending) > config.NUM_WALLETSV2:
rows_pending = rows_pending[:config.NUM_WALLETSV2]
wallets = config.WALLETSV2
for index, row in enumerate(list(rows_pending)):
process_pending_tx(wallets[index], row, slack_blocks, current_time)
rows_processing = Didtx.objects(
status=config.SERVICE_STATUS_PROCESSING, version='2')
LOG.info(f"rows processing {len(rows_processing)}")
pool = multiprocessing.Pool()
for row in rows_processing:
pool.apply_async(process_processing_tx,
args=(
row.id,
slack_blocks,
current_time,
))
pool.close()
pool.join()
LOG.info('Completed cron job: send_tx_to_did_sidechain_v2')
except Exception as err:
message = "Error: " + str(err) + "\n"
exc_type, exc_obj, exc_tb = sys.exc_info()
message += "Unexpected error: " + str(exc_type) + "\n"
message += ' File "' + exc_tb.tb_frame.f_code.co_filename + '", line ' + str(
exc_tb.tb_lineno) + "\n"
LOG.info(f"Error while running cron job: {message}")
slack_blocks[0]["text"][
"text"] = f"Error while sending tx to the blockchain at {current_time}"
slack_blocks[2]["text"]["text"] = f"Error: {message}"
send_slack_notification(slack_blocks)