def check_create2_deployed_safes_task() -> None:
"""
Check if create2 safes were deployed and store the `blockNumber` if there are enough confirmations
"""
try:
redis = RedisRepository().redis
with redis.lock('tasks:check_create2_deployed_safes_task', blocking_timeout=1, timeout=LOCK_TIMEOUT):
ethereum_client = EthereumClientProvider()
confirmations = 6
current_block_number = ethereum_client.current_block_number
for safe_creation2 in SafeCreation2.objects.pending_to_check():
tx_receipt = ethereum_client.get_transaction_receipt(safe_creation2.tx_hash)
safe_address = safe_creation2.safe.address
if tx_receipt:
block_number = tx_receipt.blockNumber
if (current_block_number - block_number) >= confirmations:
logger.info('Safe=%s with tx-hash=%s was confirmed in block-number=%d',
safe_address, safe_creation2.tx_hash, block_number)
send_create_notification.delay(safe_address, safe_creation2.owners)
safe_creation2.block_number = block_number
safe_creation2.save()
else:
# If safe was not included in any block after 35 minutes (mempool limit is 30), we try to deploy it again
if safe_creation2.modified + timedelta(minutes=35) < timezone.now():
logger.info('Safe=%s with tx-hash=%s was not deployed after 10 minutes',
safe_address, safe_creation2.tx_hash)
safe_creation2.tx_hash = None
safe_creation2.save()
deploy_create2_safe_task.delay(safe_address, retry=False)
for safe_creation2 in SafeCreation2.objects.not_deployed():
deploy_create2_safe_task.delay(safe_creation2.safe.address, retry=False)
except LockError:
pass
def add_status_and_index_to_txs(apps, schema_editor):
EthereumTx = apps.get_model('relay', 'EthereumTx')
ethereum_client = EthereumClientProvider()
for ethereum_tx in EthereumTx.objects.filter(status=None):
tx_receipt = ethereum_client.get_transaction_receipt(ethereum_tx.tx_hash)
if tx_receipt:
ethereum_tx.status = tx_receipt.get('status')
ethereum_tx.transaction_index = tx_receipt['transactionIndex']
ethereum_tx.save(update_fields=['status', 'transaction_index'])
def add_status_and_index_to_txs(apps, schema_editor):
EthereumTx = apps.get_model("relay", "EthereumTx")
ethereum_client = EthereumClientProvider()
for ethereum_tx in EthereumTx.objects.filter(status=None):
tx_receipt = ethereum_client.get_transaction_receipt(ethereum_tx.tx_hash)
if tx_receipt:
ethereum_tx.status = tx_receipt.get("status")
ethereum_tx.transaction_index = tx_receipt["transactionIndex"]
ethereum_tx.save(update_fields=["status", "transaction_index"])
def check_deployer_funded_task(self, safe_address: str, retry: bool = True) -> None:
"""
Check the `deployer_funded_tx_hash`. If receipt can be retrieved, in SafeFunding `deployer_funded=True`.
If not, after the number of retries `deployer_funded_tx_hash=None`
:param safe_address: safe account
:param retry: if True, retries are allowed, otherwise don't retry
"""
try:
redis = RedisRepository().redis
with redis.lock(f"tasks:check_deployer_funded_task:{safe_address}", blocking_timeout=1, timeout=LOCK_TIMEOUT):
ethereum_client = EthereumClientProvider()
logger.debug('Starting check deployer funded task for safe=%s', safe_address)
safe_funding = SafeFunding.objects.get(safe=safe_address)
deployer_funded_tx_hash = safe_funding.deployer_funded_tx_hash
if safe_funding.deployer_funded:
logger.warning('Tx-hash=%s for safe %s is already checked', deployer_funded_tx_hash, safe_address)
return
elif not deployer_funded_tx_hash:
logger.error('No deployer_funded_tx_hash for safe=%s', safe_address)
return
logger.debug('Checking safe=%s deployer tx-hash=%s', safe_address, deployer_funded_tx_hash)
if ethereum_client.get_transaction_receipt(deployer_funded_tx_hash):
logger.info('Found transaction to deployer of safe=%s with receipt=%s', safe_address,
deployer_funded_tx_hash)
safe_funding.deployer_funded = True
safe_funding.save()
else:
logger.debug('Not found transaction receipt for tx-hash=%s', deployer_funded_tx_hash)
# If no more retries
if not retry or (self.request.retries == self.max_retries):
safe_creation = SafeCreation.objects.get(safe=safe_address)
balance = ethereum_client.get_balance(safe_creation.deployer)
if balance >= safe_creation.wei_deploy_cost():
logger.warning('Safe=%s. Deployer=%s. Cannot find transaction receipt with tx-hash=%s, '
'but balance is there. This should never happen',
safe_address, safe_creation.deployer, deployer_funded_tx_hash)
safe_funding.deployer_funded = True
safe_funding.save()
else:
logger.error('Safe=%s. Deployer=%s. Transaction receipt with tx-hash=%s not mined after %d '
'retries. Setting `deployer_funded_tx_hash` back to `None`',
safe_address,
safe_creation.deployer,
deployer_funded_tx_hash,
self.request.retries)
safe_funding.deployer_funded_tx_hash = None
safe_funding.save()
else:
logger.debug('Retry finding transaction receipt %s', deployer_funded_tx_hash)
if retry:
raise self.retry(countdown=self.request.retries * 10 + 15) # More countdown every retry
except LockError:
logger.info('check_deployer_funded_task is locked for safe=%s', safe_address)
def create_or_update_from_tx_hash(self, tx_hash: str) -> 'EthereumTx':
ethereum_client = EthereumClientProvider()
try:
ethereum_tx = self.get(tx_hash=tx_hash)
# For txs stored before being mined
if ethereum_tx.block is None:
tx_receipt = ethereum_client.get_transaction_receipt(tx_hash)
ethereum_tx.block = EthereumBlock.objects.get_or_create_from_block_number(
tx_receipt['blockNumber'])
ethereum_tx.gas_used = tx_receipt['gasUsed']
ethereum_tx.status = tx_receipt.get('status')
ethereum_tx.transaction_index = tx_receipt['transactionIndex']
ethereum_tx.save(update_fields=[
'block', 'gas_used', 'status', 'transaction_index'
])
return ethereum_tx
except self.model.DoesNotExist:
tx_receipt = ethereum_client.get_transaction_receipt(tx_hash)
ethereum_block = EthereumBlock.objects.get_or_create_from_block_number(
tx_receipt['blockNumber'])
tx = ethereum_client.get_transaction(tx_hash)
return self.create_from_tx_dict(tx,
tx_receipt=tx_receipt,
ethereum_block=ethereum_block)
def handle(self, *args, **options):
queryset = EthereumTx.objects.filter(Q(block=None) | Q(gas_used=None))
total = queryset.count()
self.stdout.write(self.style.SUCCESS(f'Fixing ethereum txs. {total} remaining to be fixed'))
ethereum_client = EthereumClientProvider()
for i, ethereum_tx in enumerate(queryset.iterator()):
tx_receipt = ethereum_client.get_transaction_receipt(ethereum_tx.tx_hash)
if not ethereum_tx.block:
ethereum_tx.block = EthereumBlock.objects.get_or_create_from_block_number(tx_receipt['blockNumber'])
ethereum_tx.gas_used = tx_receipt['gasUsed']
ethereum_tx.status = tx_receipt.get('status')
ethereum_tx.transaction_index = tx_receipt['transactionIndex']
ethereum_tx.save(update_fields=['block', 'gas_used', 'status', 'transaction_index'])
self.stdout.write(self.style.SUCCESS(f'Processing {i}/{total} with tx-hash={ethereum_tx.tx_hash}'))
self.stdout.write(self.style.SUCCESS(f'End fixing txs. {total} have been fixed'))
def deploy_safes_task(retry: bool = True) -> None:
"""
Deploy pending safes (deployer funded and tx-hash checked). Then raw creation tx is sent to the ethereum network.
If something goes wrong (maybe a reorg), `deployer_funded` will be set False again and `check_deployer_funded_task`
is called again.
:param retry: if True, retries are allowed, otherwise don't retry
"""
try:
redis = RedisRepository().redis
with redis.lock("tasks:deploy_safes_task", blocking_timeout=1, timeout=LOCK_TIMEOUT):
ethereum_client = EthereumClientProvider()
logger.debug('Starting deploy safes task')
pending_to_deploy = SafeFunding.objects.pending_just_to_deploy()
logger.debug('%d safes pending to deploy', len(pending_to_deploy))
for safe_funding in pending_to_deploy:
safe_contract = safe_funding.safe
safe_address = safe_contract.address
safe_creation = SafeCreation.objects.get(safe=safe_contract)
safe_deployed_tx_hash = safe_funding.safe_deployed_tx_hash
if not safe_deployed_tx_hash:
# Deploy the Safe
try:
creation_tx_hash = ethereum_client.send_raw_transaction(safe_creation.signed_tx)
if creation_tx_hash:
creation_tx_hash = creation_tx_hash.hex()
logger.info('Safe=%s creation tx has just been sent to the network with tx-hash=%s',
safe_address, creation_tx_hash)
safe_funding.safe_deployed_tx_hash = creation_tx_hash
safe_funding.save()
except TransactionAlreadyImported:
logger.warning("Safe=%s transaction was already imported by the node", safe_address)
safe_funding.safe_deployed_tx_hash = safe_creation.tx_hash
safe_funding.save()
except ValueError:
# Usually "ValueError: {'code': -32000, 'message': 'insufficient funds for gas*price+value'}"
# A reorg happened
logger.warning("Safe=%s was affected by reorg, let's check again receipt for tx-hash=%s",
safe_address, safe_funding.deployer_funded_tx_hash, exc_info=True)
safe_funding.deployer_funded = False
safe_funding.save()
check_deployer_funded_task.apply_async((safe_address,), {'retry': retry}, countdown=20)
else:
# Check if safe proxy deploy transaction has already been sent to the network
logger.debug('Safe=%s creation tx has already been sent to the network with tx-hash=%s',
safe_address, safe_deployed_tx_hash)
if ethereum_client.check_tx_with_confirmations(safe_deployed_tx_hash,
settings.SAFE_FUNDING_CONFIRMATIONS):
logger.info('Safe=%s was deployed', safe_funding.safe.address)
safe_funding.safe_deployed = True
safe_funding.save()
# Send creation notification
send_create_notification.delay(safe_address, safe_creation.owners)
elif (safe_funding.modified + timedelta(minutes=10) < timezone.now()
and not ethereum_client.get_transaction_receipt(safe_deployed_tx_hash)):
# A reorg happened
logger.warning('Safe=%s deploy tx=%s was not found after 10 minutes. Trying deploying again...',
safe_funding.safe.address, safe_deployed_tx_hash)
safe_funding.safe_deployed_tx_hash = None
safe_funding.save()
except LockError:
pass
class IndexService:
def __init__(self, ethereum_client: EthereumClient, eth_reorg_blocks: int):
self.ethereum_client = ethereum_client
self.eth_reorg_blocks = eth_reorg_blocks
def block_get_or_create_from_block_number(self, block_number: int):
try:
return EthereumBlock.get(number=block_number)
except EthereumBlock.DoesNotExist:
current_block_number = self.ethereum_client.current_block_number # For reorgs
block = self.ethereum_client.get_block(block_number)
confirmed = (current_block_number - block['number']) >= self.eth_reorg_blocks
return EthereumBlock.objects.create_from_block(block, cofirmed=confirmed)
def tx_create_or_update_from_tx_hash(self, tx_hash: str) -> 'EthereumTx':
try:
ethereum_tx = EthereumTx.objects.get(tx_hash=tx_hash)
# For txs stored before being mined
if ethereum_tx.block is None:
tx_receipt = self.ethereum_client.get_transaction_receipt(tx_hash)
ethereum_block = self.block_get_or_create_from_block_number(tx_receipt['blockNumber'])
ethereum_tx.update_with_block_and_receipt(ethereum_block, tx_receipt)
return ethereum_tx
except EthereumTx.DoesNotExist:
tx_receipt = self.ethereum_client.get_transaction_receipt(tx_hash)
ethereum_block = self.block_get_or_create_from_block_number(tx_receipt['blockNumber'])
tx = self.ethereum_client.get_transaction(tx_hash)
return EthereumTx.objects.create_from_tx_dict(tx, tx_receipt=tx_receipt, ethereum_block=ethereum_block)
def txs_create_or_update_from_tx_hashes(self, tx_hashes: List[Union[str, bytes]]) -> List['EthereumTx']:
# Search first in database
ethereum_txs_dict = OrderedDict.fromkeys([HexBytes(tx_hash).hex() for tx_hash in tx_hashes])
db_ethereum_txs = EthereumTx.objects.filter(tx_hash__in=tx_hashes).exclude(block=None)
for db_ethereum_tx in db_ethereum_txs:
ethereum_txs_dict[db_ethereum_tx.tx_hash] = db_ethereum_tx
# Retrieve from the node the txs missing from database
tx_hashes_not_in_db = [tx_hash for tx_hash, ethereum_tx in ethereum_txs_dict.items() if not ethereum_tx]
if not tx_hashes_not_in_db:
return list(ethereum_txs_dict.values())
self.ethereum_client = EthereumClientProvider()
# Get receipts for hashes not in db
tx_receipts = []
for tx_hash, tx_receipt in zip(tx_hashes_not_in_db,
self.ethereum_client.get_transaction_receipts(tx_hashes_not_in_db)):
tx_receipt = tx_receipt or self.ethereum_client.get_transaction_receipt(tx_hash) # Retry fetching if failed
if not tx_receipt:
raise TransactionNotFoundException(f'Cannot find tx-receipt with tx-hash={HexBytes(tx_hash).hex()}')
elif tx_receipt.get('blockNumber') is None:
raise TransactionWithoutBlockException(f'Cannot find blockNumber for tx-receipt with '
f'tx-hash={HexBytes(tx_hash).hex()}')
else:
tx_receipts.append(tx_receipt)
# Get transactions for hashes not in db
txs = self.ethereum_client.get_transactions(tx_hashes_not_in_db)
block_numbers = set()
for tx_hash, tx in zip(tx_hashes_not_in_db, txs):
tx = tx or self.ethereum_client.get_transaction(tx_hash) # Retry fetching if failed
if not tx:
raise TransactionNotFoundException(f'Cannot find tx with tx-hash={HexBytes(tx_hash).hex()}')
elif tx.get('blockNumber') is None:
raise TransactionWithoutBlockException(f'Cannot find blockNumber for tx with '
f'tx-hash={HexBytes(tx_hash).hex()}')
block_numbers.add(tx['blockNumber'])
blocks = self.ethereum_client.get_blocks(block_numbers)
block_dict = {}
for block_number, block in zip(block_numbers, blocks):
block = block or self.ethereum_client.get_block(block_number) # Retry fetching if failed
if not block:
raise BlockNotFoundException(f'Block with number={block_number} was not found')
assert block_number == block['number']
block_dict[block['number']] = block
# Create new transactions or update them if they have no receipt
current_block_number = self.ethereum_client.current_block_number
for tx, tx_receipt in zip(txs, tx_receipts):
block = block_dict.get(tx['blockNumber'])
confirmed = (current_block_number - block['number']) >= self.eth_reorg_blocks
ethereum_block: EthereumBlock = EthereumBlock.objects.get_or_create_from_block(block, confirmed=confirmed)
if HexBytes(ethereum_block.block_hash) != block['hash']:
raise EthereumBlockHashMismatch(f'Stored block={ethereum_block.number} '
f'with hash={ethereum_block.block_hash} '
f'is not marching retrieved hash={block["hash"].hex()}')
try:
ethereum_tx = EthereumTx.objects.get(tx_hash=tx['hash'])
# For txs stored before being mined
ethereum_tx.update_with_block_and_receipt(ethereum_block, tx_receipt)
ethereum_txs_dict[ethereum_tx.tx_hash] = ethereum_tx
except EthereumTx.DoesNotExist:
ethereum_tx = EthereumTx.objects.create_from_tx_dict(tx,
tx_receipt=tx_receipt,
ethereum_block=ethereum_block)
ethereum_txs_dict[HexBytes(ethereum_tx.tx_hash).hex()] = ethereum_tx
return list(ethereum_txs_dict.values())
@transaction.atomic
def reindex_addresses(self, addresses: List[str]) -> NoReturn:
"""
Given a list of safe addresses it will delete all `SafeStatus`, conflicting `MultisigTxs` and will mark
every `InternalTxDecoded` not processed to be processed again
:param addresses: List of checksummed addresses or queryset
:return: Number of `SafeStatus` deleted
"""
if not addresses:
return
SafeStatus.objects.filter(address__in=addresses).delete()
MultisigTransaction.objects.exclude(
ethereum_tx=None
).filter(
safe__in=addresses
).delete() # Remove not indexed transactions
InternalTxDecoded.objects.filter(internal_tx___from__in=addresses).update(processed=False)
@transaction.atomic
def reindex_all(self) -> NoReturn:
MultisigConfirmation.objects.filter(signature=None).delete() # Remove onchain confirmations
MultisigTransaction.objects.exclude(ethereum_tx=None).delete() # Remove not indexed transactions
SafeStatus.objects.all().delete()
InternalTxDecoded.objects.update(processed=False)
def create_or_update_from_tx_hashes(
self, tx_hashes: List[Union[str, bytes]]) -> List['EthereumTx']:
# Search first in database
ethereum_txs_dict = OrderedDict.fromkeys(
[HexBytes(tx_hash).hex() for tx_hash in tx_hashes])
db_ethereum_txs = self.filter(tx_hash__in=tx_hashes).exclude(
block=None)
for db_ethereum_tx in db_ethereum_txs:
ethereum_txs_dict[db_ethereum_tx.tx_hash] = db_ethereum_tx
# Retrieve from the node the txs missing from database
tx_hashes_not_in_db = [
tx_hash for tx_hash, ethereum_tx in ethereum_txs_dict.items()
if not ethereum_tx
]
if not tx_hashes_not_in_db:
return list(ethereum_txs_dict.values())
ethereum_client = EthereumClientProvider()
tx_receipts = []
for tx_hash, tx_receipt in zip(
tx_hashes_not_in_db,
ethereum_client.get_transaction_receipts(tx_hashes_not_in_db)):
if not tx_receipt:
tx_receipt = ethereum_client.get_transaction_receipt(
tx_hash) # Retry fetching
if not tx_receipt:
raise TransactionNotFoundException(
f'Cannot find tx with tx-hash={HexBytes(tx_hash).hex()}')
elif tx_receipt.get('blockNumber') is None:
raise TransactionWithoutBlockException(
f'Cannot find block for tx with tx-hash={HexBytes(tx_hash).hex()}'
)
else:
tx_receipts.append(tx_receipt)
txs = ethereum_client.get_transactions(tx_hashes_not_in_db)
block_numbers = []
for tx_hash, tx in zip(tx_hashes_not_in_db, txs):
if not tx:
raise TransactionNotFoundException(
f'Cannot find tx with tx-hash={HexBytes(tx_hash).hex()}')
elif tx.get('blockNumber') is None:
raise TransactionWithoutBlockException(
f'Cannot find block for tx with tx-hash={HexBytes(tx_hash).hex()}'
)
block_numbers.append(tx['blockNumber'])
blocks = ethereum_client.get_blocks(block_numbers)
current_block_number = ethereum_client.current_block_number
for tx, tx_receipt, block in zip(txs, tx_receipts, blocks):
ethereum_block = EthereumBlock.objects.get_or_create_from_block(
block, current_block_number=current_block_number)
try:
ethereum_tx = self.get(tx_hash=tx['hash'])
# For txs stored before being mined
if ethereum_tx.block is None:
ethereum_tx.block = ethereum_block
ethereum_tx.gas_used = tx_receipt['gasUsed']
ethereum_tx.status = tx_receipt.get('status')
ethereum_tx.transaction_index = tx_receipt[
'transactionIndex']
ethereum_tx.save(update_fields=[
'block', 'gas_used', 'status', 'transaction_index'
])
ethereum_txs_dict[HexBytes(
ethereum_tx.tx_hash).hex()] = ethereum_tx
except self.model.DoesNotExist:
ethereum_tx = self.create_from_tx_dict(
tx, tx_receipt=tx_receipt, ethereum_block=ethereum_block)
ethereum_txs_dict[HexBytes(
ethereum_tx.tx_hash).hex()] = ethereum_tx
return list(ethereum_txs_dict.values())
class IndexService:
def __init__(
self,
ethereum_client: EthereumClient,
eth_reorg_blocks: int,
eth_l2_network: bool,
alert_out_of_sync_events_threshold: float,
):
self.ethereum_client = ethereum_client
self.eth_reorg_blocks = eth_reorg_blocks
self.eth_l2_network = eth_l2_network
self.alert_out_of_sync_events_threshold = alert_out_of_sync_events_threshold
def block_get_or_create_from_block_hash(self, block_hash: int):
try:
return EthereumBlock.objects.get(block_hash=block_hash)
except EthereumBlock.DoesNotExist:
current_block_number = (self.ethereum_client.current_block_number
) # For reorgs
block = self.ethereum_client.get_block(block_hash)
confirmed = (current_block_number -
block["number"]) >= self.eth_reorg_blocks
return EthereumBlock.objects.get_or_create_from_block(
block, confirmed=confirmed)
def is_service_synced(self) -> bool:
"""
:return: `True` if master copies and ERC20/721 are synced, `False` otherwise
"""
# Use number of reorg blocks to consider as not synced
reference_block_number = (self.ethereum_client.current_block_number -
self.eth_reorg_blocks)
synced = True
for safe_master_copy in SafeMasterCopy.objects.relevant().filter(
tx_block_number__lt=reference_block_number):
logger.error("Master Copy %s is out of sync",
safe_master_copy.address)
synced = False
out_of_sync_contracts = SafeContract.objects.filter(
erc20_block_number__lt=reference_block_number).count()
if out_of_sync_contracts > 0:
total_number_of_contracts = SafeContract.objects.all().count()
proportion_out_of_sync = out_of_sync_contracts / total_number_of_contracts
# Ignore less than 10% of contracts out of sync
if proportion_out_of_sync >= self.alert_out_of_sync_events_threshold:
logger.error(
"%d Safe Contracts have ERC20/721 out of sync",
out_of_sync_contracts,
)
synced = False
return synced
def tx_create_or_update_from_tx_hash(self, tx_hash: str) -> "EthereumTx":
try:
ethereum_tx = EthereumTx.objects.get(tx_hash=tx_hash)
# For txs stored before being mined
if ethereum_tx.block is None:
tx_receipt = self.ethereum_client.get_transaction_receipt(
tx_hash)
ethereum_block = self.block_get_or_create_from_block_hash(
tx_receipt["blockHash"])
ethereum_tx.update_with_block_and_receipt(
ethereum_block, tx_receipt)
return ethereum_tx
except EthereumTx.DoesNotExist:
tx_receipt = self.ethereum_client.get_transaction_receipt(tx_hash)
ethereum_block = self.block_get_or_create_from_block_hash(
tx_receipt["blockHash"])
tx = self.ethereum_client.get_transaction(tx_hash)
return EthereumTx.objects.create_from_tx_dict(
tx, tx_receipt=tx_receipt, ethereum_block=ethereum_block)
def txs_create_or_update_from_tx_hashes(
self, tx_hashes: Collection[Union[str,
bytes]]) -> List["EthereumTx"]:
# Search first in database
ethereum_txs_dict = OrderedDict.fromkeys(
[HexBytes(tx_hash).hex() for tx_hash in tx_hashes])
db_ethereum_txs = EthereumTx.objects.filter(
tx_hash__in=tx_hashes).exclude(block=None)
for db_ethereum_tx in db_ethereum_txs:
ethereum_txs_dict[db_ethereum_tx.tx_hash] = db_ethereum_tx
# Retrieve from the node the txs missing from database
tx_hashes_not_in_db = [
tx_hash for tx_hash, ethereum_tx in ethereum_txs_dict.items()
if not ethereum_tx
]
if not tx_hashes_not_in_db:
return list(ethereum_txs_dict.values())
self.ethereum_client = EthereumClientProvider()
# Get receipts for hashes not in db
tx_receipts = []
for tx_hash, tx_receipt in zip(
tx_hashes_not_in_db,
self.ethereum_client.get_transaction_receipts(
tx_hashes_not_in_db),
):
tx_receipt = tx_receipt or self.ethereum_client.get_transaction_receipt(
tx_hash) # Retry fetching if failed
if not tx_receipt:
raise TransactionNotFoundException(
f"Cannot find tx-receipt with tx-hash={HexBytes(tx_hash).hex()}"
)
if tx_receipt.get("blockHash") is None:
raise TransactionWithoutBlockException(
f"Cannot find blockHash for tx-receipt with "
f"tx-hash={HexBytes(tx_hash).hex()}")
tx_receipts.append(tx_receipt)
# Get transactions for hashes not in db
fetched_txs = self.ethereum_client.get_transactions(
tx_hashes_not_in_db)
block_hashes = set()
txs = []
for tx_hash, tx in zip(tx_hashes_not_in_db, fetched_txs):
tx = tx or self.ethereum_client.get_transaction(
tx_hash) # Retry fetching if failed
if not tx:
raise TransactionNotFoundException(
f"Cannot find tx with tx-hash={HexBytes(tx_hash).hex()}")
if tx.get("blockHash") is None:
raise TransactionWithoutBlockException(
f"Cannot find blockHash for tx with "
f"tx-hash={HexBytes(tx_hash).hex()}")
block_hashes.add(tx["blockHash"].hex())
txs.append(tx)
blocks = self.ethereum_client.get_blocks(block_hashes)
block_dict = {}
for block_hash, block in zip(block_hashes, blocks):
block = block or self.ethereum_client.get_block(
block_hash) # Retry fetching if failed
if not block:
raise BlockNotFoundException(
f"Block with hash={block_hash} was not found")
assert block_hash == block["hash"].hex()
block_dict[block["hash"]] = block
# Create new transactions or update them if they have no receipt
current_block_number = self.ethereum_client.current_block_number
for tx, tx_receipt in zip(txs, tx_receipts):
block = block_dict[tx["blockHash"]]
confirmed = (current_block_number -
block["number"]) >= self.eth_reorg_blocks
ethereum_block: EthereumBlock = (
EthereumBlock.objects.get_or_create_from_block(
block, confirmed=confirmed))
try:
with transaction.atomic():
ethereum_tx = EthereumTx.objects.create_from_tx_dict(
tx,
tx_receipt=tx_receipt,
ethereum_block=ethereum_block)
ethereum_txs_dict[HexBytes(
ethereum_tx.tx_hash).hex()] = ethereum_tx
except IntegrityError: # Tx exists
ethereum_tx = EthereumTx.objects.get(tx_hash=tx["hash"])
# For txs stored before being mined
ethereum_tx.update_with_block_and_receipt(
ethereum_block, tx_receipt)
ethereum_txs_dict[ethereum_tx.tx_hash] = ethereum_tx
return list(ethereum_txs_dict.values())
@transaction.atomic
def _reprocess(self, addresses: List[str]):
"""
Trigger processing of traces again. If addresses is empty, everything is reprocessed
:param addresses:
:return:
"""
queryset = MultisigConfirmation.objects.filter(signature=None)
if not addresses:
logger.info("Remove onchain confirmations")
queryset.delete()
logger.info("Remove transactions automatically indexed")
queryset = MultisigTransaction.objects.exclude(
ethereum_tx=None).filter(Q(origin=None) | Q(origin=""))
if addresses:
queryset = queryset.filter(safe__in=addresses)
queryset.delete()
logger.info("Remove module transactions")
queryset = ModuleTransaction.objects.all()
if addresses:
queryset = queryset.filter(safe__in=addresses)
queryset.delete()
logger.info("Remove Safe statuses")
queryset = SafeStatus.objects.all()
if addresses:
queryset = queryset.filter(address__in=addresses)
queryset.delete()
logger.info("Mark all internal txs decoded as not processed")
queryset = InternalTxDecoded.objects.all()
if addresses:
queryset = queryset.filter(internal_tx___from__in=addresses)
queryset.update(processed=False)
def reprocess_addresses(self, addresses: List[str]):
"""
Given a list of safe addresses it will delete all `SafeStatus`, conflicting `MultisigTxs` and will mark
every `InternalTxDecoded` not processed to be processed again
:param addresses: List of checksummed addresses or queryset
:return: Number of `SafeStatus` deleted
"""
if not addresses:
return None
return self._reprocess(addresses)
def reprocess_all(self):
return self._reprocess(None)
def reindex_master_copies(
self,
from_block_number: int,
to_block_number: Optional[int] = None,
block_process_limit: int = 100,
addresses: Optional[ChecksumAddress] = None,
):
"""
Reindexes master copies in parallel with the current running indexer, so service will have no missing txs
while reindexing
:param from_block_number: Block number to start indexing from
:param to_block_number: Block number to stop indexing on
:param block_process_limit: Number of blocks to process each time
:param addresses: Master Copy or Safes(for L2 event processing) addresses. If not provided,
all master copies will be used
"""
assert (not to_block_number) or to_block_number > from_block_number
from ..indexers import (
EthereumIndexer,
InternalTxIndexerProvider,
SafeEventsIndexerProvider,
)
indexer_provider = (SafeEventsIndexerProvider if self.eth_l2_network
else InternalTxIndexerProvider)
indexer: EthereumIndexer = indexer_provider()
ethereum_client = EthereumClientProvider()
if addresses:
indexer.IGNORE_ADDRESSES_ON_LOG_FILTER = (
False # Just process addresses provided
)
else:
addresses = list(
indexer.database_queryset.values_list("address", flat=True))
if not addresses:
logger.warning("No addresses to process")
else:
logger.info("Start reindexing Safe Master Copy addresses %s",
addresses)
current_block_number = ethereum_client.current_block_number
stop_block_number = (min(current_block_number, to_block_number)
if to_block_number else current_block_number)
block_number = from_block_number
while block_number < stop_block_number:
elements = indexer.find_relevant_elements(
addresses, block_number,
block_number + block_process_limit)
indexer.process_elements(elements)
block_number += block_process_limit
logger.info(
"Current block number %d, found %d traces/events",
block_number,
len(elements),
)
logger.info("End reindexing addresses %s", addresses)