async def test_tx_race_condition(self):
"""Test race condition caused when job2 replaces job1 and job1's clean up is close to be executed.
In this case job1's clean up was cleaning job2 instead.
"""
job1 = TxJob(TX1_DATA, timeout=10)
ret1 = self.manager.add_job(job1)
self.assertTrue(ret1)
# Wait until job1 is marked as timeout.
await self.advance(10)
self.assertEqual(job1.status, JobStatus.TIMEOUT)
self.assertNotIn(job1, self.manager.tx_queue)
self.assertIn(job1.uuid, self.manager.tx_jobs)
# We are 1 second away to cleanup the tx.
await self.advance(self.manager.TX_CLEAN_UP_INTERVAL - 1)
# Resubmit a similar job.
job2 = TxJob(TX1_DATA, timeout=10)
ret2 = self.manager.add_job(job2)
self.assertTrue(ret2)
self.assertIn(job2, self.manager.tx_queue)
self.assertIn(job2.uuid, self.manager.tx_jobs)
# Reach the cleanup time of job1.
await self.advance(2)
self.assertEqual(job2.status, JobStatus.ENQUEUED)
self.assertIn(job2, self.manager.tx_queue)
self.assertIn(job2.uuid, self.manager.tx_jobs)
# Job2 timeouts.
await self.advance(15)
self.assertEqual(job2.status, JobStatus.TIMEOUT)
def cancel_job(self, job: TxJob) -> None:
"""Cancel tx mining job."""
if job.status in JobStatus.get_after_mining_states():
raise ValueError('Job has already finished')
job.status = JobStatus.CANCELLED
self.tx_jobs.pop(job.uuid)
self.tx_queue.remove(job)
self.stop_mining_tx(job)
self.log.info('TxJob cancelled', job=job.to_dict())
def _job_timeout_if_possible(self, job: TxJob) -> None:
"""Stop mining a tx job because it timeout."""
if job.status in JobStatus.get_after_mining_states():
return
self.log.info('TxJob timeout', job=job.to_dict())
self.txs_timeout += 1
job.status = JobStatus.TIMEOUT
self.tx_queue.remove(job)
self.stop_mining_tx(job)
# Schedule to clean it up.
self.schedule_job_clean_up(job)
async def add_parents(self, job: TxJob) -> None:
"""Add tx parents to job, then enqueue it."""
job.status = JobStatus.GETTING_PARENTS
try:
parents: List[bytes] = await self.backend.get_tx_parents()
except Exception as e:
job.status = JobStatus.FAILED
job.message = 'Unhandled exception: {}'.format(e)
# Schedule to clean it up.
self.schedule_job_clean_up(job)
else:
job.set_parents(parents)
self.enqueue_tx_job(job)
def test_mining_tx_connection_lost(self):
conn1 = self._get_ready_miner('HVZjvL1FJ23kH3buGNuttVRsRKq66WHUVZ')
self.assertIsNotNone(conn1.current_job)
self.assertTrue(conn1.current_job.is_block)
self.assertEqual(0, conn1.current_job.height)
conn2 = self._get_ready_miner('HVZjvL1FJ23kH3buGNuttVRsRKq66WHUVZ')
self.assertIsNotNone(conn2.current_job)
self.assertTrue(conn2.current_job.is_block)
self.assertEqual(0, conn2.current_job.height)
job = TxJob(TX1_DATA)
ret = self.manager.add_job(job)
self.assertTrue(ret)
self.assertFalse(conn1.current_job.is_block)
self.assertEqual(conn1.current_job.tx_job, job)
self.assertEqual(conn2.current_job.tx_job, job)
# Miner 1 disconnects.
conn1.connection_lost(exc=None)
self.assertFalse(conn2.current_job.is_block)
self.assertEqual(conn2.current_job.tx_job, job)
# Miner 2 disconnects. Tx stays on the queue.
conn2.connection_lost(exc=None)
self.assertEqual(deque([job]), self.manager.tx_queue)
# Miner 3 connects. Tx is sent to the new miner.
conn3 = self._get_ready_miner('HVZjvL1FJ23kH3buGNuttVRsRKq66WHUVZ')
self.assertFalse(conn3.current_job.is_block)
self.assertEqual(conn3.current_job.tx_job, job)
def schedule_job_clean_up(self, job: TxJob) -> None:
"""Schedule to have a TxJob cleaned up."""
if job._cleanup_timer:
job._cleanup_timer.cancel()
loop = asyncio.get_event_loop()
job._cleanup_timer = loop.call_later(self.TX_CLEAN_UP_INTERVAL,
self._job_clean_up, job)
def test_one_miner_two_txs(self):
conn = self._get_ready_miner('HVZjvL1FJ23kH3buGNuttVRsRKq66WHUVZ')
self.assertIsNotNone(conn.current_job)
self.assertTrue(conn.current_job.is_block)
self.assertEqual(0, conn.current_job.height)
job1 = TxJob(TX1_DATA)
job2 = TxJob(TX2_DATA)
ret1 = self.manager.add_job(job1)
ret2 = self.manager.add_job(job2)
self.assertFalse(conn.current_job.is_block)
self.assertEqual(conn.current_job.tx_job, job1)
self.assertTrue(ret1)
self.assertTrue(ret2)
# First submission: success
params = {
'job_id': conn.current_job.uuid.hex(),
'nonce': TX1_NONCE,
}
conn.send_error = MagicMock(return_value=None)
conn.send_result = MagicMock(return_value=None)
conn.method_submit(params=params, msgid=None)
conn.send_error.assert_not_called()
conn.send_result.assert_called_once_with(None, 'ok')
# Run loop and check that the miner gets the next tx
self._run_all_pending_events()
self.assertFalse(conn.current_job.is_block)
self.assertEqual(conn.current_job.tx_job, job2)
# First submission: success
params = {
'job_id': conn.current_job.uuid.hex(),
'nonce': TX2_NONCE,
}
conn.send_error = MagicMock(return_value=None)
conn.send_result = MagicMock(return_value=None)
conn.method_submit(params=params, msgid=None)
conn.send_error.assert_not_called()
conn.send_result.assert_called_once_with(None, 'ok')
# Run loop and check that the miner gets a block
self._run_all_pending_events()
self.assertTrue(conn.current_job.is_block)
self.assertEqual(0, conn.current_job.height)
async def test_tx_resubmit(self):
job1 = TxJob(TX1_DATA, timeout=10)
ret1 = self.manager.add_job(job1)
self.assertTrue(ret1)
# When a similar job is submitted, manager declines it.
job2 = TxJob(TX1_DATA)
ret2 = self.manager.add_job(job2)
self.assertFalse(ret2)
# Wait until job1 is marked as timeout.
await self.advance(15)
self.assertEqual(job1.status, JobStatus.TIMEOUT)
# Try to resubmit a similar job.
job3 = TxJob(TX1_DATA)
ret3 = self.manager.add_job(job3)
self.assertTrue(ret3)
def schedule_job_timeout(self, job: TxJob) -> None:
"""Schedule to have a TxJob marked as timeout."""
if job._timeout_timer:
job._timeout_timer.cancel()
if job.timeout is not None:
loop = asyncio.get_event_loop()
job._timeout_timer = loop.call_later(job.timeout,
self._job_timeout_if_possible,
job)
def enqueue_tx_job(self, job: TxJob) -> None:
"""Enqueue a tx job to be mined."""
assert job not in self.tx_queue
job.status = JobStatus.ENQUEUED
# When the total hashrate is unknown, `expected_mining_time` is set to -1. Thus, we need to
# skip negative values when calculating the `expected_queue_time`.
job.expected_queue_time = sum(x.expected_mining_time
for x in self.tx_queue
if x.expected_mining_time > 0)
self.tx_queue.append(job)
if job.timeout:
self.schedule_job_timeout(job)
if len(self.tx_queue) > 1:
# If the queue is not empty, do nothing.
return
for _, protocol in self.miners.items():
self.update_miner_job(protocol)
def add_job(self, job: TxJob) -> bool:
"""Add new tx to be mined."""
if job.uuid in self.tx_jobs:
prev_job = self.tx_jobs[job.uuid]
if prev_job.status == JobStatus.TIMEOUT:
self._job_clean_up(prev_job)
else:
return False
self.tx_jobs[job.uuid] = job
miners_hashrate_ghs = sum(x.hashrate_ghs for x in self.miners.values())
job.expected_mining_time = calculate_expected_mining_time(
miners_hashrate_ghs, job.get_weight())
self.log.info('New TxJob', job=job.to_dict())
if job.add_parents:
asyncio.ensure_future(self.add_parents(job))
else:
self.enqueue_tx_job(job)
return True
def test_no_miners_at_start(self):
from txstratum.constants import DEFAULT_EXPECTED_MINING_TIME
expected_queue_time = 0
job1 = TxJob(TX1_DATA)
self.assertTrue(self.manager.add_job(job1))
self.assertEqual(DEFAULT_EXPECTED_MINING_TIME, job1.expected_mining_time)
self.assertEqual(0, job1.expected_queue_time)
self.assertEqual(1, len(self.manager.tx_queue))
if DEFAULT_EXPECTED_MINING_TIME > 0:
expected_queue_time += DEFAULT_EXPECTED_MINING_TIME
job2 = TxJob(TX2_DATA)
self.assertTrue(self.manager.add_job(job2))
self.assertEqual(DEFAULT_EXPECTED_MINING_TIME, job2.expected_mining_time)
self.assertEqual(expected_queue_time, job2.expected_queue_time)
self.assertEqual(2, len(self.manager.tx_queue))
if DEFAULT_EXPECTED_MINING_TIME > 0:
expected_queue_time += DEFAULT_EXPECTED_MINING_TIME
job3 = TxJob(TOKEN_CREATION_TX_DATA)
self.assertTrue(self.manager.add_job(job3))
self.assertEqual(DEFAULT_EXPECTED_MINING_TIME, job3.expected_mining_time)
self.assertEqual(expected_queue_time, job3.expected_queue_time)
self.assertEqual(3, len(self.manager.tx_queue))
self.assertEqual([job1, job2, job3], list(self.manager.tx_queue))
# First miner connects and receives job1.
conn1 = self._get_ready_miner('HVZjvL1FJ23kH3buGNuttVRsRKq66WHUVZ')
self.assertIsNotNone(conn1.current_job)
self.assertEqual(job1, conn1.current_job.tx_job)
# Second miner connects and receives job1.
conn2 = self._get_ready_miner('HVZjvL1FJ23kH3buGNuttVRsRKq66WHUVZ')
self.assertIsNotNone(conn2.current_job)
self.assertEqual(job1, conn2.current_job.tx_job)
async def test_tx_timeout_and_cleanup(self):
job1 = TxJob(TX1_DATA, timeout=10)
ret1 = self.manager.add_job(job1)
self.assertTrue(ret1)
self.assertIn(job1, self.manager.tx_queue)
self.assertIn(job1.uuid, self.manager.tx_jobs)
# Wait until job1 is marked as timeout.
await self.advance(15)
self.assertEqual(job1.status, JobStatus.TIMEOUT)
self.assertNotIn(job1, self.manager.tx_queue)
self.assertIn(job1.uuid, self.manager.tx_jobs)
# Wait until job1 is cleared.
await self.advance(self.manager.TX_CLEAN_UP_INTERVAL)
self.assertNotIn(job1, self.manager.tx_queue)
self.assertNotIn(job1.uuid, self.manager.tx_jobs)
def _run_basic_tx_tests(self, conn, tx_data, tx_nonce):
job = TxJob(tx_data)
ret = self.manager.add_job(job)
self.assertFalse(conn.current_job.is_block)
self.assertEqual(conn.current_job.tx_job, job)
self.assertTrue(ret)
# First submission: wrong nonce
params = {
'job_id': conn.current_job.uuid.hex(),
'nonce': '84e20000',
}
conn.send_error = MagicMock(return_value=None)
conn.send_result = MagicMock(return_value=None)
conn.method_submit(params=params, msgid=None)
conn.send_error.assert_called_once_with(None, conn.INVALID_SOLUTION)
conn.send_result.assert_not_called()
self.assertFalse(conn.current_job.is_block)
# Second submission: success
params = {
'job_id': conn.current_job.uuid.hex(),
'nonce': tx_nonce,
}
conn.send_error = MagicMock(return_value=None)
conn.send_result = MagicMock(return_value=None)
conn.method_submit(params=params, msgid=None)
conn.send_error.assert_not_called()
conn.send_result.assert_called_once_with(None, 'ok')
# Third submission: stale
conn.send_error = MagicMock(return_value=None)
conn.send_result = MagicMock(return_value=None)
conn.method_submit(params=params, msgid=None)
conn.send_error.assert_called_once_with(None, conn.STALE_JOB, ANY)
conn.send_result.assert_not_called()
async def submit_job(self, request: web.Request) -> web.Response:
"""Submit a new tx job to the manager.
Method: POST
Format: json
Params:
- tx: str, hex dump of the transaction
- propagate: bool, propagate tx to Hathor’s full node after it is solved
- add_parents: bool, add parents before resolving the tx
"""
try:
data = await request.json()
except json.decoder.JSONDecodeError:
return web.json_response({'error': 'cannot-decode-json'},
status=400)
if not isinstance(data, dict):
return web.json_response({'error': 'json-must-be-an-object'},
status=400)
tx_hex = data.get('tx')
if not tx_hex:
return web.json_response({'error': 'missing-tx'}, status=400)
try:
tx_bytes = bytes.fromhex(tx_hex)
tx = tx_or_block_from_bytes(tx_bytes)
except (ValueError, TxValidationError):
return web.json_response({'error': 'invalid-tx'}, status=400)
if not isinstance(tx, (Transaction, TokenCreationTransaction)):
return web.json_response({'error': 'invalid-tx'}, status=400)
if tx.weight > self.max_tx_weight:
self.log.debug('tx-weight-is-too-high', data=data)
return web.json_response({'error': 'tx-weight-is-too-high'},
status=400)
for txout in tx.outputs:
if len(txout.script) > self.max_output_script_size:
self.log.debug('txout-script-is-too-big', data=data)
return web.json_response({'error': 'txout-script-is-too-big'},
status=400)
if self.only_standard_script:
p2pkh = P2PKH.parse_script(txout.script)
if p2pkh is None:
return web.json_response(
{'error': 'txout-non-standard-script'}, status=400)
now = txstratum.time.time()
if abs(tx.timestamp - now) > self.max_timestamp_delta:
if self.fix_invalid_timestamp:
tx.timestamp = int(now)
tx_bytes = bytes(tx)
else:
return web.json_response({'error': 'tx-timestamp-invalid'},
status=400)
if 'timeout' not in data:
timeout = self.tx_timeout
else:
try:
timeout = min(self.tx_timeout, float(data['timeout']))
except ValueError:
return web.json_response({'error': 'invalid-timeout'},
status=400)
if timeout <= 0:
return web.json_response({'error': 'invalid-timeout'},
status=400)
add_parents = data.get('add_parents', False)
propagate = data.get('propagate', False)
job = TxJob(tx_bytes,
add_parents=add_parents,
propagate=propagate,
timeout=timeout)
success = self.manager.add_job(job)
if not success:
return web.json_response({'error': 'job-already-exists'},
status=400)
return web.json_response(job.to_dict())