def insert(cls, params):
idx_start, idx_end = params
param_list = []
for index in range(idx_start, idx_end, cls.concurrency):
curr_batch_size = min(cls.concurrency, idx_end - index)
for i in range(0, curr_batch_size):
tx = blocksci.Tx(index + i, cls.chain)
param_list.append(tx_summary(tx))
results = execute_concurrent_with_args(session=cls.session,
statement=cls.prepared_stmt,
parameters=param_list,
concurrency=cls.concurrency)
for (i, (success, _)) in enumerate(results):
if not success:
while True:
try:
tx = blocksci.Tx(index + i, cls.chain)
cls.session.execute(cls.prepared_stmt,
tx_summary(tx))
except Exception as e:
print(e)
continue
break
param_list = []
with cls.counter.get_lock():
cls.counter.value += curr_batch_size
if (cls.counter.value % 1e4) == 0:
print(f'#tx {cls.counter.value:,.0f}')
def insert(cls, params):
idx_start, idx_end = params
batch_size = 500
batch_stmt = BatchStatement()
for index in range(idx_start, idx_end, batch_size):
curr_batch_size = min(batch_size, idx_end - index)
for i in range(0, curr_batch_size):
tx = blocksci.Tx(index + i, cls.chain)
batch_stmt.add(cls.prepared_stmt, tx_summary(tx))
try:
cls.session.execute(batch_stmt)
except Exception as e:
# ingest single transactions if batch ingest fails
# (batch too large error)
print(e)
for i in range(0, curr_batch_size):
while True:
try:
tx = blocksci.Tx(index + i, cls.chain)
cls.session.execute(cls.prepared_stmt,
tx_summary(tx))
except Exception as e:
print(e)
continue
break
batch_stmt.clear()
with cls.counter.get_lock():
cls.counter.value += curr_batch_size
print('#tx {:,.0f}'.format(cls.counter.value), end='\r')