def avg(*vals: NumberStr,
dp: OptNumStr = '8',
rounding: str = None) -> Decimal:
vals = [conv_dec(v) for v in list(vals) if not empty(v)]
if len(vals) == 1:
return vals[0] if empty(dp) else dec_round(vals[0], dp, rounding)
_avg = conv_dec(sum(vals)) / Decimal(len(vals))
return _avg if empty(dp) else dec_round(_avg, dp, rounding)
async def main():
ss = SteemAsync(HIVE_NODES)
ss.config_set('batch_size', BATCH_SIZE)
print(
f"\n [{datetime.utcnow()!s}] Loading last {NUM_BLOCKS} blocks using steem-async ... \n\n"
)
start_time = time.time_ns()
blks = await ss.get_blocks(-NUM_BLOCKS)
end_time = time.time_ns()
print(f"\n [{datetime.utcnow()!s}] Total blocks:", len(blks), "\n")
start_time, end_time = Decimal(start_time), Decimal(end_time)
start_secs = start_time / SECS_NS
end_secs = end_time / SECS_NS
print("Start Time:", dec_round(start_secs, 4), "seconds")
print("End Time:", dec_round(end_secs, 4), "seconds\n")
print("Total Time:", dec_round(end_secs - start_secs, 4), "seconds\n")
def med_avg(*vals: NumberStr,
dp: OptNumStr = '8',
rounding: str = None) -> Decimal:
"""
Standard median average.
If there are 3 or less values, the midpoint value will be returned.
If there are 4 or more values, the midpoint, and value before the midpoint will be added together, and then
divided by two to get the median average.
:param NumberStr vals: Two or more values to median average.
:param NumberStr dp: Decimal places to round to (can be ``None`` to disable rounding)
:param str rounding: Optional rounding method, e.g. ``ROUND_HALF_DOWN`` or ``ROUND_UP``
:return Decimal med_avg: The median average of ``vals``
"""
if len(vals) == 0:
raise ValueError(
"No values passed to med_avg... you must pass at least one number!"
)
dp = int(dp)
rate_vals = sorted(list([conv_dec(v) for v in vals]))
midpoint = int(len(rate_vals) // 2)
if len(rate_vals) == 1: return rate_vals[0]
if len(rate_vals) <= 3:
return rate_vals[midpoint] if empty(dp) else dec_round(
rate_vals[midpoint], dp, rounding)
# mavg = avg(rate_vals[midpoint - 1], rate_vals[midpoint], dp=dp, rounding=rounding)
# mavg = Decimal((rate_vals[midpoint - 1] + rate_vals[midpoint]) / Decimal('2'))
# return mavg if empty(dp) else dec_round(mavg, dp, rounding)
return avg(rate_vals[midpoint - 1],
rate_vals[midpoint],
dp=dp,
rounding=rounding)
def res_time(self) -> Optional[Decimal]:
if len(self.timing) > 0:
time_total = 0.0
for time_type, time in self.timing.items():
time_total += time
avg_res = time_total / len(self.timing)
return dec_round(Decimal('{:.2f}'.format(avg_res)), dp=2)
return None
def send(self,
amount: Decimal,
address: str,
from_address: str = None,
memo: str = None,
trigger_data: Union[dict, list] = None) -> dict:
# Try from_address first. If that's empty, try using self.coin.our_account. If both are empty, abort.
if empty(from_address):
if empty(self.coin.our_account):
raise AttributeError(
"Both 'from_address' and 'coin.our_account' are empty. Cannot send."
)
from_address = self.coin.our_account
if not self.address_valid(address):
raise exceptions.AccountNotFound(
f'Account "{address}" does not exist.')
if not self.address_valid(from_address):
raise exceptions.AccountNotFound(
f'Account "{address}" does not exist.')
memo = "" if empty(memo) else memo
prec = self.precision
sym = self.symbol.upper()
amount = dec_round(Decimal(amount), dp=prec, rounding=ROUND_DOWN)
if amount < Decimal(pow(10, -prec)):
log.warning(
'Amount %s was passed, but is lower than precision for %s',
amount, sym)
raise ArithmeticError(
'Amount {} is lower than token {}s precision of {} DP'.format(
amount, sym, prec))
bal = self.balance(from_address)
if bal < amount:
raise exceptions.NotEnoughBalance(
'Account {} has balance {} but needs {} to send this tx'.
format(from_address, bal, amount))
###
# Broadcast the transfer transaction on the network, and return the necessary data
###
log.info('Sending %f %s to @%s', amount, sym, address)
_, wif = self.get_priv(from_address)
t = self.rpc.transfer(to=address,
amount=amount,
from_account=from_address,
memo=memo,
asset=sym,
wif=wif)
return {
'txid': t['id'],
'coin': self.symbol,
'amount': amount,
'fee': Decimal(0),
'from': from_address,
'send_type': 'send'
}
async def main():
blocks = []
hive = Hive(HIVE_NODES)
chain = Blockchain(blockchain_instance=hive)
print(f"\n [{datetime.utcnow()!s}] Loading last {NUM_BLOCKS} blocks using beem ... \n\n")
start_time = time.time_ns()
current_num = chain.get_current_block_num()
for block in chain.blocks(start=current_num - NUM_BLOCKS, stop=current_num):
blocks.append(block)
end_time = time.time_ns()
print(f"\n [{datetime.utcnow()!s}] Total blocks:", len(blocks), "\n")
start_time, end_time = Decimal(start_time), Decimal(end_time)
start_secs = start_time / SECS_NS
end_secs = end_time / SECS_NS
print("Start Time:", dec_round(start_secs, 4), "seconds")
print("End Time:", dec_round(end_secs, 4), "seconds\n")
print("Total Time:", dec_round(end_secs - start_secs, 4), "seconds\n")
def conv_dec(obj: Optional[AnyNum], dp=4) -> Optional[Decimal]:
if empty(obj):
return None
try:
d = Decimal(obj)
return dec_round(d, dp=dp)
except Exception as e:
log.warning(
"Error while converting object '%s' to decimal. Reason: %s %s",
obj, type(e), str(e))
return None
def j_gen_tx(account="", address=None, amount=None, category=None, **kwargs):
"""
Generate a Bitcoin transaction and return it as a dict.
If any transaction attributes aren't specified, fake data will be automatically generated using :py:mod:`random` or
:py:mod:`faker` to fill the attributes.
:param account: Wallet account to label the transaction under
:param address: **Our** address, that we're sending from or receiving into.
:param amount: The amount of BTC transferred
:param category: Either ``'receive'`` or ``'send'``
:param kwargs: Any additional dict keys to put into the TX data
:return dict tx: The generated TX
"""
address = random.choice(
internal["addresses"]) if address is None else address
category = random.choice(['receive', 'send'
]) if category is None else category
amount = Decimal(random.random(), 7) if amount is None else Decimal(amount)
amount = dec_round(amount, dp=8)
# If an amount is being sent, then the amount becomes negative.
# If an amount is being received, the amount must be positive.
if (category == 'send' and amount > 0) or (category == 'receive'
and amount < 0):
amount = -amount
tx = dict(
account=account,
address=address,
amount=amount,
category=category,
)
tx = {**tx, **kwargs}
tx['txid'] = tx.get('txid', fake.sha256())
tx['confirmations'] = tx.get('confirmations', random.randint(1, 30))
if 'time' not in tx:
tx['time'] = int(
fake.unix_time(start_datetime=datetime.utcnow() -
timedelta(days=5)))
tx['label'] = tx.get('label', '')
tx['vout'] = tx.get('vout', 0)
tx['generated'] = is_true(tx.get('generated', False))
return tx
async def get_pair(self,
from_coin: str,
to_coin: str,
rate='last',
use_proxy: bool = True,
dp: int = 8):
if not self.loaded:
await self.load_exchanges()
pair = f"{from_coin.upper()}_{to_coin.upper()}"
if pair not in self.ex_pair_map:
if not use_proxy:
raise PairNotFound(
f"Pair {pair} was not found in ex_pair_map, and no proxying requested."
)
try:
log.debug(
"Pair %s not found - trying to find a proxy route...",
pair)
proxy = await self._find_proxy(from_coin, to_coin)
log.debug("Found proxy to %s via %s - trying proxy...",
to_coin, proxy)
rate_proxy = await self.try_proxy(from_coin,
to_coin,
proxy,
rate=rate)
log.debug("Got proxy rate: %f %s per %s", rate_proxy, to_coin,
from_coin)
return rate_proxy
except ProxyMissingPair:
raise PairNotFound(
f"Pair {pair} not found, nor a viable proxy route.")
adp = self.ex_pair_map[pair][0]
log.debug("Pair %s found - getting exchange rate directly", pair)
data = await adp.get_pair(from_coin, to_coin)
return dec_round(data[rate], dp=dp)
def __post_init__(self):
conv_bool_keys = ['available', 'is_premium_name']
conv_dec_keys = [
'premium_registration_price',
'premium_renewal_price',
'premium_restore_price',
'premium_transfer_price',
'icann_fee',
'eap_fee',
]
self.error_no = None if empty(self.error_no) else int(self.error_no)
for k in conv_bool_keys:
v = getattr(self, k, None)
if not empty(v):
setattr(self, k, is_true(v))
for k in conv_dec_keys:
v = getattr(self, k, None)
if empty(v):
log.debug("Skipping %s - is empty", k)
continue
log.debug("Converting %s to decimal", k)
setattr(self, k, dec_round(Decimal(v), dp=4))
async def sync_blocks(cls, start_block=None, start_type=None, **options):
lck = cls.lock_sync_blocks
log.info("Main sync_blocks loop started.")
try:
await cls.check_celery()
except (KeyboardInterrupt, CancelledError):
await cls.clean_import_threads()
return
except Exception:
log.exception(
'ERROR - Something went wrong checking Celery queue length.')
if not options['skip_gaps']:
await cls.fill_gaps()
if options['gaps_only']:
log.info('Requested gaps_only, not skipping blocks...')
return
end_block = options.pop('end_block')
relative_end = options.pop('relative_end', False)
if start_block is None:
start_block = settings.EOS_START_BLOCK
if EOSBlock.objects.count() > 0:
start_block = EOSBlock.objects.aggregate(
Max('number'))['number__max']
start_type = 'exact'
log.info(
'Found existing blocks. Starting from block %d (changed start_type to exact)',
start_block)
if end_block is not None and relative_end:
_end = int(end_block)
end_block = start_block + _end
with LockMgr(lck) as lm:
log.info('Obtained lock name %s', lck)
_start_block = settings.EOS_START_BLOCK if start_block is None else start_block
_start_block = int(_start_block)
start_type = settings.EOS_START_TYPE if start_type is None else start_type
_node = random.choice(settings.EOS_NODE)
log.info("Getting blockchain info from RPC node: %s", _node)
a = eos.Api(url=_node)
info = await a.get_info()
head_block = int(info['head_block_num'])
start_block = int(_start_block)
if start_type.lower() == 'relative':
start_block = head_block - int(_start_block)
if end_block is not None:
if end_block > head_block:
log.error(
"ERROR: End block '%d' is higher than actual head block '%d'. Cannot sync.",
end_block, head_block)
return
else:
end_block = head_block
current_block = int(start_block)
total_blocks = end_block - start_block
log.info(
"Importing blocks starting from %d - to end block %d. Total blocks to load: %d",
start_block, end_block, total_blocks)
time_start = timezone.now()
i = 0
while current_block < end_block:
lm.renew(expires=300, add_time=False)
blocks_left = end_block - current_block
if i > 0 and (i % 100 == 0 or current_block == end_block):
log.info('End block: %d // Head block: %d', end_block,
head_block)
log.info('Current block: %d', current_block)
log.info(
' >>> Queued %d blocks out of %d blocks to import.', i,
total_blocks)
log.info(' >>> %d blocks remaining. Progress: %f%%',
blocks_left,
dec_round(Decimal((i / total_blocks) * 100)))
time_taken = timezone.now() - time_start
time_taken_sec = time_taken.total_seconds()
log.info(' >>> Started at %s', time_start)
bps = Decimal(i / time_taken_sec)
eta_secs = blocks_left // bps
log.info(' >>> Estd. blocks per second %f', dec_round(bps))
log.info(' >>> Estd. finish in %f seconds //// %f hours',
eta_secs, eta_secs / 60 / 60)
log.info(' >>> Estd. finish date/time: %s',
timezone.now() + timedelta(seconds=int(eta_secs)))
_end = current_block + settings.EOS_SYNC_MAX_QUEUE
_end = end_block if _end > end_block else _end
blocks_queued = _end - current_block
try:
await cls.sync_between(current_block, _end, renew=lck)
await cls.clean_import_threads()
await asyncio.sleep(3)
except (KeyboardInterrupt, CancelledError):
log.error(
'CTRL-C detected. Please wait while threads terminate...'
)
await cls.clean_import_threads()
return
current_block += blocks_queued
i += blocks_queued
if not options['skip_gaps']:
log.info(
'============================================================================='
)
log.info(
'Finished syncing blocks. Waiting for Celery queue to empty completely,'
)
log.info('then filling any leftover gaps.')
log.info(
'============================================================================='
)
await cls.check_celery(max_queue=2)
await cls.fill_gaps()
await cls.check_celery(max_queue=2)
print(
"\n============================================================================================\n"
"\nFinished importing " + str(total_blocks) + " blocks!\n"
"\n============================================================================================\n"
)
