def __post_init__(self):
if not self.tokens_locked and 'tokensLocked' in self.raw_data:
self.tokens_locked = self.raw_data.tokensLocked
self.tokens_locked = conv_dec(self.tokens_locked)
self.quantity, self.price = conv_dec(self.quantity), conv_dec(
self.price)
self.timestamp = convert_datetime(self.timestamp)
self.expiration = convert_datetime(self.expiration)
self.symbol = self.symbol.upper()
self.account = str(self.account).lower()
self.txid = self.raw_data.get(
'txId', self.txid) if not self.txid else str(self.txid)
def __post_init__(self):
if not self.direction and 'type' in self.raw_data:
self.direction = self.raw_data.get('type')
if self.direction not in ['buy', 'sell']:
raise AttributeError('SETrade.type must be either buy or sell')
self.timestamp = convert_datetime(self.timestamp)
self.quantity, self.price, self.volume = conv_dec(
self.quantity), conv_dec(self.price), conv_dec(self.volume)
def clean_txs(self, account: str, symbol: str, transactions: Iterable[SETransaction]) -> Generator[dict, None, None]:
"""
Filters a list of transactions by the receiving account, yields dict's conforming with
:class:`payments.models.Deposit`
:param str account: The 'to' account to filter by
:param str symbol: The symbol of the token being filtered
:param list<dict> transactions: A list<dict> of transactions to filter
:return: A generator yielding ``dict`` s conforming to :class:`payments.models.Deposit`
"""
for tx in transactions:
try:
log.debug("Cleaning SENG transaction: %s", tx)
if not tx.sender or not tx.to:
log.debug("SENG TX missing from/to - skipping")
continue
if tx.sender.lower() in ['tokens', 'market']:
log.debug("SENG TX from tokens/market - skipping")
continue # Ignore token issues and market transactions
if tx.to.lower() != account.lower():
log.debug("SENG TX is to account '%s' - but we're account '%s' - skipping", tx['to'].lower(), account.lower())
continue # If we aren't the receiver, we don't need it.
# Cache the token for 5 mins, so we aren't spamming the token API
token = cache.get_or_set('stmeng:'+symbol, lambda: self.get_rpc(symbol).get_token(symbol), 300)
q = tx.quantity
if type(q) == float:
q = ('{0:.' + str(token['precision']) + 'f}').format(tx.quantity)
_txid = tx.raw_data.get('txid', tx.raw_data.get('transactionId'))
clean_tx = dict(
txid=_txid, coin=self.coins[symbol].symbol, tx_timestamp=convert_datetime(tx['timestamp']),
from_account=tx.sender, to_account=tx.to, memo=tx.memo,
amount=Decimal(q)
)
yield clean_tx
except:
log.exception('Error parsing transaction data. Skipping this TX. tx = %s', tx)
continue
def host_sorter(
self,
host: str,
key: str,
fallback=USE_ORIG_VAR
) -> Union[str, int, float, bool, datetime, Decimal]:
"""
Usage::
>>> scanner = RPCScanner(['https://hived.privex.io', 'https://anyx.io', 'https://hived.hive-engine.com'])
>>> await scanner.scan_nodes()
>>> sorted(scanner.node_objs, key=lambda el: scanner.host_sorter(host=el.host, key='online_status'))
Useful notes about string sorting:
* ``!`` appears to be the most preferred (comes first when sorting) ASCII character
* ``~`` appears to be the least preferred (comes last when sorting) ASCII character
* Symbols are not linearly grouped together. Some symbols will be sorted first, some will be sorted after numbers,
some will be sorted after uppercase letters, and some will be sorted after lowercase letters.
* Uppercase and lowercase letters are not grouped together. As per the previous bulletpoint - both uppercase and
lowercase letters have symbols before + after their preference group.
* The Python string ASCII sorting order seems to be:
* Certain common symbols such as exclamation ``!``, quote ``"``, hash ``#``, dollar ``$`` and others
* Numbers ``0`` to ``9``
* Less common symbols such as colon ``:``, semi-colon ``;``, lessthan ``<``, equals ``=`` and greaterthan ``=`` (and more)
* Uppercase alphabet characters ``A`` to ``Z``
* Even less common symbols such as open square bracket ``[``, backslash ``\``, close square bracket ``]`` and others.
* Lowercase alphabet characters ``a`` to ``z``
* Rarer symbols such as curly braces ``{`` ``}``, pipe ``|``, tilde ``~`` and more.
The tilde '~' character appears to be one of the least favorable string characters, coming in last
place when I did some basic testing in the REPL on Python 3.8, with the following character set (sorted)::
>>> x = list('!"#$%&\\'()*+,-./0123456789:;<=>?@ABC[\\]^_`abc{|}~')
>>> x = list(set(x)) # Remove any potential duplicate characters
Tested with::
>>> print(''.join(sorted(list(set(x)), key=lambda el: str(el))))
!"#$%&'()*+,-./0123456789:;<=>?@ABC[\\]^_`abc{|}~
Note: extra backslashes have been added to the character set example above, due to IDEs thinking it's an escape
for the docs - and thus complaining.
:param str host: The host being sorted, e.g. ``https://hived.privex.io``
:param str key: The key being sorted / special sort code, e.g. ``head_block`` or ``online_status``
:param fallback:
:return:
"""
node = self.get_node(host)
key = self.table_sort_aliases.get(key, key)
real_key = str(key)
if key in ['api_tests', 'plugins']:
if empty(node.plugins, True, True): return 0
return len(node.plugins) / len(TEST_PLUGINS_LIST)
if '_status' in key:
content = node.status + 1
# When sorting by a specific status key, we handle out-of-sync nodes by simply emitting
# status 0 (best) if we're prioritising out-of-sync nodes, or status 2 (unreliable) when
# sorting by any other status.
if node.time_behind:
if node.time_behind.total_seconds() > 60:
return 0 if key == 'outofsync_status' else 2
if key == 'online_status' and node.status >= 3:
return 0
if key == 'dead_status' and node.status <= 0:
return 0
if key == 'outofsync_status': pass
return content
if key.endswith('_network'):
real_key = 'network'
if real_key not in node:
log.error(
f"RPCScanner.host_sorter called with non-existent key '{key}'. Falling back to sorting by 'status'."
)
key, real_key = 'status', 'status'
content = node.get(real_key, '')
strcont = str(content)
has_err = 'error' in strcont.lower() or 'none' in strcont.lower()
def_reverse = real_key in self.table_default_reverse
log.debug(
f"Key: {key} || Real Key: {real_key} has_err: {has_err} || def_reverse: {def_reverse} "
f"|| content: {content} || strcont: {strcont}")
# If a specific network sort type is given, then return '!' if this node matches that network.
# The exclamation mark symbol '!' is very high ranking with python string sorts (higher than numbers and letters)
if key == "hive_network":
return '!' if 'hive' in strcont.lower() else strcont
if key == "steem_network":
return '!' if 'steem' in strcont.lower() else strcont
if key == "whaleshares_network":
return '!' if 'whaleshares' in strcont.lower() else strcont
if key == "golos_network":
return '!' if 'golos' in strcont.lower() else strcont
# If 'table_types' tells us that the column we're sorting by - should be handled as a certain type,
# then we need to change how we handle the default fallback value for errors, and any casting
# we should use.
if key in self.table_types:
tt = self.table_types[key]
log.info(f"Key {key} has table type: {tt}")
if tt is bool:
if has_err or empty(content):
return False if def_reverse else True
return is_true(content)
if tt is datetime:
fallback = (datetime.utcnow() +
timedelta(weeks=260, hours=12)).replace(
tzinfo=pytz.UTC)
if def_reverse:
fallback = datetime(1980,
1,
1,
1,
1,
1,
1,
tzinfo=pytz.UTC)
if has_err or empty(content): return fallback
return convert_datetime(content, if_empty=fallback)
if tt is float:
if has_err and isinstance(fallback, float): return fallback
if has_err or empty(content):
return float(0.0) if def_reverse else float(
999999999.99999)
return float(content)
if tt is Decimal:
if has_err and isinstance(fallback, Decimal): return fallback
if has_err or empty(content):
return Decimal('0') if def_reverse else Decimal(
'999999999')
return Decimal(content)
if tt is int:
if has_err and isinstance(fallback, int): return fallback
if has_err or empty(content):
return int(0) if def_reverse else int(999999999)
return int(content)
if has_err or empty(content):
# We use the placeholder type 'USE_ORIG_VAR' instead of 'None' or 'False', allowing us the user to specify
# 'None', 'False', '""' etc. as fallbacks without conflict
if fallback is not USE_ORIG_VAR: return fallback
# The '!' character is used as the default fallback value if the table is reversed by default,
# since '!' appears to be the most preferred string character, and thus would be at the
# bottom of a reversed list.
if def_reverse: return '!'
# The tilde '~' character appears to be one of the least favorable string characters, coming in last
# place when I did some basic testing in the REPL on Python 3.8 (see pydoc block for this method),
# so it's used as the default for ``fallback``.
return '~'
# If we don't have a known type for this column, or any special handling needed like for 'api_tests', then we
# simply return the stringified content of the key on the node object.
return strcont
def time_behind(self) -> Optional[timedelta]:
if empty(self.block_time): return None
end = self.scanned_at if self.scanned_at else datetime.utcnow()
start = convert_datetime(self.block_time).replace(tzinfo=pytz.UTC)
end = end.replace(tzinfo=pytz.UTC)
return end - start
def time_date(self) -> datetime:
return convert_datetime(self.time / 1000) if isinstance(
self.time, int) else convert_datetime(self.time)
async def _store_path(self, p: SanePath) -> Prefix:
p_dic = dict(p)
# Obtain AS name via in-memory cache, database cache, or DNS lookup
await self.get_as_name(p.source_asn)
asn_id = p.source_asn
communities = list(p_dic['communities'])
del p_dic['family']
del p_dic['source_asn']
del p_dic['first_hop']
del p_dic['communities']
for x, nh in enumerate(p_dic['next_hops']):
p_dic['next_hops'][x] = Inet(nh)
async with self.pg_pool.acquire() as conn:
pfx = await conn.fetchrow(
"SELECT * FROM prefix WHERE asn_id = $1 AND prefix.prefix = $2 LIMIT 1;",
asn_id, p_dic['prefix']) # type: Record
# pfx = Prefix.query.filter_by(source_asn=asn_id, prefix=p_dic['prefix']).first() # type: Prefix
# new_pfx = dict(**p_dic, asn_id=asn_id, last_seen=datetime.utcnow())
age = convert_datetime(p_dic.get('age'),
fail_empty=False,
if_empty=None)
if age is not None:
age = age.replace(tzinfo=None)
if not pfx:
await conn.execute(
"INSERT INTO prefix ("
" asn_id, asn_path, prefix, next_hops, neighbor, ixp, last_seen, "
" age, created_at, updated_at"
")"
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);",
asn_id, p_dic.get('asn_path', []), p_dic['prefix'],
p_dic.get('next_hops', []), p_dic.get('neighbor'),
p_dic.get('ixp'), datetime.utcnow(), age,
datetime.utcnow(), datetime.utcnow())
pfx = await conn.fetchrow(
"SELECT * FROM prefix WHERE asn_id = $1 AND prefix.prefix = $2 LIMIT 1;",
asn_id, p_dic['prefix'])
else:
await conn.execute(
"UPDATE prefix SET asn_path = $1, next_hops = $2, neighbor = $3, ixp = $4, last_seen = $5, "
"age = $6, updated_at = $7 WHERE prefix = $8 AND asn_id = $9;",
p_dic.get('asn_path', []), p_dic.get('next_hops', []),
p_dic.get('neighbor'), p_dic.get('ixp'), datetime.utcnow(),
age, datetime.utcnow(), p_dic['prefix'], asn_id)
# for k, v in p_dic.items():
# setattr(pfx, k, v)
for c in communities: # type: LargeCommunity
if c not in self._cache['community_in_db']:
try:
await conn.execute(
"insert into community (id, created_at, updated_at) values ($1, $2, $2);",
c, datetime.utcnow())
except asyncpg.UniqueViolationError:
pass
try:
await conn.execute(
"insert into prefix_communities (prefix_id, community_id) values ($1, $2);",
pfx['id'], c)
except asyncpg.UniqueViolationError:
pass
# pfx.communities.append(comm)
return pfx
def time_behind(self) -> Optional[timedelta]:
if empty(self.block_time): return None
dt = convert_datetime(self.block_time).replace(tzinfo=pytz.UTC)
now = datetime.utcnow().replace(tzinfo=pytz.UTC)
return now - dt