def request_url_post(self, cmd, json_params, timeout=None):
verbose_logger.debug(
"--> Verbose : Command is |{}|, Params are |{}|".format(
cmd, json_params))
url = self.get_node_url() + cmd
headers = {
'content-type': "application/json",
'cache-control': "no-cache"
}
try:
response = self._do_request(method="POST",
url=url,
json_params=json_params,
headers=headers,
timeout=timeout)
except Exception:
return -1, "TimeOut"
if response.status_code != HTTPStatus.OK:
return response.status_code, "Code" + str(response.status_code)
output = response.json()
verbose_logger.debug("<-- Verbose : Answer is |{}|".format(output))
return response.status_code, output
def execute(self, cmd, verbose_override=None, timeout=None):
if verbose_override is None:
verbose_override = True
if verbose_override:
verbose_logger.debug("--> Verbose : Command is |{}|".format(cmd))
try:
os.environ["TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER"] = "Y"
output = check_output(cmd,
shell=True,
stderr=STDOUT,
timeout=timeout,
encoding='utf8')
except TimeoutExpired as e:
logger.info("Command timed out")
raise e
except CalledProcessError as e:
logger.info("Command failed, error is |{}|".format(e.output))
return False, e.output
# output = output.decode('utf-8')
output = clear_terminal_chars(output)
output = output.strip()
verbose_logger.debug("<-- Verbose : Answer is |{}|".format(output))
return True, output
def get_liquidity_providers_list(self, big_map_id, snapshot_block):
offset = 0
listLPs = {}
resp = ' '
while resp != []:
uri = self.api + balance_LP_call.format(big_map_id, offset,
snapshot_block)
offset += 100
verbose_logger.debug("Requesting LP balances, {}".format(uri))
resp = requests.get(uri, timeout=5)
verbose_logger.debug("Response from tzstats is {}".format(
resp.content.decode("utf8")))
if resp.status_code != HTTPStatus.OK:
# This means something went wrong.
raise ApiProviderException('GET {} {}'.format(
uri, resp.status_code))
resp = resp.json()
for item in resp:
listLPs[item['key']] = int(item['value']['balance'])
return listLPs
def __fetch_current_balance(self, address_list):
param_txt = ''
for address in address_list:
param_txt += address + ','
param_txt = param_txt[:-1]
uri = self.api + single_current_balance_call.format(param_txt)
sleep(0.5) # be nice to tzstats
verbose_logger.debug("Requesting current balance of delegator, phase 2, {}".format(uri))
resp = requests.get(uri, timeout=5)
verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8")))
if resp.status_code != 200:
# This means something went wrong.
raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
resp = resp.json()
ret_list = {}
for item in resp:
ret_list[item[idx_cb_delegator_address]] = int(1e6 * float(item[idx_cb_current_balance]))
return ret_list
def do_rpc_request(self, request, time_out=120):
verbose_logger.debug("[do_rpc_request] Requesting URL {:s}".format(request))
sleep(0.1) # be nice to public node service
try:
resp = requests.get(request, timeout=time_out)
except requests.exceptions.Timeout:
# Catches both ConnectTimeout and ReadTimeout
message = (
"[do_rpc_request] Requesting URL '{:s}' timed out after {:d}s".format(
request, time_out
)
)
logger.error(message)
raise ApiProviderException(message)
except requests.exceptions.RequestException as e:
# Catches all other requests exceptions
message = (
"[do_rpc_request] Requesting URL '{:s}' Generic Error: {:s}".format(
request, str(e)
)
)
logger.error(message)
raise ApiProviderException(message)
# URL not found
if resp.status_code == HTTPStatus.NOT_FOUND:
raise ApiProviderException(
"RPC URL '{}' not found. Is this node in archive mode?".format(request)
)
# URL returned something broken from the client side 4xx
# server side errors 5xx can pass for a retry
if (
HTTPStatus.BAD_REQUEST
<= resp.status_code
< HTTPStatus.INTERNAL_SERVER_ERROR
):
message = "[do_rpc_request] Requesting URL '{:s}' failed ({:d})".format(
request, resp.status_code
)
if "CF-RAY" in resp.headers:
message += ", unique request_id: {:s}".format(resp.headers["CF-RAY"])
raise ApiProviderException(message)
# URL fetch succeeded; parse to JSON object
response = resp.json()
verbose_logger.debug("[do_rpc_request] Response {:s}".format(str(response)))
return response
def parse_list_known_contracts_response(self, response):
dict = {}
for line in response.splitlines():
line = line.strip()
if ":" in line and not_indicator_line(line):
alias, pkh = line.split(":", maxsplit=1)
dict[alias.strip()] = pkh.strip()
verbose_logger.debug("known contracts: {}".format(dict))
return dict
def parse_get_manager_for_contract_response(self, response):
manager = None
for line in response.splitlines():
line = line.strip()
if line.startswith("tz"):
line = line.replace(" (", ':')
manager, alias_plus = line.split(":", maxsplit=1)
break
verbose_logger.debug("Manager address is : {}".format(manager))
return manager
def postMessage(self, chatId, msg, msgType):
payload = {"chat_id": chatId, "parse_mode": "html", "text": msg}
resp = requests.post(self.api_url, params=payload)
respJ = resp.json()
logger.info(
"[TelegramPlugin] Sent {:s} notification to chatId: {:}".format(
msgType, chatId))
verbose_logger.debug("[TelegramPlugin] {:s} raw response: {:}".format(
msgType, respJ))
if not respJ["ok"]:
logger.error(
"[TelegramPlugin] Error sending {:s} message: {:s}".format(
msgType, respJ.description))
def get_current_level(self):
uri = self.head_api + '/explorer/tip'
verbose_logger.debug("Requesting {}".format(uri))
resp = requests.get(uri, timeout=5)
root = resp.json()
verbose_logger.debug("Response from tzstats is: {}".format(root))
current_level = int(root["status"]["blocks"])
return current_level
def get_delegatable(self, pkh):
try:
uri = self.head_api + "/explorer/account/{}".format(pkh)
verbose_logger.debug("Requesting {}".format(uri))
response = requests.get(uri)
account = response.json()
return bool(account["is_delegate"]) and bool(account["is_active_delegate"])
except requests.exceptions.RequestException as e:
message = "[{}] - Unable to fetch delegate: {:s}".format(
__class__.__name__, str(e)
)
logger.error(message)
raise ApiProviderException(message)
def get_current_cycle_and_level(self):
uri = self.head_api + "/explorer/tip"
verbose_logger.debug("Requesting {}".format(uri))
resp = requests.get(uri, timeout=5)
root = resp.json()
verbose_logger.debug("Response from tzstats is: {}".format(root))
current_cycle = int(root["cycle"])
current_level = int(root["height"])
return (current_cycle, current_level)
def request_url(self, cmd, timeout=None):
verbose_logger.debug("--> Verbose : Command is |{}|".format(cmd))
url = self.get_node_url() + cmd
response = self._do_request(method="GET", url=url, timeout=timeout)
if response is None:
return -1, "TimeOut"
if response.status_code != HTTPStatus.OK:
return response.status_code, "Code" + str(response.status_code)
output = response.json()
verbose_logger.debug("<-- Verbose : Answer is |{}|".format(output))
return response.status_code, output
def _request(self, path, **params):
data = {
key: value
for key, value in params.items() if value is not None
}
if path.startswith("/"):
url = self.base_url + path
else:
url = self.base_url + "/" + path
verbose_logger.debug("Requesting {}".format(url))
try:
response = requests.get(
url=url,
params=data,
timeout=self.timeout,
headers={"User-Agent": f"trd-{VERSION}"},
)
except requests.Timeout:
raise TzKTApiError("Request timeout")
except requests.ConnectionError:
raise TzKTApiError("DNS lookup failed")
except requests.HTTPError as e:
raise TzKTApiError("HTTP Error occurred: {}".format(e))
except requests.RequestException as e:
raise TzKTApiError(e)
# Raise exception for client side errors (4xx)
if (HTTPStatus.BAD_REQUEST <= response.status_code <
HTTPStatus.INTERNAL_SERVER_ERROR):
raise TzKTApiError(
f"TzKT returned {response.status_code} error:\n{response.text}"
)
# Return None if empty content
# or we get a server side error (5xx)
if (response.status_code == HTTPStatus.NO_CONTENT) or (
response.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR):
return None
try:
res = response.json()
except JSONDecodeError:
raise TzKTApiError(f"Failed to decode JSON:\n{response.text}")
verbose_logger.debug(f"Response from TzKT is:\n{pformat(res)}")
return res
def get_big_map_id(self, contract_id):
uri = self.api + contract_storage.format(contract_id)
verbose_logger.debug("Requesting contract storage, {}".format(uri))
resp = requests.get(uri, timeout=5)
verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8")))
if resp.status_code != 200:
# This means something went wrong.
raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
resp = resp.json()
return resp['value']['accounts']
def send_notification(self,
title,
message,
attachments=None,
reward_data=None):
# Add sparkles emoji to message
message = "✨ <b>{:s}</b>\n{:s}".format(title, message)
for c in self.chat_ids:
payload = {"chat_id": c, "parse_mode": "html", "text": message}
resp = requests.post(self.api_url, params=payload)
verbose_logger.debug("[TelegramPlugin] Response: {:}".format(
resp.json()))
logger.info("[TelegramPlugin] Notification '{:s}' sent".format(title))
def send_admin_notification(self,
subject,
message,
attachments=None,
reward_data=None):
admin_text = "<b>{:s}</b>\n{:s}".format(subject, message)
for c in self.admin_chat_ids:
payload = {"chat_id": c, "parse_mode": "html", "text": admin_text}
resp = requests.post(self.api_url, params=payload)
verbose_logger.debug("[TelegramPlugin] Admin Response: {:}".format(
resp.json()))
logger.info(
"[TelegramPlugin] Admin Notification '{:s}' sent".format(subject))
def send_payout_notification(self, cycle, payout_amount, nb_delegators):
# Add sparkles emoji to message
message = self.telegram_text \
.replace("%CYCLE%", str(cycle)) \
.replace("%TREWARDS%", str(round(payout_amount / MUTEZ, 2))) \
.replace("%NDELEGATORS%", str(nb_delegators))
for c in self.payouts_chat_ids:
payload = {"chat_id": c, "parse_mode": "html", "text": message}
resp = requests.post(self.api_url, params=payload)
verbose_logger.debug("[TelegramPlugin] Public Response: {:}".format(
resp.json()))
logger.info(
"[TelegramPlugin] Public Notification '{:s}...' sent".format(
message[:20]))
def extract_json_part(input):
verbose_logger.debug(
"->will parse json response_str is '{}'".format(input))
# because of disclaimer header; find beginning of response
idx = input.find("{")
if idx < 0:
idx = input.find("[")
if idx < 0:
idx = input.find("\"")
if idx < 0:
return None
extracted_json_part = input[idx:].strip()
verbose_logger.debug(
"<-parsed json response_str is '{}'".format(extracted_json_part))
return extracted_json_part
def parse_list_known_addresses_response(self, response):
dict = {}
for line in response.splitlines():
line = line.strip()
if ":" in line and not_indicator_line(line):
alias, pkh_plus_braces = line.split(":", maxsplit=1)
pkh_plus_braces = pkh_plus_braces.replace(' (', ':')
if ':' in pkh_plus_braces:
pkh, sk_section = pkh_plus_braces.split(":", maxsplit=1)
else:
pkh = pkh_plus_braces.strip()
sk_section = ""
sk_known = "sk known" in sk_section
pkh = pkh.strip()
alias = alias.strip()
dict[pkh] = {"alias": alias, "sk": sk_known}
verbose_logger.debug("known addresses: {}".format(dict))
return dict
def do_rpc_request(self, request, time_out=120):
verbose_logger.debug("[do_rpc_request] Requesting URL {:s}".format(request))
sleep(0.1) # be nice to public node service
try:
resp = requests.get(request, timeout=time_out)
except requests.exceptions.Timeout:
# Catches both ConnectTimeout and ReadTimeout
message = "[do_rpc_request] Requesting URL '{:s}' timed out after {:d}s".format(request, time_out)
logger.error(message)
raise ApiProviderException(message)
except requests.exceptions.RequestException as e:
# Catches all other requests exceptions
message = "[do_rpc_request] Requesting URL '{:s}' Generic Error: {:s}".format(request, str(e))
logger.error(message)
raise ApiProviderException(message)
# URL not found
if resp.status_code == 404:
raise ApiProviderException("RPC URL '{}' not found. Is this node in archive mode?".format(request))
# URL returned something broken
if resp.status_code != HTTPStatus.OK:
message = "[do_rpc_request] Requesting URL '{:s}' failed ({:d})".format(request, resp.status_code)
if "CF-RAY" in resp.headers:
message += ", unique request_id: {:s}".format(resp.headers['CF-RAY'])
raise ApiProviderException(message)
# URL fetch succeeded; parse to JSON object
response = resp.json()
verbose_logger.debug("[do_rpc_request] Response {:s}".format(str(response)))
return response
def _request(self, path, **params):
data = {
key: value
for key, value in params.items() if value is not None
}
url = join(self.base_url, path)
verbose_logger.debug("Requesting {}".format(url))
try:
response = requests.get(url=url,
params=data,
timeout=self.timeout,
headers={'User-Agent': f'trd-{VERSION}'})
except requests.Timeout:
raise TzKTApiError('Request timeout')
except requests.ConnectionError:
raise TzKTApiError('DNS lookup failed')
except requests.HTTPError as e:
raise TzKTApiError('HTTP Error occurred: {}'.format(e))
except requests.RequestException as e:
raise TzKTApiError(e)
if response.status_code not in [200, 204]:
raise TzKTApiError(
f'TzKT returned {response.status_code} error:\n{response.text}'
)
try:
res = response.json()
except JSONDecodeError:
raise TzKTApiError(f'Failed to decode JSON:\n{response.text}')
verbose_logger.debug(f'Response from TzKT is:\n{pformat(res)}')
return res
def get_rewards_for_cycle(self, cycle, expected_rewards=False):
root = {"delegate_staking_balance": 0, "total_reward_amount": 0, "delegators_balances": {}}
#
# Get rewards breakdown for cycle
#
uri = self.api + rewards_split_call.format(self.baking_address, cycle)
sleep(0.5) # be nice to tzstats
verbose_logger.debug("Requesting rewards breakdown, {}".format(uri))
resp = requests.get(uri, timeout=5)
verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8")))
if resp.status_code != 200:
# This means something went wrong.
raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
resp = resp.json()[0]
if expected_rewards:
root["total_reward_amount"] = int(1e6 * float(resp[idx_income_expected_income]))
else:
root["total_reward_amount"] = int(1e6 * (float(resp[idx_income_baking_income])
+ float(resp[idx_income_endorsing_income])
+ float(resp[idx_income_seed_income])
+ float(resp[idx_income_fees_income])
- float(resp[idx_income_lost_accusation_fees])
- float(resp[idx_income_lost_accusation_rewards])
- float(resp[idx_income_lost_revelation_fees])
- float(resp[idx_income_lost_revelation_rewards])))
#
# Get staking balances of delegators at snapshot block
#
uri = self.api + delegators_call.format(cycle - self.preserved_cycles - 2, self.baking_address)
sleep(0.5) # be nice to tzstats
verbose_logger.debug("Requesting staking balances of delegators, {}".format(uri))
resp = requests.get(uri, timeout=5)
verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8")))
if resp.status_code != 200:
# This means something went wrong.
raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
resp = resp.json()
for delegator in resp:
if delegator[idx_delegator_address] == self.baking_address:
root["delegate_staking_balance"] = int(
1e6 * (float(delegator[idx_balance]) + float(delegator[idx_baker_delegated])))
else:
delegator_info = {"staking_balance": 0, "current_balance": 0}
delegator_info["staking_balance"] = int(1e6 * float(delegator[idx_balance]))
root["delegators_balances"][delegator[idx_delegator_address]] = delegator_info
#
# Get current balance of delegates
#
# This is done in 2 phases. 1) make a single API call to tzstats, retrieving an array
# of arrays with current balance of each delegator who "currently" delegates to delegate. There may
# be a case where the delegator has changed delegations and would therefor not be in this array.
# Thus, 2) determines which delegators are not in the first result, and makes individual
# calls to get their balance. This approach should reduce the overall number of API calls made to tzstats.
#
# Phase 1
#
uri = self.api + batch_current_balance_call.format(self.baking_address)
sleep(0.5) # be nice to tzstats
verbose_logger.debug("Requesting current balance of delegators, phase 1, {}".format(uri))
resp = requests.get(uri, timeout=5)
verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8")))
if resp.status_code != 200:
# This means something went wrong.
raise ApiProviderException('GET {} {}'.format(uri, resp.status_code))
resp = resp.json()
# Will use these two lists to determine who has/has not been fetched
staked_bal_delegators = root["delegators_balances"].keys()
curr_bal_delegators = []
for delegator in resp:
delegator_addr = delegator[idx_cb_delegator_address]
# If delegator is in this batch, but has no staking balance for this reward cycle,
# then they must be a new delegator and are not receiving rewards at this time.
# We can ignore them.
if delegator_addr not in staked_bal_delegators:
continue
root["delegators_balances"][delegator_addr]["current_balance"] = int(
1e6 * float(delegator[idx_cb_current_balance]))
curr_bal_delegators.append(delegator_addr)
# Phase 2
#
# Who was not in this result?
need_curr_balance_fetch = list(set(staked_bal_delegators) - set(curr_bal_delegators))
# Fetch individual not in original batch
if len(need_curr_balance_fetch) > 0:
split_addresses = split(need_curr_balance_fetch, 50)
for list_address in split_addresses:
list_curr_balances = self.__fetch_current_balance(list_address)
for d in list_address:
root["delegators_balances"][d]["current_balance"] = list_curr_balances[d]
curr_bal_delegators.append(d)
# All done fetching balances.
# Sanity check.
n_curr_balance = len(curr_bal_delegators)
n_stake_balance = len(staked_bal_delegators)
if n_curr_balance != n_stake_balance:
raise ApiProviderException('Did not fetch all balances {}/{}'.format(n_curr_balance, n_stake_balance))
return root
