def request_url_post(self, cmd, json_params, timeout=None): verbose_logger.debug( "--> Verbose : Command is |{}|, Params are |{}|".format( cmd, json_params)) url = self.get_node_url() + cmd headers = { 'content-type': "application/json", 'cache-control': "no-cache" } try: response = self._do_request(method="POST", url=url, json_params=json_params, headers=headers, timeout=timeout) except Exception: return -1, "TimeOut" if response.status_code != HTTPStatus.OK: return response.status_code, "Code" + str(response.status_code) output = response.json() verbose_logger.debug("<-- Verbose : Answer is |{}|".format(output)) return response.status_code, output
def execute(self, cmd, verbose_override=None, timeout=None): if verbose_override is None: verbose_override = True if verbose_override: verbose_logger.debug("--> Verbose : Command is |{}|".format(cmd)) try: os.environ["TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER"] = "Y" output = check_output(cmd, shell=True, stderr=STDOUT, timeout=timeout, encoding='utf8') except TimeoutExpired as e: logger.info("Command timed out") raise e except CalledProcessError as e: logger.info("Command failed, error is |{}|".format(e.output)) return False, e.output # output = output.decode('utf-8') output = clear_terminal_chars(output) output = output.strip() verbose_logger.debug("<-- Verbose : Answer is |{}|".format(output)) return True, output
def get_liquidity_providers_list(self, big_map_id, snapshot_block): offset = 0 listLPs = {} resp = ' ' while resp != []: uri = self.api + balance_LP_call.format(big_map_id, offset, snapshot_block) offset += 100 verbose_logger.debug("Requesting LP balances, {}".format(uri)) resp = requests.get(uri, timeout=5) verbose_logger.debug("Response from tzstats is {}".format( resp.content.decode("utf8"))) if resp.status_code != HTTPStatus.OK: # This means something went wrong. raise ApiProviderException('GET {} {}'.format( uri, resp.status_code)) resp = resp.json() for item in resp: listLPs[item['key']] = int(item['value']['balance']) return listLPs
def __fetch_current_balance(self, address_list): param_txt = '' for address in address_list: param_txt += address + ',' param_txt = param_txt[:-1] uri = self.api + single_current_balance_call.format(param_txt) sleep(0.5) # be nice to tzstats verbose_logger.debug("Requesting current balance of delegator, phase 2, {}".format(uri)) resp = requests.get(uri, timeout=5) verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) resp = resp.json() ret_list = {} for item in resp: ret_list[item[idx_cb_delegator_address]] = int(1e6 * float(item[idx_cb_current_balance])) return ret_list
def do_rpc_request(self, request, time_out=120): verbose_logger.debug("[do_rpc_request] Requesting URL {:s}".format(request)) sleep(0.1) # be nice to public node service try: resp = requests.get(request, timeout=time_out) except requests.exceptions.Timeout: # Catches both ConnectTimeout and ReadTimeout message = ( "[do_rpc_request] Requesting URL '{:s}' timed out after {:d}s".format( request, time_out ) ) logger.error(message) raise ApiProviderException(message) except requests.exceptions.RequestException as e: # Catches all other requests exceptions message = ( "[do_rpc_request] Requesting URL '{:s}' Generic Error: {:s}".format( request, str(e) ) ) logger.error(message) raise ApiProviderException(message) # URL not found if resp.status_code == HTTPStatus.NOT_FOUND: raise ApiProviderException( "RPC URL '{}' not found. Is this node in archive mode?".format(request) ) # URL returned something broken from the client side 4xx # server side errors 5xx can pass for a retry if ( HTTPStatus.BAD_REQUEST <= resp.status_code < HTTPStatus.INTERNAL_SERVER_ERROR ): message = "[do_rpc_request] Requesting URL '{:s}' failed ({:d})".format( request, resp.status_code ) if "CF-RAY" in resp.headers: message += ", unique request_id: {:s}".format(resp.headers["CF-RAY"]) raise ApiProviderException(message) # URL fetch succeeded; parse to JSON object response = resp.json() verbose_logger.debug("[do_rpc_request] Response {:s}".format(str(response))) return response
def parse_list_known_contracts_response(self, response): dict = {} for line in response.splitlines(): line = line.strip() if ":" in line and not_indicator_line(line): alias, pkh = line.split(":", maxsplit=1) dict[alias.strip()] = pkh.strip() verbose_logger.debug("known contracts: {}".format(dict)) return dict
def parse_get_manager_for_contract_response(self, response): manager = None for line in response.splitlines(): line = line.strip() if line.startswith("tz"): line = line.replace(" (", ':') manager, alias_plus = line.split(":", maxsplit=1) break verbose_logger.debug("Manager address is : {}".format(manager)) return manager
def postMessage(self, chatId, msg, msgType): payload = {"chat_id": chatId, "parse_mode": "html", "text": msg} resp = requests.post(self.api_url, params=payload) respJ = resp.json() logger.info( "[TelegramPlugin] Sent {:s} notification to chatId: {:}".format( msgType, chatId)) verbose_logger.debug("[TelegramPlugin] {:s} raw response: {:}".format( msgType, respJ)) if not respJ["ok"]: logger.error( "[TelegramPlugin] Error sending {:s} message: {:s}".format( msgType, respJ.description))
def get_current_level(self): uri = self.head_api + '/explorer/tip' verbose_logger.debug("Requesting {}".format(uri)) resp = requests.get(uri, timeout=5) root = resp.json() verbose_logger.debug("Response from tzstats is: {}".format(root)) current_level = int(root["status"]["blocks"]) return current_level
def get_delegatable(self, pkh): try: uri = self.head_api + "/explorer/account/{}".format(pkh) verbose_logger.debug("Requesting {}".format(uri)) response = requests.get(uri) account = response.json() return bool(account["is_delegate"]) and bool(account["is_active_delegate"]) except requests.exceptions.RequestException as e: message = "[{}] - Unable to fetch delegate: {:s}".format( __class__.__name__, str(e) ) logger.error(message) raise ApiProviderException(message)
def get_current_cycle_and_level(self): uri = self.head_api + "/explorer/tip" verbose_logger.debug("Requesting {}".format(uri)) resp = requests.get(uri, timeout=5) root = resp.json() verbose_logger.debug("Response from tzstats is: {}".format(root)) current_cycle = int(root["cycle"]) current_level = int(root["height"]) return (current_cycle, current_level)
def request_url(self, cmd, timeout=None): verbose_logger.debug("--> Verbose : Command is |{}|".format(cmd)) url = self.get_node_url() + cmd response = self._do_request(method="GET", url=url, timeout=timeout) if response is None: return -1, "TimeOut" if response.status_code != HTTPStatus.OK: return response.status_code, "Code" + str(response.status_code) output = response.json() verbose_logger.debug("<-- Verbose : Answer is |{}|".format(output)) return response.status_code, output
def _request(self, path, **params): data = { key: value for key, value in params.items() if value is not None } if path.startswith("/"): url = self.base_url + path else: url = self.base_url + "/" + path verbose_logger.debug("Requesting {}".format(url)) try: response = requests.get( url=url, params=data, timeout=self.timeout, headers={"User-Agent": f"trd-{VERSION}"}, ) except requests.Timeout: raise TzKTApiError("Request timeout") except requests.ConnectionError: raise TzKTApiError("DNS lookup failed") except requests.HTTPError as e: raise TzKTApiError("HTTP Error occurred: {}".format(e)) except requests.RequestException as e: raise TzKTApiError(e) # Raise exception for client side errors (4xx) if (HTTPStatus.BAD_REQUEST <= response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR): raise TzKTApiError( f"TzKT returned {response.status_code} error:\n{response.text}" ) # Return None if empty content # or we get a server side error (5xx) if (response.status_code == HTTPStatus.NO_CONTENT) or ( response.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR): return None try: res = response.json() except JSONDecodeError: raise TzKTApiError(f"Failed to decode JSON:\n{response.text}") verbose_logger.debug(f"Response from TzKT is:\n{pformat(res)}") return res
def get_big_map_id(self, contract_id): uri = self.api + contract_storage.format(contract_id) verbose_logger.debug("Requesting contract storage, {}".format(uri)) resp = requests.get(uri, timeout=5) verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) resp = resp.json() return resp['value']['accounts']
def send_notification(self, title, message, attachments=None, reward_data=None): # Add sparkles emoji to message message = "✨ <b>{:s}</b>\n{:s}".format(title, message) for c in self.chat_ids: payload = {"chat_id": c, "parse_mode": "html", "text": message} resp = requests.post(self.api_url, params=payload) verbose_logger.debug("[TelegramPlugin] Response: {:}".format( resp.json())) logger.info("[TelegramPlugin] Notification '{:s}' sent".format(title))
def send_admin_notification(self, subject, message, attachments=None, reward_data=None): admin_text = "<b>{:s}</b>\n{:s}".format(subject, message) for c in self.admin_chat_ids: payload = {"chat_id": c, "parse_mode": "html", "text": admin_text} resp = requests.post(self.api_url, params=payload) verbose_logger.debug("[TelegramPlugin] Admin Response: {:}".format( resp.json())) logger.info( "[TelegramPlugin] Admin Notification '{:s}' sent".format(subject))
def send_payout_notification(self, cycle, payout_amount, nb_delegators): # Add sparkles emoji to message message = self.telegram_text \ .replace("%CYCLE%", str(cycle)) \ .replace("%TREWARDS%", str(round(payout_amount / MUTEZ, 2))) \ .replace("%NDELEGATORS%", str(nb_delegators)) for c in self.payouts_chat_ids: payload = {"chat_id": c, "parse_mode": "html", "text": message} resp = requests.post(self.api_url, params=payload) verbose_logger.debug("[TelegramPlugin] Public Response: {:}".format( resp.json())) logger.info( "[TelegramPlugin] Public Notification '{:s}...' sent".format( message[:20]))
def extract_json_part(input): verbose_logger.debug( "->will parse json response_str is '{}'".format(input)) # because of disclaimer header; find beginning of response idx = input.find("{") if idx < 0: idx = input.find("[") if idx < 0: idx = input.find("\"") if idx < 0: return None extracted_json_part = input[idx:].strip() verbose_logger.debug( "<-parsed json response_str is '{}'".format(extracted_json_part)) return extracted_json_part
def parse_list_known_addresses_response(self, response): dict = {} for line in response.splitlines(): line = line.strip() if ":" in line and not_indicator_line(line): alias, pkh_plus_braces = line.split(":", maxsplit=1) pkh_plus_braces = pkh_plus_braces.replace(' (', ':') if ':' in pkh_plus_braces: pkh, sk_section = pkh_plus_braces.split(":", maxsplit=1) else: pkh = pkh_plus_braces.strip() sk_section = "" sk_known = "sk known" in sk_section pkh = pkh.strip() alias = alias.strip() dict[pkh] = {"alias": alias, "sk": sk_known} verbose_logger.debug("known addresses: {}".format(dict)) return dict
def do_rpc_request(self, request, time_out=120): verbose_logger.debug("[do_rpc_request] Requesting URL {:s}".format(request)) sleep(0.1) # be nice to public node service try: resp = requests.get(request, timeout=time_out) except requests.exceptions.Timeout: # Catches both ConnectTimeout and ReadTimeout message = "[do_rpc_request] Requesting URL '{:s}' timed out after {:d}s".format(request, time_out) logger.error(message) raise ApiProviderException(message) except requests.exceptions.RequestException as e: # Catches all other requests exceptions message = "[do_rpc_request] Requesting URL '{:s}' Generic Error: {:s}".format(request, str(e)) logger.error(message) raise ApiProviderException(message) # URL not found if resp.status_code == 404: raise ApiProviderException("RPC URL '{}' not found. Is this node in archive mode?".format(request)) # URL returned something broken if resp.status_code != HTTPStatus.OK: message = "[do_rpc_request] Requesting URL '{:s}' failed ({:d})".format(request, resp.status_code) if "CF-RAY" in resp.headers: message += ", unique request_id: {:s}".format(resp.headers['CF-RAY']) raise ApiProviderException(message) # URL fetch succeeded; parse to JSON object response = resp.json() verbose_logger.debug("[do_rpc_request] Response {:s}".format(str(response))) return response
def _request(self, path, **params): data = { key: value for key, value in params.items() if value is not None } url = join(self.base_url, path) verbose_logger.debug("Requesting {}".format(url)) try: response = requests.get(url=url, params=data, timeout=self.timeout, headers={'User-Agent': f'trd-{VERSION}'}) except requests.Timeout: raise TzKTApiError('Request timeout') except requests.ConnectionError: raise TzKTApiError('DNS lookup failed') except requests.HTTPError as e: raise TzKTApiError('HTTP Error occurred: {}'.format(e)) except requests.RequestException as e: raise TzKTApiError(e) if response.status_code not in [200, 204]: raise TzKTApiError( f'TzKT returned {response.status_code} error:\n{response.text}' ) try: res = response.json() except JSONDecodeError: raise TzKTApiError(f'Failed to decode JSON:\n{response.text}') verbose_logger.debug(f'Response from TzKT is:\n{pformat(res)}') return res
def get_rewards_for_cycle(self, cycle, expected_rewards=False): root = {"delegate_staking_balance": 0, "total_reward_amount": 0, "delegators_balances": {}} # # Get rewards breakdown for cycle # uri = self.api + rewards_split_call.format(self.baking_address, cycle) sleep(0.5) # be nice to tzstats verbose_logger.debug("Requesting rewards breakdown, {}".format(uri)) resp = requests.get(uri, timeout=5) verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) resp = resp.json()[0] if expected_rewards: root["total_reward_amount"] = int(1e6 * float(resp[idx_income_expected_income])) else: root["total_reward_amount"] = int(1e6 * (float(resp[idx_income_baking_income]) + float(resp[idx_income_endorsing_income]) + float(resp[idx_income_seed_income]) + float(resp[idx_income_fees_income]) - float(resp[idx_income_lost_accusation_fees]) - float(resp[idx_income_lost_accusation_rewards]) - float(resp[idx_income_lost_revelation_fees]) - float(resp[idx_income_lost_revelation_rewards]))) # # Get staking balances of delegators at snapshot block # uri = self.api + delegators_call.format(cycle - self.preserved_cycles - 2, self.baking_address) sleep(0.5) # be nice to tzstats verbose_logger.debug("Requesting staking balances of delegators, {}".format(uri)) resp = requests.get(uri, timeout=5) verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) resp = resp.json() for delegator in resp: if delegator[idx_delegator_address] == self.baking_address: root["delegate_staking_balance"] = int( 1e6 * (float(delegator[idx_balance]) + float(delegator[idx_baker_delegated]))) else: delegator_info = {"staking_balance": 0, "current_balance": 0} delegator_info["staking_balance"] = int(1e6 * float(delegator[idx_balance])) root["delegators_balances"][delegator[idx_delegator_address]] = delegator_info # # Get current balance of delegates # # This is done in 2 phases. 1) make a single API call to tzstats, retrieving an array # of arrays with current balance of each delegator who "currently" delegates to delegate. There may # be a case where the delegator has changed delegations and would therefor not be in this array. # Thus, 2) determines which delegators are not in the first result, and makes individual # calls to get their balance. This approach should reduce the overall number of API calls made to tzstats. # # Phase 1 # uri = self.api + batch_current_balance_call.format(self.baking_address) sleep(0.5) # be nice to tzstats verbose_logger.debug("Requesting current balance of delegators, phase 1, {}".format(uri)) resp = requests.get(uri, timeout=5) verbose_logger.debug("Response from tzstats is {}".format(resp.content.decode("utf8"))) if resp.status_code != 200: # This means something went wrong. raise ApiProviderException('GET {} {}'.format(uri, resp.status_code)) resp = resp.json() # Will use these two lists to determine who has/has not been fetched staked_bal_delegators = root["delegators_balances"].keys() curr_bal_delegators = [] for delegator in resp: delegator_addr = delegator[idx_cb_delegator_address] # If delegator is in this batch, but has no staking balance for this reward cycle, # then they must be a new delegator and are not receiving rewards at this time. # We can ignore them. if delegator_addr not in staked_bal_delegators: continue root["delegators_balances"][delegator_addr]["current_balance"] = int( 1e6 * float(delegator[idx_cb_current_balance])) curr_bal_delegators.append(delegator_addr) # Phase 2 # # Who was not in this result? need_curr_balance_fetch = list(set(staked_bal_delegators) - set(curr_bal_delegators)) # Fetch individual not in original batch if len(need_curr_balance_fetch) > 0: split_addresses = split(need_curr_balance_fetch, 50) for list_address in split_addresses: list_curr_balances = self.__fetch_current_balance(list_address) for d in list_address: root["delegators_balances"][d]["current_balance"] = list_curr_balances[d] curr_bal_delegators.append(d) # All done fetching balances. # Sanity check. n_curr_balance = len(curr_bal_delegators) n_stake_balance = len(staked_bal_delegators) if n_curr_balance != n_stake_balance: raise ApiProviderException('Did not fetch all balances {}/{}'.format(n_curr_balance, n_stake_balance)) return root