def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: options: Dict[str, Union[str, int]] = { 'pageSize': 200, # max page size according to their docs 'startDate': timestamp_to_iso8601(start_ts, utc_as_z=True), 'endDate': timestamp_to_iso8601(end_ts, utc_as_z=True), } raw_data = self._paginated_api_query(endpoint='deposits/closed', options=options.copy()) raw_data.extend( self._paginated_api_query(endpoint='withdrawals/closed', options=options.copy()), ) log.debug('bittrex deposit/withdrawal history result', results_num=len(raw_data)) movements = [] for raw_movement in raw_data: movement = self._deserialize_asset_movement(raw_movement) if movement and movement.timestamp >= start_ts and movement.timestamp <= end_ts: movements.append(movement) return movements
def query_online_deposits_withdrawals( self, # pylint: disable=no-self-use start_ts: Timestamp, # pylint: disable=unused-argument end_ts: Timestamp, # pylint: disable=unused-argument ) -> List[AssetMovement]: if self.account_guids is None: self.query_balances() # do a balance query to populate the account guids movements = [] for guid in self.account_guids: # type: ignore # we know its not None try: resp = self._gather_paginated_data( path='GetTransactions', extra_options={ 'accountGuid': guid, 'fromTimestampUtc': timestamp_to_iso8601(start_ts, utc_as_z=True), 'toTimestampUtc': timestamp_to_iso8601(end_ts, utc_as_z=True), # if we filter by tx type in my tests I started getting # {"Message":"A server error occurred. Please wait a few minutes and try again."} # noqa: E501 # 'txTypes': 'Deposit,Withdrawal', # there is also DepositFee }, ) except KeyError as e: self.msg_aggregator.add_error( f'Error processing IndependentReserve transactions response. ' f'Missing key: {str(e)}.', ) return [] for entry in resp: entry_type = entry.get('Type') if entry_type is None or entry_type not in ('Deposit', 'Withdrawal'): continue try: movement = _asset_movement_from_independentreserve(entry) if movement: movements.append(movement) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown IndependentReserve asset {e.asset_name}. ' f'Ignoring the deposit/withdrawal containing it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize an IndependentReserve deposit/withdrawal. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing an IndependentReserve deposit/withdrawal.', raw_asset_movement=entry, error=msg, ) continue return movements
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, market: Optional[TradePair] = None, ) -> List[Trade]: options: Dict[str, Union[str, int]] = { 'pageSize': 200, # max page size according to their docs 'startDate': timestamp_to_iso8601(start_ts, utc_as_z=True), 'endDate': timestamp_to_iso8601(end_ts, utc_as_z=True), } if market is not None: options['marketSymbol'] = world_pair_to_bittrex(market) raw_data = self._paginated_api_query(endpoint='orders/closed', options=options) log.debug('bittrex order history result', results_num=len(raw_data)) trades = [] for raw_trade in raw_data: try: trade = trade_from_bittrex(raw_trade) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found bittrex trade with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found bittrex trade with unsupported asset ' f'{e.asset_name}. Ignoring it.', ) continue except UnprocessableTradePair as e: self.msg_aggregator.add_error( f'Found bittrex trade with unprocessable pair ' f'{e.pair}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Error processing a bittrex trade. Check logs ' 'for details. Ignoring it.', ) log.error( 'Error processing a bittrex trade', trade=raw_trade, error=msg, ) continue trades.append(trade) return trades
def _generate_reports( self, start_ts: Timestamp, end_ts: Timestamp, report_type: Literal['fills', 'account'], tempdir: str, ) -> List[str]: """ Generates all the reports to get historical data from coinbase. https://docs.pro.coinbase.com/#reports There are 2 type of reports. 1. Fill reports which are per product id (market) 2. Account reports which are per account id The fill reports have the following data format: portfolio,trade id,product,side,created at,size,size unit,price,fee, total,price/fee/total unit The account reports have the following data format: portfolio,type,time,amount,balance,amount/balance unit,transfer id,trade id,order id Returns a list of filepaths where the reports were written. - Raises the same exceptions as _api_query() - Can raise KeyError if the API does not return the expected response format. """ start_date = timestamp_to_iso8601(start_ts) end_date = timestamp_to_iso8601(end_ts) if report_type == 'fills': account_or_product_ids = self._get_products_ids() identifier_key = 'product_id' else: account_or_product_ids = self._get_account_ids() identifier_key = 'account_id' report_ids = [] options = { 'type': report_type, 'start_date': start_date, 'end_date': end_date, 'format': 'csv', # The only way to disable emailing the report link is to give an invalid link 'email': '*****@*****.**', } for identifier in account_or_product_ids: options[identifier_key] = identifier post_result = self._api_query('reports', request_method='POST', options=options) report_ids.append(post_result['id']) # At this point all reports must have been queued for creation at the server # Now wait until they are ready and pull them one by one report_paths = [] last_change_ts = ts_now() while True: finished_ids_indices = [] for idx, report_id in enumerate(report_ids): get_result = self._api_query(f'reports/{report_id}', request_method='GET') # Have to add assert here for mypy since the endpoint string is # a variable string and can't be overloaded and type checked assert isinstance(get_result, dict) if get_result['status'] != 'ready': continue # a report is ready here so let's reset the timer last_change_ts = ts_now() file_url = get_result['file_url'] response = requests.get(file_url) length = len(response.content) # empty fill reports have length of 95, empty account reports 85 # So we assume a report of more than 100 chars has data. if length > 100: log.debug( f'Got a populated report for id: {report_id}. Writing it to disk' ) filepath = os.path.join(tempdir, f'report_{report_id}.csv') with open(filepath, 'wb') as f: f.write(response.content) report_paths.append(filepath) else: log.debug( f'Got report for id: {report_id} with length {length}. Skipping it' ) finished_ids_indices.append(idx) if ts_now() - last_change_ts > SECS_TO_WAIT_FOR_REPORT: raise RemoteError( f'There has been no response from CoinbasePro reports for over ' f' {MINS_TO_WAIT_FOR_REPORT} minutes. Bailing out.', ) # Delete the report ids that have been downloaded. Note: reverse order # so that we don't mess up the indices for idx in reversed(finished_ids_indices): del report_ids[idx] # When there is no more ids to query break out of the loop if len(report_ids) == 0: break return report_paths