def process_polo_loans(data, start_ts, end_ts): new_data = list() for loan in reversed(data): close_time = createTimeStamp(loan['close'], formatstr="%Y-%m-%d %H:%M:%S") open_time = createTimeStamp(loan['open'], formatstr="%Y-%m-%d %H:%M:%S") if open_time < start_ts: continue if close_time > end_ts: break loan_data = { 'open_time': open_time, 'close_time': close_time, 'currency': loan['currency'], 'fee': FVal(loan['fee']), 'earned': FVal(loan['earned']), 'amount_lent': FVal(loan['amount']), } log.debug('processing poloniex loan', **make_sensitive(loan_data)) new_data.append(loan_data) new_data.sort(key=lambda loan: loan['open_time']) return new_data
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade: """Turn a poloniex trade returned from poloniex trade history to our common trade history format Throws: - UnsupportedAsset due to asset_from_poloniex() """ trade_type = trade_type_from_string(poloniex_trade['type']) amount = FVal(poloniex_trade['amount']) rate = FVal(poloniex_trade['rate']) perc_fee = FVal(poloniex_trade['fee']) base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first')) quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second')) timestamp = createTimeStamp(poloniex_trade['date'], formatstr="%Y-%m-%d %H:%M:%S") cost = rate * amount if trade_type == TradeType.BUY: fee = amount * perc_fee fee_currency = quote_currency elif trade_type == TradeType.SELL: fee = cost * perc_fee fee_currency = base_currency else: raise ValueError('Got unexpected trade type "{}" for poloniex trade'.format(trade_type)) if poloniex_trade['category'] == 'settlement': if trade_type == TradeType.BUY: trade_type = TradeType.SETTLEMENT_BUY else: trade_type = TradeType.SETTLEMENT_SELL log.debug( 'Processing poloniex Trade', sensitive_log=True, timestamp=timestamp, order_type=trade_type, pair=pair, base_currency=base_currency, quote_currency=quote_currency, amount=amount, fee=fee, rate=rate, ) # Use the converted assets in our pair pair = trade_pair_from_assets(base_currency, quote_currency) # Since in Poloniex the base currency is the cost currency, iow in poloniex # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it # into the Rotkehlchen way which is following the base/quote approach. pair = invert_pair(pair) return Trade( timestamp=timestamp, location='poloniex', pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, )
def query_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, market: Optional[TradePair] = None, count: Optional[int] = None, ) -> List: options: Dict[str, Union[str, int]] = dict() cache = self.check_trades_cache_list(start_ts, end_at_least_ts) if market is not None: options['market'] = world_pair_to_bittrex(market) elif cache is not None: return cache if count is not None: options['count'] = count order_history = self.api_query('getorderhistory', options) log.debug('binance order history result', results_num=len(order_history)) returned_history = list() for order in order_history: order_timestamp = createTimeStamp(order['TimeStamp'], formatstr="%Y-%m-%dT%H:%M:%S.%f") if start_ts is not None and order_timestamp < start_ts: continue if end_ts is not None and order_timestamp > end_ts: break order['TimeStamp'] = order_timestamp returned_history.append(order) self.update_trades_cache(returned_history, start_ts, end_ts) return returned_history
def process_polo_loans( self, data: List[Dict], start_ts: Timestamp, end_ts: Timestamp, ) -> List[Dict]: """Takes in the list of loans from poloniex as returned by the returnLendingHistory api call, processes it and returns it into our loan format """ new_data = list() for loan in reversed(data): close_time = createTimeStamp(loan['close'], formatstr="%Y-%m-%d %H:%M:%S") open_time = createTimeStamp(loan['open'], formatstr="%Y-%m-%d %H:%M:%S") if open_time < start_ts: continue if close_time > end_ts: break try: loan_data = { 'open_time': open_time, 'close_time': close_time, 'currency': asset_from_poloniex(loan['currency']), 'fee': FVal(loan['fee']), 'earned': FVal(loan['earned']), 'amount_lent': FVal(loan['amount']), } except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found poloniex loan with unsupported asset' f' {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found poloniex loan with unknown asset' f' {e.asset_name}. Ignoring it.', ) continue log.debug('processing poloniex loan', **make_sensitive(loan_data)) new_data.append(loan_data) new_data.sort(key=lambda loan: loan['open_time']) return new_data
def _post_process(before: Dict) -> Dict: """Poloniex uses datetimes so turn them into timestamps here""" after = before if ('return' in after): if (isinstance(after['return'], list)): for x in range(0, len(after['return'])): if (isinstance(after['return'][x], dict)): if ('datetime' in after['return'][x] and 'timestamp' not in after['return'][x]): after['return'][x]['timestamp'] = float( createTimeStamp(after['return'][x]['datetime']), ) return after
def from_otc_trade(trade: Dict[str, Any]) -> Dict[str, Any]: ts = createTimeStamp(trade['otc_timestamp'], formatstr='%d/%m/%Y %H:%M') new_trade = { 'timestamp': ts, 'location': 'external', 'pair': trade['otc_pair'], 'type': trade['otc_type'], 'amount': str(trade['otc_amount']), 'rate': str(trade['otc_rate']), 'fee': str(trade['otc_fee']), 'fee_currency': trade['otc_fee_currency'], 'link': trade['otc_link'], 'notes': trade['otc_notes'], } if 'otc_id' in trade: new_trade['id'] = trade['otc_id'] return new_trade
def deserialize_timestamp_from_date(date: str, formatstr: str, location: str) -> Timestamp: """Deserializes a timestamp from a date entry depending on the format str Can throw DeserializationError if the data is not as expected """ if not date: raise DeserializationError( f'Failed to deserialize a timestamp from a null entry in {location}', ) if not isinstance(date, str): raise DeserializationError( f'Failed to deserialize a timestamp from a {type(date)} entry in {location}', ) try: return Timestamp(createTimeStamp(datestr=date, formatstr=formatstr)) except ValueError: raise DeserializationError( f'Failed to deserialize {date} {location} timestamp entry')
def __new__( cls, data_directory: FilePath = None, history_date_start: str = None, cryptocompare: 'Cryptocompare' = None, ): if PriceHistorian.__instance is not None: return PriceHistorian.__instance assert data_directory, 'arguments should be given at the first instantiation' assert history_date_start, 'arguments should be given at the first instantiation' assert cryptocompare, 'arguments should be given at the first instantiation' PriceHistorian.__instance = object.__new__(cls) # get the start date for historical data PriceHistorian._historical_data_start = createTimeStamp( datestr=history_date_start, formatstr="%d/%m/%Y", ) PriceHistorian._cryptocompare = cryptocompare return PriceHistorian.__instance
def __init__( self, user_directory: FilePath, db: DBHandler, eth_accounts: List[EthAddress], historical_data_start: str, msg_aggregator: MessagesAggregator, ): self.poloniex = None self.kraken = None self.bittrex = None self.bitmex = None self.binance = None self.msg_aggregator = msg_aggregator self.user_directory = user_directory self.db = db self.eth_accounts = eth_accounts # get the start date for historical data self.historical_data_start = createTimeStamp(historical_data_start, formatstr="%d/%m/%Y") # If this flag is true we attempt to read from the manually logged margin positions file self.read_manual_margin_positions = True
def timerange_check( asset_symbol: str, our_asset: Dict[str, Any], our_data: Dict[str, Any], paprika_data: Dict[str, Any], cmc_data: Dict[str, Any], always_keep_our_time: bool, token_address: EthAddress = None, ) -> Dict[str, Any]: """Process the started timestamps from coin paprika and coinmarketcap. Then compare to our data and provide choices to clean up the data. """ if asset_symbol in FIAT_CURRENCIES: # Fiat does not have started date (or we don't care about it) return our_data paprika_started = None if paprika_data: paprika_started = paprika_data['started_at'] cmc_started = None if cmc_data: cmc_started = cmc_data['first_historical_data'] if not cmc_started and not paprika_started and not token_address: print( f'Did not find a started date for asset {asset_symbol} in any of the external APIs' ) return our_data paprika_started_ts = None if paprika_started: paprika_started_ts = createTimeStamp(paprika_started, formatstr='%Y-%m-%dT%H:%M:%SZ') cmc_started_ts = None if cmc_data: cmc_started_ts = iso8601ts_to_timestamp(cmc_started) if asset_symbol in PREFER_OUR_STARTED: assert 'started' in our_asset # Already manually checked return our_data our_started = our_asset.get('started', None) # if it's an eth token entry, get the contract creation time too if token_address: contract_creation_ts = get_token_contract_creation_time(token_address) if not our_started: # If we don't have any data and CMC and paprika agree just use their timestamp if cmc_started == paprika_started and cmc_started is not None: our_data[asset_symbol]['started'] = cmc_started return our_data if our_started and always_keep_our_time: return our_data if our_started is None or our_started != cmc_started or our_started != paprika_started: choices = (1, 2, 3) msg = ( f'For asset {asset_symbol} the started times are: \n' f'(1) Our data: {our_started} -- {tsToDate(our_started) if our_started else ""}\n' f'(2) Coinpaprika: {paprika_started_ts} -- ' f'{tsToDate(paprika_started_ts) if paprika_started_ts else ""}\n' f'(3) Coinmarketcap: {cmc_started_ts} -- ' f'{tsToDate(cmc_started_ts) if cmc_started_ts else ""} \n') if token_address: msg += ( f'(4) Contract creation: {contract_creation_ts} -- ' f'{tsToDate(contract_creation_ts) if contract_creation_ts else ""}\n' ) choices = (1, 2, 3, 4) msg += f'Choose a number (1)-({choices[-1]}) to choose which timestamp to use: ' choice = choose_multiple(msg, choices) if choice == 1: if not our_started: print( 'Chose our timestamp but we got no timestamp. Bailing ...') sys.exit(1) timestamp = our_started elif choice == 2: if not paprika_started_ts: print( "Chose coin paprika's timestamp but it's empty. Bailing ..." ) sys.exit(1) timestamp = paprika_started_ts elif choice == 3: if not cmc_started_ts: print( "Chose coinmarketcap's timestamp but it's empty. Bailing ..." ) sys.exit(1) timestamp = cmc_started_ts elif choice == 4: if not contract_creation_ts: print( "Chose contract creation timestamp but it's empty. Bailing ..." ) sys.exit(1) timestamp = contract_creation_ts our_data[asset_symbol]['started'] = timestamp return our_data
def query_loan_history( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, from_csv: Optional[bool] = False, ) -> List: """ WARNING: Querying from returnLendingHistory endpoing instead of reading from the CSV file can potentially return unexpected/wrong results. That is because the `returnLendingHistory` endpoint has a hidden limit of 12660 results. In our code we use the limit of 12000 but poloniex may change the endpoint to have a lower limit at which case this code will break. To be safe compare results of both CSV and endpoint to make sure they agree! """ try: if from_csv: return self.parseLoanCSV() except (OSError, IOError, csv.Error): pass with self.lock: # We know Loan history cache is a list cache = self.check_trades_cache_list( start_ts=start_ts, end_ts=end_at_least_ts, special_name='loan_history', ) if cache is not None: return cache loans_query_return_limit = 12000 result = self.returnLendingHistory( start_ts=start_ts, end_ts=end_ts, limit=loans_query_return_limit, ) data = list(result) log.debug('Poloniex loan history query', results_num=len(data)) # since I don't think we have any guarantees about order of results # using a set of loan ids is one way to make sure we get no duplicates # if poloniex can guarantee me that the order is going to be ascending/descending # per open/close time then this can be improved id_set = set() while len(result) == loans_query_return_limit: # Find earliest timestamp to re-query the next batch min_ts = end_ts for loan in result: ts = createTimeStamp(loan['close'], formatstr="%Y-%m-%d %H:%M:%S") min_ts = min(min_ts, ts) id_set.add(loan['id']) result = self.returnLendingHistory( start_ts=start_ts, end_ts=min_ts, limit=loans_query_return_limit, ) log.debug('Poloniex loan history query', results_num=len(result)) for loan in result: if loan['id'] not in id_set: data.append(loan) with self.lock: self.update_trades_cache(data, start_ts, end_ts, special_name='loan_history') return data
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]: """ Takes in the trade data dictionary, validates it and returns a trade instance If there is an error it returns an error message in the second part of the tuple """ for field in otc_fields: if field not in data: return None, f'{field} was not provided' if data[field] in ('', None) and field not in otc_optional_fields: return None, f'{field} was empty' if field in otc_numerical_fields and not is_number(data[field]): return None, f'{field} should be a number' # Satisfy mypy typing assert isinstance(data['otc_pair'], str) assert isinstance(data['otc_fee_currency'], str) assert isinstance(data['otc_fee'], str) pair = TradePair(data['otc_pair']) try: first = get_pair_position_asset(pair, 'first') second = get_pair_position_asset(pair, 'second') fee_currency = Asset(data['otc_fee_currency']) except UnknownAsset as e: return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen' try: trade_type = deserialize_trade_type(str(data['otc_type'])) amount = deserialize_asset_amount(data['otc_amount']) rate = deserialize_price(data['otc_rate']) fee = deserialize_fee(data['otc_fee']) except DeserializationError as e: return None, f'Deserialization Error: {str(e)}' try: assert isinstance(data['otc_timestamp'], str) timestamp = createTimeStamp(data['otc_timestamp'], formatstr='%d/%m/%Y %H:%M') except ValueError as e: return None, f'Could not process the given datetime: {e}' log.debug( 'Creating OTC trade data', sensitive_log=True, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, ) if data['otc_fee_currency'] not in (first, second): return None, 'Trade fee currency should be one of the two in the currency pair' if data['otc_type'] not in ('buy', 'sell'): return None, 'Trade type can only be buy or sell' trade = Trade( timestamp=timestamp, location='external', pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(data['otc_link']), notes=str(data['otc_notes']), ) return trade, ''