def process_polo_loans(data, start_ts, end_ts):
new_data = list()
for loan in reversed(data):
close_time = createTimeStamp(loan['close'],
formatstr="%Y-%m-%d %H:%M:%S")
open_time = createTimeStamp(loan['open'],
formatstr="%Y-%m-%d %H:%M:%S")
if open_time < start_ts:
continue
if close_time > end_ts:
break
loan_data = {
'open_time': open_time,
'close_time': close_time,
'currency': loan['currency'],
'fee': FVal(loan['fee']),
'earned': FVal(loan['earned']),
'amount_lent': FVal(loan['amount']),
}
log.debug('processing poloniex loan', **make_sensitive(loan_data))
new_data.append(loan_data)
new_data.sort(key=lambda loan: loan['open_time'])
return new_data
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
"""
trade_type = trade_type_from_string(poloniex_trade['type'])
amount = FVal(poloniex_trade['amount'])
rate = FVal(poloniex_trade['rate'])
perc_fee = FVal(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = createTimeStamp(poloniex_trade['date'], formatstr="%Y-%m-%d %H:%M:%S")
cost = rate * amount
if trade_type == TradeType.BUY:
fee = amount * perc_fee
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = cost * perc_fee
fee_currency = base_currency
else:
raise ValueError('Got unexpected trade type "{}" for poloniex trade'.format(trade_type))
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location='poloniex',
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
def query_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
market: Optional[TradePair] = None,
count: Optional[int] = None,
) -> List:
options: Dict[str, Union[str, int]] = dict()
cache = self.check_trades_cache_list(start_ts, end_at_least_ts)
if market is not None:
options['market'] = world_pair_to_bittrex(market)
elif cache is not None:
return cache
if count is not None:
options['count'] = count
order_history = self.api_query('getorderhistory', options)
log.debug('binance order history result',
results_num=len(order_history))
returned_history = list()
for order in order_history:
order_timestamp = createTimeStamp(order['TimeStamp'],
formatstr="%Y-%m-%dT%H:%M:%S.%f")
if start_ts is not None and order_timestamp < start_ts:
continue
if end_ts is not None and order_timestamp > end_ts:
break
order['TimeStamp'] = order_timestamp
returned_history.append(order)
self.update_trades_cache(returned_history, start_ts, end_ts)
return returned_history
def process_polo_loans(
self,
data: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Dict]:
"""Takes in the list of loans from poloniex as returned by the returnLendingHistory
api call, processes it and returns it into our loan format
"""
new_data = list()
for loan in reversed(data):
close_time = createTimeStamp(loan['close'],
formatstr="%Y-%m-%d %H:%M:%S")
open_time = createTimeStamp(loan['open'],
formatstr="%Y-%m-%d %H:%M:%S")
if open_time < start_ts:
continue
if close_time > end_ts:
break
try:
loan_data = {
'open_time': open_time,
'close_time': close_time,
'currency': asset_from_poloniex(loan['currency']),
'fee': FVal(loan['fee']),
'earned': FVal(loan['earned']),
'amount_lent': FVal(loan['amount']),
}
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found poloniex loan with unsupported asset'
f' {e.asset_name}. Ignoring it.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found poloniex loan with unknown asset'
f' {e.asset_name}. Ignoring it.', )
continue
log.debug('processing poloniex loan', **make_sensitive(loan_data))
new_data.append(loan_data)
new_data.sort(key=lambda loan: loan['open_time'])
return new_data
def _post_process(before: Dict) -> Dict:
"""Poloniex uses datetimes so turn them into timestamps here"""
after = before
if ('return' in after):
if (isinstance(after['return'], list)):
for x in range(0, len(after['return'])):
if (isinstance(after['return'][x], dict)):
if ('datetime' in after['return'][x]
and 'timestamp' not in after['return'][x]):
after['return'][x]['timestamp'] = float(
createTimeStamp(after['return'][x]['datetime']), )
return after
def from_otc_trade(trade: Dict[str, Any]) -> Dict[str, Any]:
ts = createTimeStamp(trade['otc_timestamp'], formatstr='%d/%m/%Y %H:%M')
new_trade = {
'timestamp': ts,
'location': 'external',
'pair': trade['otc_pair'],
'type': trade['otc_type'],
'amount': str(trade['otc_amount']),
'rate': str(trade['otc_rate']),
'fee': str(trade['otc_fee']),
'fee_currency': trade['otc_fee_currency'],
'link': trade['otc_link'],
'notes': trade['otc_notes'],
}
if 'otc_id' in trade:
new_trade['id'] = trade['otc_id']
return new_trade
def deserialize_timestamp_from_date(date: str, formatstr: str,
location: str) -> Timestamp:
"""Deserializes a timestamp from a date entry depending on the format str
Can throw DeserializationError if the data is not as expected
"""
if not date:
raise DeserializationError(
f'Failed to deserialize a timestamp from a null entry in {location}',
)
if not isinstance(date, str):
raise DeserializationError(
f'Failed to deserialize a timestamp from a {type(date)} entry in {location}',
)
try:
return Timestamp(createTimeStamp(datestr=date, formatstr=formatstr))
except ValueError:
raise DeserializationError(
f'Failed to deserialize {date} {location} timestamp entry')
def __new__(
cls,
data_directory: FilePath = None,
history_date_start: str = None,
cryptocompare: 'Cryptocompare' = None,
):
if PriceHistorian.__instance is not None:
return PriceHistorian.__instance
assert data_directory, 'arguments should be given at the first instantiation'
assert history_date_start, 'arguments should be given at the first instantiation'
assert cryptocompare, 'arguments should be given at the first instantiation'
PriceHistorian.__instance = object.__new__(cls)
# get the start date for historical data
PriceHistorian._historical_data_start = createTimeStamp(
datestr=history_date_start,
formatstr="%d/%m/%Y",
)
PriceHistorian._cryptocompare = cryptocompare
return PriceHistorian.__instance
def __init__(
self,
user_directory: FilePath,
db: DBHandler,
eth_accounts: List[EthAddress],
historical_data_start: str,
msg_aggregator: MessagesAggregator,
):
self.poloniex = None
self.kraken = None
self.bittrex = None
self.bitmex = None
self.binance = None
self.msg_aggregator = msg_aggregator
self.user_directory = user_directory
self.db = db
self.eth_accounts = eth_accounts
# get the start date for historical data
self.historical_data_start = createTimeStamp(historical_data_start,
formatstr="%d/%m/%Y")
# If this flag is true we attempt to read from the manually logged margin positions file
self.read_manual_margin_positions = True
def timerange_check(
asset_symbol: str,
our_asset: Dict[str, Any],
our_data: Dict[str, Any],
paprika_data: Dict[str, Any],
cmc_data: Dict[str, Any],
always_keep_our_time: bool,
token_address: EthAddress = None,
) -> Dict[str, Any]:
"""Process the started timestamps from coin paprika and coinmarketcap.
Then compare to our data and provide choices to clean up the data.
"""
if asset_symbol in FIAT_CURRENCIES:
# Fiat does not have started date (or we don't care about it)
return our_data
paprika_started = None
if paprika_data:
paprika_started = paprika_data['started_at']
cmc_started = None
if cmc_data:
cmc_started = cmc_data['first_historical_data']
if not cmc_started and not paprika_started and not token_address:
print(
f'Did not find a started date for asset {asset_symbol} in any of the external APIs'
)
return our_data
paprika_started_ts = None
if paprika_started:
paprika_started_ts = createTimeStamp(paprika_started,
formatstr='%Y-%m-%dT%H:%M:%SZ')
cmc_started_ts = None
if cmc_data:
cmc_started_ts = iso8601ts_to_timestamp(cmc_started)
if asset_symbol in PREFER_OUR_STARTED:
assert 'started' in our_asset
# Already manually checked
return our_data
our_started = our_asset.get('started', None)
# if it's an eth token entry, get the contract creation time too
if token_address:
contract_creation_ts = get_token_contract_creation_time(token_address)
if not our_started:
# If we don't have any data and CMC and paprika agree just use their timestamp
if cmc_started == paprika_started and cmc_started is not None:
our_data[asset_symbol]['started'] = cmc_started
return our_data
if our_started and always_keep_our_time:
return our_data
if our_started is None or our_started != cmc_started or our_started != paprika_started:
choices = (1, 2, 3)
msg = (
f'For asset {asset_symbol} the started times are: \n'
f'(1) Our data: {our_started} -- {tsToDate(our_started) if our_started else ""}\n'
f'(2) Coinpaprika: {paprika_started_ts} -- '
f'{tsToDate(paprika_started_ts) if paprika_started_ts else ""}\n'
f'(3) Coinmarketcap: {cmc_started_ts} -- '
f'{tsToDate(cmc_started_ts) if cmc_started_ts else ""} \n')
if token_address:
msg += (
f'(4) Contract creation: {contract_creation_ts} -- '
f'{tsToDate(contract_creation_ts) if contract_creation_ts else ""}\n'
)
choices = (1, 2, 3, 4)
msg += f'Choose a number (1)-({choices[-1]}) to choose which timestamp to use: '
choice = choose_multiple(msg, choices)
if choice == 1:
if not our_started:
print(
'Chose our timestamp but we got no timestamp. Bailing ...')
sys.exit(1)
timestamp = our_started
elif choice == 2:
if not paprika_started_ts:
print(
"Chose coin paprika's timestamp but it's empty. Bailing ..."
)
sys.exit(1)
timestamp = paprika_started_ts
elif choice == 3:
if not cmc_started_ts:
print(
"Chose coinmarketcap's timestamp but it's empty. Bailing ..."
)
sys.exit(1)
timestamp = cmc_started_ts
elif choice == 4:
if not contract_creation_ts:
print(
"Chose contract creation timestamp but it's empty. Bailing ..."
)
sys.exit(1)
timestamp = contract_creation_ts
our_data[asset_symbol]['started'] = timestamp
return our_data
def query_loan_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
from_csv: Optional[bool] = False,
) -> List:
"""
WARNING: Querying from returnLendingHistory endpoing instead of reading from
the CSV file can potentially return unexpected/wrong results.
That is because the `returnLendingHistory` endpoint has a hidden limit
of 12660 results. In our code we use the limit of 12000 but poloniex may change
the endpoint to have a lower limit at which case this code will break.
To be safe compare results of both CSV and endpoint to make sure they agree!
"""
try:
if from_csv:
return self.parseLoanCSV()
except (OSError, IOError, csv.Error):
pass
with self.lock:
# We know Loan history cache is a list
cache = self.check_trades_cache_list(
start_ts=start_ts,
end_ts=end_at_least_ts,
special_name='loan_history',
)
if cache is not None:
return cache
loans_query_return_limit = 12000
result = self.returnLendingHistory(
start_ts=start_ts,
end_ts=end_ts,
limit=loans_query_return_limit,
)
data = list(result)
log.debug('Poloniex loan history query', results_num=len(data))
# since I don't think we have any guarantees about order of results
# using a set of loan ids is one way to make sure we get no duplicates
# if poloniex can guarantee me that the order is going to be ascending/descending
# per open/close time then this can be improved
id_set = set()
while len(result) == loans_query_return_limit:
# Find earliest timestamp to re-query the next batch
min_ts = end_ts
for loan in result:
ts = createTimeStamp(loan['close'], formatstr="%Y-%m-%d %H:%M:%S")
min_ts = min(min_ts, ts)
id_set.add(loan['id'])
result = self.returnLendingHistory(
start_ts=start_ts,
end_ts=min_ts,
limit=loans_query_return_limit,
)
log.debug('Poloniex loan history query', results_num=len(result))
for loan in result:
if loan['id'] not in id_set:
data.append(loan)
with self.lock:
self.update_trades_cache(data, start_ts, end_ts, special_name='loan_history')
return data
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
"""
Takes in the trade data dictionary, validates it and returns a trade instance
If there is an error it returns an error message in the second part of the tuple
"""
for field in otc_fields:
if field not in data:
return None, f'{field} was not provided'
if data[field] in ('', None) and field not in otc_optional_fields:
return None, f'{field} was empty'
if field in otc_numerical_fields and not is_number(data[field]):
return None, f'{field} should be a number'
# Satisfy mypy typing
assert isinstance(data['otc_pair'], str)
assert isinstance(data['otc_fee_currency'], str)
assert isinstance(data['otc_fee'], str)
pair = TradePair(data['otc_pair'])
try:
first = get_pair_position_asset(pair, 'first')
second = get_pair_position_asset(pair, 'second')
fee_currency = Asset(data['otc_fee_currency'])
except UnknownAsset as e:
return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
try:
trade_type = deserialize_trade_type(str(data['otc_type']))
amount = deserialize_asset_amount(data['otc_amount'])
rate = deserialize_price(data['otc_rate'])
fee = deserialize_fee(data['otc_fee'])
except DeserializationError as e:
return None, f'Deserialization Error: {str(e)}'
try:
assert isinstance(data['otc_timestamp'], str)
timestamp = createTimeStamp(data['otc_timestamp'],
formatstr='%d/%m/%Y %H:%M')
except ValueError as e:
return None, f'Could not process the given datetime: {e}'
log.debug(
'Creating OTC trade data',
sensitive_log=True,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
if data['otc_fee_currency'] not in (first, second):
return None, 'Trade fee currency should be one of the two in the currency pair'
if data['otc_type'] not in ('buy', 'sell'):
return None, 'Trade type can only be buy or sell'
trade = Trade(
timestamp=timestamp,
location='external',
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(data['otc_link']),
notes=str(data['otc_notes']),
)
return trade, ''