def convertTrades(modelList, data,
files): # convert all read csv data to tradelist
tradeList = core.TradeList()
feeList = core.TradeList()
matches = []
i = 0
for frame in data: # convert every DataFrame
localLogger.info('convert ' + files[i])
heading = frame.columns.tolist()
trades, fees, frameMatched, skippedRows = convertTradesSingle(
modelList, frame, files[i])
if skippedRows > 0:
localLogger.info(str(skippedRows) + ' skiped in ' + str(heading))
if frameMatched == False:
localLogger.info('no match for ' + str(heading))
elif trades.isEmpty():
localLogger.info('no trades converted for ' + str(heading))
else:
tradeList.mergeTradeList(trades)
if fees.isEmpty():
localLogger.info('no fees converted for ' + str(heading))
else:
feeList.mergeTradeList(fees)
matches.append(frameMatched)
i += 1
return tradeList, feeList, matches
def modelCallback_exodus(headernames, dataFrame):
tradeList = core.TradeList()
feeList = core.TradeList()
skippedRows = 0
exchangeRegex = re.compile(r'^(exchange)$')
for row in range(dataFrame.shape[0]):
exchangeMatch = exchangeRegex.match(dataFrame[headernames[1]][row]) # check type
isFee = str(dataFrame[headernames[4]][row]) != 'nan'
if not exchangeMatch and not isFee: # no trade and no fee
skippedRows += 1
continue # skip row
if exchangeMatch: # if exchange
tempTrade_out = core.Trade() # out
tempTrade_in = core.Trade() # in
# get id
tempTrade_out.externId = str(dataFrame[headernames[6]][row])
tempTrade_in.externId = str(dataFrame[headernames[10]][row])
# get date
tempTrade_out.date = convertDate(dataFrame[headernames[0]][row])
tempTrade_in.date = convertDate(dataFrame[headernames[0]][row])
# get type
tempTrade_out.tradeType = 'trade'
tempTrade_in.tradeType = 'trade'
# get coin
tempTrade_out.coin = str(dataFrame[headernames[3]][row])
tempTrade_in.coin = str(dataFrame[headernames[9]][row])
# swap Coin Name
swapCoinName(tempTrade_out)
swapCoinName(tempTrade_in)
# get amount
tempTrade_out.amount = dataFrame[headernames[2]][row]
tempTrade_in.amount = dataFrame[headernames[8]][row]
# set exchange and wallet
exchange = 'shapeshift'
tempTrade_out.exchange = tempTrade_in.exchange = 'shapeshift'
tempTrade_out.wallet = tempTrade_in.wallet = 'exodus'
# set id
if not tempTrade_out.tradeID:
tempTrade_out.generateID()
if not tempTrade_in.tradeID:
tempTrade_in.generateID()
# add trades to tradeList
if not tradeList.addTradePair(tempTrade_out, tempTrade_in):
skippedRows += 1
else:
exchange = ''
# fees
if isFee: # if fee
fee = createFee(date=convertDate(dataFrame[headernames[0]][row]),
amountStr=dataFrame[headernames[4]][row],
maincoin=dataFrame[headernames[5]][row],
exchange=exchange,
wallet='exodus')
fee.generateID()
feeList.addTrade(fee)
return tradeList, feeList, skippedRows
def undoRemoveAddTrades(self):
if self.changedNumberStack:
num = self.changedNumberStack.pop()
if num < 0: # trades have been removed
tradeList = core.TradeList()
tradeList.trades = self.deletedTradesStack[num:]
self.deletedTradesStack = self.deletedTradesStack[0:num]
rows = self.deletedIndexesStack[num:]
self.deletedIndexesStack = self.deletedIndexesStack[0:num]
self.insertTrades(rows, tradeList)
if num > 0: # trades have been added
tradeList = core.TradeList()
tradeList.trades = self.addedTradesStack[-num:]
self.addedTradesStack = self.addedTradesStack[0:-num]
self.removeTrades(tradeList)
def finishImport(self):
newTrades = core.TradeList()
newTrades.mergeTradeList(self.getNewTrades())
newTrades.mergeTradeList(self.getNewFees())
self.controller.tradeList.addTrades(newTrades)
# jump to TradesPage
self.controller.showFrame(self.controller.TRADESPAGEINDEX)
def modelCallback_3(headernames, dataFrame):
tradeList = core.TradeList()
feeList = core.TradeList()
skippedRows = 0
for row in range(dataFrame.shape[0]):
tempTrade_sub = core.Trade()
# get id
if headernames[4]:
tempTrade_sub.externId = str(dataFrame[headernames[4]][row])
# get date
tempTrade_sub.date = convertDate(dataFrame[headernames[0]][row])
# get type
tempTrade_sub.tradeType = 'trade'
# get coin
tempTrade_sub.coin = (dataFrame[headernames[2]][row]).upper()
# swap Coin Name
swapCoinName(tempTrade_sub)
# get amount sub
if isinstance(dataFrame[headernames[3]][row], numbers.Number): # if amount is number
amount = dataFrame[headernames[3]][row]
else: # now number so use regex to extract the number
amount = float(pat.NUMBER_REGEX.match(dataFrame[headernames[3]][row]).group(1))
if amount:
tempTrade_sub.amount = amount
else:
skippedRows += 1
continue
# add row to tradeList
if not tempTrade_sub.tradeID:
tempTrade_sub.generateID()
if not tradeList.addTrade(tempTrade_sub):
skippedRows += 1
# fees
try:
if headernames[5]: # if fee
feecoin = tempTrade_sub.coin
fee = createFee(date=tempTrade_sub.date, amountStr=dataFrame[headernames[5]][row], maincoin=feecoin,
exchange=tempTrade_sub.exchange, externId=tempTrade_sub.externId)
fee.generateID()
feeList.addTrade(fee)
except Exception as ex: # do not skip line if error, just ignore fee
localLogger.warning('error in Converter: ' + str(ex))
return tradeList, feeList, skippedRows
def convertTradesSingle(
modelList, frame,
filename): # convert one row of read csv data to tradelist
heading = frame.columns.tolist()
trades = core.TradeList()
fees = core.TradeList()
skippedRows = 0
frameMatched = False
for model in modelList: # match with every model until a match is found
if model.isMatch(heading):
frameMatched = True
try:
trades, fees, skippedRows = model.convertDataFrame(
frame) # convert frame using first matching model
# if trades could be extracted skip other models, otherwise keep trying
if not (trades.isEmpty() and fees.isEmpty()):
break
except Exception as ex:
#if exception occurs, try other models
localLogger.warning('converting ' + str(filename) +
' failed with model :' +
str(model.modelCallback) + '; ' + str(ex))
return trades, fees, frameMatched, skippedRows
def deleteTrades(self, rows):
self.beginResetModel()
rows.sort(reverse=True)
self.changedNumberStack.append(-len(rows))
tradeList = core.TradeList()
for row in rows:
tradeList.addTrade(self.trades[row])
self.deletedTradesStack.append(self.trades[row])
self.deletedIndexesStack.append(row)
for row in rows:
self.beginRemoveRows(qtcore.QModelIndex(), row, row)
self.trades.pop(row)
self.endRemoveRows()
# save new trades
self.saveTrades()
# emit trades added
self.tradesRemoved.emit(tradeList)
self.endResetModel()
return True
def insertTrades(self, rows, trades):
try:
for index in sorted(range(len(rows)), key=rows.__getitem__):
row = rows[index]
trade = trades[index]
self.beginInsertRows(qtcore.QModelIndex(), row, row)
self.trades.insert(row, trade)
self.endInsertRows()
except TypeError:
row = rows
trade = trades
self.beginInsertRows(qtcore.QModelIndex(), row, row)
self.trades.insert(row, trade)
self.endInsertRows()
trades = core.TradeList()
trades.addTrade(trade)
# save new trades
self.saveTrades()
# emit trades added
self.tradesAdded.emit(trades)
def modelCallback_TradeList(headernames, dataFrame):
tradeList = core.TradeList()
feeList = core.TradeList()
skippedRows = 0
for row in range(dataFrame.shape[0]):
try:
trade = core.Trade()
# date
trade.date = convertDate(dataFrame[headernames[0]][row])
# type
trade.tradeType = dataFrame[headernames[1]][row]
trade.coin = dataFrame[headernames[2]][row]
trade.amount = float(dataFrame[headernames[3]][row])
trade.valueLoaded = False
if headernames[6]:
valueLoaded = dataFrame[headernames[6]][row]
if valueLoaded:
headers = dataFrame.columns.tolist()
keysImport = []
valueHeaders = []
for header in headers: # check all header for included historical values
valueMatch = pat.TRADELIST_VALUE_REGEX.match(header)
if valueMatch:
valueHeaders.append(valueMatch.group(0))
keysImport.append(valueMatch.group(1))
trade.valueLoaded = True
for key in core.CoinValue(): # check if all needed currencies are included
if key not in keysImport:
trade.valueLoaded = False
for valueInd in range(len(valueHeaders)): # load all included historical values
trade.setValue(keysImport[valueInd], float(dataFrame[valueHeaders[valueInd]][row]))
# exchange
if headernames[7]:
exchange = str(dataFrame[headernames[7]][row])
if exchange != 'nan':
trade.exchange = exchange
# extern id
if headernames[8]:
externId = str(dataFrame[headernames[8]][row])
if externId != 'nan':
trade.externId = externId
# wallet
if headernames[9]:
wallet = str(dataFrame[headernames[9]][row])
if wallet != 'nan':
trade.wallet = wallet
# id
if headernames[4]:
trade.tradeID = str(dataFrame[headernames[4]][row])
else:
trade.generateID()
# partner id
if headernames[5]:
tradePartnerId = str(dataFrame[headernames[5]][row])
if tradePartnerId != 'nan':
trade.tradePartnerId = tradePartnerId
swapCoinName(trade)
if trade.tradeType == 'fee':
feeList.addTrade(trade)
else:
tradeList.addTrade(trade)
except Exception as ex:
localLogger.warning('error in Converter: ' + str(ex))
skippedRows += 1
return tradeList, feeList, skippedRows
def modelCallback_Template1(headernames, dataFrame):
tradeList = core.TradeList()
feeList = core.TradeList()
skippedRows = 0
for row in range(dataFrame.shape[0]):
try:
date = convertDate(dataFrame[headernames[0]][row])
tradeType = dataFrame[headernames[1]][row].lower()
if headernames[6]:
exchange = str(dataFrame[headernames[6]][row]).lower()
if exchange == 'nan':
exchange = ''
if tradeType == 'trade':
tempTrade_sell = core.Trade() # sell
tempTrade_buy = core.Trade() # buy
# get date
tempTrade_sell.date = date
tempTrade_buy.date = date
# get type
tempTrade_sell.tradeType = tradeType
tempTrade_buy.tradeType = tradeType
# get coin
tempTrade_sell.coin = (dataFrame[headernames[5]][row]).upper()
tempTrade_buy.coin = (dataFrame[headernames[3]][row]).upper()
# swap Coin Name
swapCoinName(tempTrade_sell)
swapCoinName(tempTrade_buy)
# get amount
tempTrade_sell.amount = - abs(dataFrame[headernames[4]][row])
tempTrade_buy.amount = abs(dataFrame[headernames[2]][row])
# set exchange
tempTrade_sell.exchange = exchange
tempTrade_buy.exchange = exchange
# set id
if not tempTrade_sell.tradeID:
tempTrade_sell.generateID()
if not tempTrade_buy.tradeID:
tempTrade_buy.generateID()
# add trades to tradeList
if not tradeList.addTradePair(tempTrade_sell, tempTrade_buy):
skippedRows += 1
except Exception as ex:
localLogger.warning('error in Converter Template1: ' + str(ex))
skippedRows += 1
# rewards
try:
date = convertDate(dataFrame[headernames[0]][row])
tradeType = dataFrame[headernames[1]][row].lower()
if headernames[6]:
exchange = str(dataFrame[headernames[6]][row]).lower()
if exchange == 'nan':
exchange = ''
if tradeType == 'reward':
tempTrade_buy = core.Trade() # buy
# get date
tempTrade_buy.date = date
# get type
tempTrade_buy.tradeType = tradeType
# get coin
tempTrade_buy.coin = (dataFrame[headernames[3]][row]).upper()
# swap Coin Name
swapCoinName(tempTrade_buy)
# get amount
tempTrade_buy.amount = abs(dataFrame[headernames[2]][row])
# set exchange
tempTrade_buy.exchange = exchange
# set id
if not tempTrade_buy.tradeID:
tempTrade_buy.generateID()
# add trades to tradeList
if not tradeList.addTrade(tempTrade_buy):
skippedRows += 1
except Exception as ex:
localLogger.warning('error in Converter Template1: ' + str(ex))
skippedRows += 1
# fees
try:
tradeType = dataFrame[headernames[1]][row].lower()
if tradeType == "fee":
if headernames[7] and str(dataFrame[headernames[7]][row]) != 'nan': # if fee
if headernames[8] and str(dataFrame[headernames[8]][row]) != 'nan': # if fee coin
# use fee coin
feecoin = dataFrame[headernames[8]][row]
else: # no fee coin
# use buy coin
feecoin = tempTrade_buy.coin
# set coin amount
fee = createFee(date=date, amountStr=dataFrame[headernames[7]][row], maincoin=feecoin,
subcoin=tempTrade_sell.coin,
exchange=exchange)
fee.generateID()
feeList.addTrade(fee)
except Exception as ex: # do not skip line if error, just ignore fee
localLogger.warning('error in Converter Template1: ' + str(ex))
return tradeList, feeList, skippedRows
def modelCallback_5(headernames, dataFrame):
tradeList = core.TradeList()
feeList = core.TradeList()
skippedRows = 0
for row in range(dataFrame.shape[0]):
if headernames[8]: # if state included
if re.match(r'^.*?(CANCELED).*?$', dataFrame[headernames[8]][row], re.IGNORECASE): # check state
skippedRows += 1
continue # skip row
tempTrade_sub = core.Trade() # sub
tempTrade_main = core.Trade() # main
# get id
if headernames[4]:
tempTrade_sub.externId = str(dataFrame[headernames[4]][row])
tempTrade_main.externId = str(dataFrame[headernames[4]][row])
# get date
tempTrade_sub.date = convertDate(dataFrame[headernames[0]][row])
tempTrade_main.date = convertDate(dataFrame[headernames[0]][row])
# get type
tempTrade_sub.tradeType = 'trade'
tempTrade_main.tradeType = 'trade'
# get coin
tempTrade_sub.coin = re.match(r'^(.*)/.*$', dataFrame[headernames[1]][row]).group(1).upper()
tempTrade_main.coin = re.match(r'^.*/(.*)$', dataFrame[headernames[1]][row]).group(1).upper()
# swap Coin Name
swapCoinName(tempTrade_sub)
swapCoinName(tempTrade_main)
# get amount
if isinstance(dataFrame[headernames[3]][row], numbers.Number): # if amount is number
amount = dataFrame[headernames[3]][row]
else: # now number so use regex to extract the number
amount = float(pat.NUMBER_REGEX.match(dataFrame[headernames[3]][row]).group(1))
if isinstance(dataFrame[headernames[2]][row], numbers.Number): # if price is number
price = dataFrame[headernames[2]][row] * amount * -1
else: # no number so use regex
price = float(pat.NUMBER_REGEX.match(dataFrame[headernames[2]][row]).group(1)) * amount * -1
tempTrade_sub.amount = amount
tempTrade_main.amount = price
# set id
if not tempTrade_sub.tradeID:
tempTrade_sub.generateID()
if not tempTrade_main.tradeID:
tempTrade_main.generateID()
# add trades to tradeList
if not tradeList.addTradePair(tempTrade_sub, tempTrade_main):
skippedRows += 1
# fees
try:
if headernames[6]: # if fee
if headernames[7]: # if fee coin
# use fee coin
feecoin = dataFrame[headernames[7]][row]
else: # no fee coin
# use main coin
feecoin = tempTrade_main.coin
# set coin amount
fee = createFee(date=tempTrade_main.date, amountStr=dataFrame[headernames[6]][row], maincoin=feecoin,
subcoin=tempTrade_sub.coin,
exchange=tempTrade_main.exchange, externId=tempTrade_main.externId)
fee.generateID()
feeList.addTrade(fee)
except Exception as ex: # do not skip line if error, just ignore fee
localLogger.warning('error in Converter: ' + str(ex))
return tradeList, feeList, skippedRows
def modelCallback_4(headernames, dataFrame):
tradeList = core.TradeList()
feeList = core.TradeList()
skippedRows = 0
for row in range(dataFrame.shape[0]):
if 'Gebühr' in dataFrame[headernames[1]][row]:
# fees
date = convertDate(dataFrame[headernames[0]][row])
if headernames[7]:
externId = str(dataFrame[headernames[7]][row])
fees = dataFrame[headernames[8]][row]
coin = str(dataFrame[headernames[2]][row])
fee = createFee(date=date, amountStr=fees, maincoin=coin, exchange='bitcoinde', externId=externId)
fee.generateID()
feeList.addTrade(fee)
else:
tempTrade_sub = core.Trade() # sub
tempTrade_main = core.Trade() # main
# get id
if headernames[7]:
tempTrade_sub.externId = str(dataFrame[headernames[7]][row])
tempTrade_main.externId = str(dataFrame[headernames[7]][row])
# get date
tempTrade_sub.date = convertDate(dataFrame[headernames[0]][row])
tempTrade_main.date = convertDate(dataFrame[headernames[0]][row])
# set type
tempTrade_sub.tradeType = 'trade'
tempTrade_main.tradeType = 'trade'
# check content type
isBuy = False
if re.match(r'^.*?(Kauf).*?$', dataFrame[headernames[1]][row], re.IGNORECASE):
isBuy = True
elif re.match(r'^.*?(Verkauf).*?$', dataFrame[headernames[1]][row], re.IGNORECASE):
isBuy = False
else: # invalid type
skippedRows += 1
continue
# get coin
if headernames[12]:
coinPairIndex = 12
else:
coinPairIndex = 2
tempTrade_sub.coin = re.match(r'^(.*) / .*$', dataFrame[headernames[coinPairIndex]][row]).group(1).upper()
tempTrade_main.coin = re.match(r'^.* / (.*)$', dataFrame[headernames[coinPairIndex]][row]).group(1).upper()
# swap Coin Name
swapCoinName(tempTrade_sub)
swapCoinName(tempTrade_main)
# get amount wo fees
if isinstance(dataFrame[headernames[4]][row], numbers.Number): # if amount is number
amount_wo_fees = abs(dataFrame[headernames[4]][row])
else: # now number so use regex to extract the number
amount_wo_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[4]][row]).group(1)))
# get price wo fees
if isinstance(dataFrame[headernames[3]][row], numbers.Number): # if price is number
price_wo_fees = abs(dataFrame[headernames[3]][row])
else: # no number so use regex
price_wo_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[3]][row]).group(1)))
# get amount w fees
if isinstance(dataFrame[headernames[6]][row], numbers.Number): # if amount is number
amount_w_fees = abs(dataFrame[headernames[6]][row])
else: # now number so use regex to extract the number
amount_w_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[6]][row]).group(1)))
# get price w fees
if headernames[9]: # if fidor fee included
mainFeeIndex = 9
else:
mainFeeIndex = 5
if isinstance(dataFrame[headernames[mainFeeIndex]][row], numbers.Number): # if price is number
price_w_fees = abs(dataFrame[headernames[mainFeeIndex]][row])
else: # no number so use regex
price_w_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[mainFeeIndex]][row]).group(1)))
# get sign
if isBuy == True:
tempTrade_sub.amount = amount_wo_fees
tempTrade_main.amount = -price_w_fees
fees_sub = amount_wo_fees - amount_w_fees
fees_main = price_wo_fees - price_w_fees
else:
tempTrade_sub.amount = amount_w_fees
tempTrade_main.amount = -price_wo_fees
fees_sub = -(amount_wo_fees - amount_w_fees)
fees_main = -(price_wo_fees - price_w_fees)
# set exchange
tempTrade_main.exchange = 'bitcoinde'
tempTrade_sub.exchange = 'bitcoinde'
# set id
if not tempTrade_sub.tradeID:
tempTrade_sub.generateID()
if not tempTrade_main.tradeID:
tempTrade_main.generateID()
# add trades to tradeList
if not tradeList.addTradePair(tempTrade_sub, tempTrade_main):
skippedRows += 1
# fees
try:
# get fee
feeMain = createFee(date=tempTrade_main.date, amountStr=fees_main, maincoin=tempTrade_main.coin,
exchange='bitcoinde', externId=tempTrade_main.externId)
feeSub = createFee(date=tempTrade_sub.date, amountStr=fees_sub, maincoin=tempTrade_sub.coin,
exchange='bitcoinde', externId=tempTrade_sub.externId)
feeMain.generateID()
feeSub.generateID()
feeList.addTrade(feeMain)
feeList.addTrade(feeSub)
except Exception as ex: # do not skip line if error, just ignore fee
localLogger.warning('error in Converter: ' + str(ex))
return tradeList, feeList, skippedRows
def __init__(self):
super(WebApiInterface, self).__init__()
# self.histPrices = {}
self.tradeBuffer = core.TradeList()
# %% import data
mypath = r'D:\workspace\python\PyCryptoPortfolio\importdata'
# mypath = r'D:\workspace\spyder\PyCryptoPortfolio\data\kaj'
files, content = importer.loadTrades(mypath)
settings.mySettings.setPath('D:\workspace\python\PyCryptoPortfolio\Data')
headings = [frame.columns.tolist() for frame in content]
# fileheadings = []
# for i in range(len(files)):
# fileheadings.append([files[i],headings[i]])
# %% convert data
# tradeList, matches = importer.convertTrades(models.IMPORT_MODEL_LIST, content)
tradeList = core.TradeList()
tradeList.fromCsv(r'D:\workspace\python\PyCryptoPortfolio\Data\Trades.csv')
# tradeList.updateValues()
tradeFrame = tradeList.toDataFrameComplete()
coinList = core.CoinList()
coinList.addTrades(tradeList)
coinList.updateCurrentValues()
coinFrame = coinList.toDataFrame()
coinDict = coinFrame.to_dict('index')
coinIndex = 3
# tradeMatcher = coinList.coins[coinIndex].tradeMatcher
# matchedFrame = coinList.coins[coinIndex].tradeMatcher.toDataFrame('EUR')