def convertTrades(modelList, data, files): # convert all read csv data to tradelist tradeList = core.TradeList() feeList = core.TradeList() matches = [] i = 0 for frame in data: # convert every DataFrame localLogger.info('convert ' + files[i]) heading = frame.columns.tolist() trades, fees, frameMatched, skippedRows = convertTradesSingle( modelList, frame, files[i]) if skippedRows > 0: localLogger.info(str(skippedRows) + ' skiped in ' + str(heading)) if frameMatched == False: localLogger.info('no match for ' + str(heading)) elif trades.isEmpty(): localLogger.info('no trades converted for ' + str(heading)) else: tradeList.mergeTradeList(trades) if fees.isEmpty(): localLogger.info('no fees converted for ' + str(heading)) else: feeList.mergeTradeList(fees) matches.append(frameMatched) i += 1 return tradeList, feeList, matches
def modelCallback_exodus(headernames, dataFrame): tradeList = core.TradeList() feeList = core.TradeList() skippedRows = 0 exchangeRegex = re.compile(r'^(exchange)$') for row in range(dataFrame.shape[0]): exchangeMatch = exchangeRegex.match(dataFrame[headernames[1]][row]) # check type isFee = str(dataFrame[headernames[4]][row]) != 'nan' if not exchangeMatch and not isFee: # no trade and no fee skippedRows += 1 continue # skip row if exchangeMatch: # if exchange tempTrade_out = core.Trade() # out tempTrade_in = core.Trade() # in # get id tempTrade_out.externId = str(dataFrame[headernames[6]][row]) tempTrade_in.externId = str(dataFrame[headernames[10]][row]) # get date tempTrade_out.date = convertDate(dataFrame[headernames[0]][row]) tempTrade_in.date = convertDate(dataFrame[headernames[0]][row]) # get type tempTrade_out.tradeType = 'trade' tempTrade_in.tradeType = 'trade' # get coin tempTrade_out.coin = str(dataFrame[headernames[3]][row]) tempTrade_in.coin = str(dataFrame[headernames[9]][row]) # swap Coin Name swapCoinName(tempTrade_out) swapCoinName(tempTrade_in) # get amount tempTrade_out.amount = dataFrame[headernames[2]][row] tempTrade_in.amount = dataFrame[headernames[8]][row] # set exchange and wallet exchange = 'shapeshift' tempTrade_out.exchange = tempTrade_in.exchange = 'shapeshift' tempTrade_out.wallet = tempTrade_in.wallet = 'exodus' # set id if not tempTrade_out.tradeID: tempTrade_out.generateID() if not tempTrade_in.tradeID: tempTrade_in.generateID() # add trades to tradeList if not tradeList.addTradePair(tempTrade_out, tempTrade_in): skippedRows += 1 else: exchange = '' # fees if isFee: # if fee fee = createFee(date=convertDate(dataFrame[headernames[0]][row]), amountStr=dataFrame[headernames[4]][row], maincoin=dataFrame[headernames[5]][row], exchange=exchange, wallet='exodus') fee.generateID() feeList.addTrade(fee) return tradeList, feeList, skippedRows
def undoRemoveAddTrades(self): if self.changedNumberStack: num = self.changedNumberStack.pop() if num < 0: # trades have been removed tradeList = core.TradeList() tradeList.trades = self.deletedTradesStack[num:] self.deletedTradesStack = self.deletedTradesStack[0:num] rows = self.deletedIndexesStack[num:] self.deletedIndexesStack = self.deletedIndexesStack[0:num] self.insertTrades(rows, tradeList) if num > 0: # trades have been added tradeList = core.TradeList() tradeList.trades = self.addedTradesStack[-num:] self.addedTradesStack = self.addedTradesStack[0:-num] self.removeTrades(tradeList)
def finishImport(self): newTrades = core.TradeList() newTrades.mergeTradeList(self.getNewTrades()) newTrades.mergeTradeList(self.getNewFees()) self.controller.tradeList.addTrades(newTrades) # jump to TradesPage self.controller.showFrame(self.controller.TRADESPAGEINDEX)
def modelCallback_3(headernames, dataFrame): tradeList = core.TradeList() feeList = core.TradeList() skippedRows = 0 for row in range(dataFrame.shape[0]): tempTrade_sub = core.Trade() # get id if headernames[4]: tempTrade_sub.externId = str(dataFrame[headernames[4]][row]) # get date tempTrade_sub.date = convertDate(dataFrame[headernames[0]][row]) # get type tempTrade_sub.tradeType = 'trade' # get coin tempTrade_sub.coin = (dataFrame[headernames[2]][row]).upper() # swap Coin Name swapCoinName(tempTrade_sub) # get amount sub if isinstance(dataFrame[headernames[3]][row], numbers.Number): # if amount is number amount = dataFrame[headernames[3]][row] else: # now number so use regex to extract the number amount = float(pat.NUMBER_REGEX.match(dataFrame[headernames[3]][row]).group(1)) if amount: tempTrade_sub.amount = amount else: skippedRows += 1 continue # add row to tradeList if not tempTrade_sub.tradeID: tempTrade_sub.generateID() if not tradeList.addTrade(tempTrade_sub): skippedRows += 1 # fees try: if headernames[5]: # if fee feecoin = tempTrade_sub.coin fee = createFee(date=tempTrade_sub.date, amountStr=dataFrame[headernames[5]][row], maincoin=feecoin, exchange=tempTrade_sub.exchange, externId=tempTrade_sub.externId) fee.generateID() feeList.addTrade(fee) except Exception as ex: # do not skip line if error, just ignore fee localLogger.warning('error in Converter: ' + str(ex)) return tradeList, feeList, skippedRows
def convertTradesSingle( modelList, frame, filename): # convert one row of read csv data to tradelist heading = frame.columns.tolist() trades = core.TradeList() fees = core.TradeList() skippedRows = 0 frameMatched = False for model in modelList: # match with every model until a match is found if model.isMatch(heading): frameMatched = True try: trades, fees, skippedRows = model.convertDataFrame( frame) # convert frame using first matching model # if trades could be extracted skip other models, otherwise keep trying if not (trades.isEmpty() and fees.isEmpty()): break except Exception as ex: #if exception occurs, try other models localLogger.warning('converting ' + str(filename) + ' failed with model :' + str(model.modelCallback) + '; ' + str(ex)) return trades, fees, frameMatched, skippedRows
def deleteTrades(self, rows): self.beginResetModel() rows.sort(reverse=True) self.changedNumberStack.append(-len(rows)) tradeList = core.TradeList() for row in rows: tradeList.addTrade(self.trades[row]) self.deletedTradesStack.append(self.trades[row]) self.deletedIndexesStack.append(row) for row in rows: self.beginRemoveRows(qtcore.QModelIndex(), row, row) self.trades.pop(row) self.endRemoveRows() # save new trades self.saveTrades() # emit trades added self.tradesRemoved.emit(tradeList) self.endResetModel() return True
def insertTrades(self, rows, trades): try: for index in sorted(range(len(rows)), key=rows.__getitem__): row = rows[index] trade = trades[index] self.beginInsertRows(qtcore.QModelIndex(), row, row) self.trades.insert(row, trade) self.endInsertRows() except TypeError: row = rows trade = trades self.beginInsertRows(qtcore.QModelIndex(), row, row) self.trades.insert(row, trade) self.endInsertRows() trades = core.TradeList() trades.addTrade(trade) # save new trades self.saveTrades() # emit trades added self.tradesAdded.emit(trades)
def modelCallback_TradeList(headernames, dataFrame): tradeList = core.TradeList() feeList = core.TradeList() skippedRows = 0 for row in range(dataFrame.shape[0]): try: trade = core.Trade() # date trade.date = convertDate(dataFrame[headernames[0]][row]) # type trade.tradeType = dataFrame[headernames[1]][row] trade.coin = dataFrame[headernames[2]][row] trade.amount = float(dataFrame[headernames[3]][row]) trade.valueLoaded = False if headernames[6]: valueLoaded = dataFrame[headernames[6]][row] if valueLoaded: headers = dataFrame.columns.tolist() keysImport = [] valueHeaders = [] for header in headers: # check all header for included historical values valueMatch = pat.TRADELIST_VALUE_REGEX.match(header) if valueMatch: valueHeaders.append(valueMatch.group(0)) keysImport.append(valueMatch.group(1)) trade.valueLoaded = True for key in core.CoinValue(): # check if all needed currencies are included if key not in keysImport: trade.valueLoaded = False for valueInd in range(len(valueHeaders)): # load all included historical values trade.setValue(keysImport[valueInd], float(dataFrame[valueHeaders[valueInd]][row])) # exchange if headernames[7]: exchange = str(dataFrame[headernames[7]][row]) if exchange != 'nan': trade.exchange = exchange # extern id if headernames[8]: externId = str(dataFrame[headernames[8]][row]) if externId != 'nan': trade.externId = externId # wallet if headernames[9]: wallet = str(dataFrame[headernames[9]][row]) if wallet != 'nan': trade.wallet = wallet # id if headernames[4]: trade.tradeID = str(dataFrame[headernames[4]][row]) else: trade.generateID() # partner id if headernames[5]: tradePartnerId = str(dataFrame[headernames[5]][row]) if tradePartnerId != 'nan': trade.tradePartnerId = tradePartnerId swapCoinName(trade) if trade.tradeType == 'fee': feeList.addTrade(trade) else: tradeList.addTrade(trade) except Exception as ex: localLogger.warning('error in Converter: ' + str(ex)) skippedRows += 1 return tradeList, feeList, skippedRows
def modelCallback_Template1(headernames, dataFrame): tradeList = core.TradeList() feeList = core.TradeList() skippedRows = 0 for row in range(dataFrame.shape[0]): try: date = convertDate(dataFrame[headernames[0]][row]) tradeType = dataFrame[headernames[1]][row].lower() if headernames[6]: exchange = str(dataFrame[headernames[6]][row]).lower() if exchange == 'nan': exchange = '' if tradeType == 'trade': tempTrade_sell = core.Trade() # sell tempTrade_buy = core.Trade() # buy # get date tempTrade_sell.date = date tempTrade_buy.date = date # get type tempTrade_sell.tradeType = tradeType tempTrade_buy.tradeType = tradeType # get coin tempTrade_sell.coin = (dataFrame[headernames[5]][row]).upper() tempTrade_buy.coin = (dataFrame[headernames[3]][row]).upper() # swap Coin Name swapCoinName(tempTrade_sell) swapCoinName(tempTrade_buy) # get amount tempTrade_sell.amount = - abs(dataFrame[headernames[4]][row]) tempTrade_buy.amount = abs(dataFrame[headernames[2]][row]) # set exchange tempTrade_sell.exchange = exchange tempTrade_buy.exchange = exchange # set id if not tempTrade_sell.tradeID: tempTrade_sell.generateID() if not tempTrade_buy.tradeID: tempTrade_buy.generateID() # add trades to tradeList if not tradeList.addTradePair(tempTrade_sell, tempTrade_buy): skippedRows += 1 except Exception as ex: localLogger.warning('error in Converter Template1: ' + str(ex)) skippedRows += 1 # rewards try: date = convertDate(dataFrame[headernames[0]][row]) tradeType = dataFrame[headernames[1]][row].lower() if headernames[6]: exchange = str(dataFrame[headernames[6]][row]).lower() if exchange == 'nan': exchange = '' if tradeType == 'reward': tempTrade_buy = core.Trade() # buy # get date tempTrade_buy.date = date # get type tempTrade_buy.tradeType = tradeType # get coin tempTrade_buy.coin = (dataFrame[headernames[3]][row]).upper() # swap Coin Name swapCoinName(tempTrade_buy) # get amount tempTrade_buy.amount = abs(dataFrame[headernames[2]][row]) # set exchange tempTrade_buy.exchange = exchange # set id if not tempTrade_buy.tradeID: tempTrade_buy.generateID() # add trades to tradeList if not tradeList.addTrade(tempTrade_buy): skippedRows += 1 except Exception as ex: localLogger.warning('error in Converter Template1: ' + str(ex)) skippedRows += 1 # fees try: tradeType = dataFrame[headernames[1]][row].lower() if tradeType == "fee": if headernames[7] and str(dataFrame[headernames[7]][row]) != 'nan': # if fee if headernames[8] and str(dataFrame[headernames[8]][row]) != 'nan': # if fee coin # use fee coin feecoin = dataFrame[headernames[8]][row] else: # no fee coin # use buy coin feecoin = tempTrade_buy.coin # set coin amount fee = createFee(date=date, amountStr=dataFrame[headernames[7]][row], maincoin=feecoin, subcoin=tempTrade_sell.coin, exchange=exchange) fee.generateID() feeList.addTrade(fee) except Exception as ex: # do not skip line if error, just ignore fee localLogger.warning('error in Converter Template1: ' + str(ex)) return tradeList, feeList, skippedRows
def modelCallback_5(headernames, dataFrame): tradeList = core.TradeList() feeList = core.TradeList() skippedRows = 0 for row in range(dataFrame.shape[0]): if headernames[8]: # if state included if re.match(r'^.*?(CANCELED).*?$', dataFrame[headernames[8]][row], re.IGNORECASE): # check state skippedRows += 1 continue # skip row tempTrade_sub = core.Trade() # sub tempTrade_main = core.Trade() # main # get id if headernames[4]: tempTrade_sub.externId = str(dataFrame[headernames[4]][row]) tempTrade_main.externId = str(dataFrame[headernames[4]][row]) # get date tempTrade_sub.date = convertDate(dataFrame[headernames[0]][row]) tempTrade_main.date = convertDate(dataFrame[headernames[0]][row]) # get type tempTrade_sub.tradeType = 'trade' tempTrade_main.tradeType = 'trade' # get coin tempTrade_sub.coin = re.match(r'^(.*)/.*$', dataFrame[headernames[1]][row]).group(1).upper() tempTrade_main.coin = re.match(r'^.*/(.*)$', dataFrame[headernames[1]][row]).group(1).upper() # swap Coin Name swapCoinName(tempTrade_sub) swapCoinName(tempTrade_main) # get amount if isinstance(dataFrame[headernames[3]][row], numbers.Number): # if amount is number amount = dataFrame[headernames[3]][row] else: # now number so use regex to extract the number amount = float(pat.NUMBER_REGEX.match(dataFrame[headernames[3]][row]).group(1)) if isinstance(dataFrame[headernames[2]][row], numbers.Number): # if price is number price = dataFrame[headernames[2]][row] * amount * -1 else: # no number so use regex price = float(pat.NUMBER_REGEX.match(dataFrame[headernames[2]][row]).group(1)) * amount * -1 tempTrade_sub.amount = amount tempTrade_main.amount = price # set id if not tempTrade_sub.tradeID: tempTrade_sub.generateID() if not tempTrade_main.tradeID: tempTrade_main.generateID() # add trades to tradeList if not tradeList.addTradePair(tempTrade_sub, tempTrade_main): skippedRows += 1 # fees try: if headernames[6]: # if fee if headernames[7]: # if fee coin # use fee coin feecoin = dataFrame[headernames[7]][row] else: # no fee coin # use main coin feecoin = tempTrade_main.coin # set coin amount fee = createFee(date=tempTrade_main.date, amountStr=dataFrame[headernames[6]][row], maincoin=feecoin, subcoin=tempTrade_sub.coin, exchange=tempTrade_main.exchange, externId=tempTrade_main.externId) fee.generateID() feeList.addTrade(fee) except Exception as ex: # do not skip line if error, just ignore fee localLogger.warning('error in Converter: ' + str(ex)) return tradeList, feeList, skippedRows
def modelCallback_4(headernames, dataFrame): tradeList = core.TradeList() feeList = core.TradeList() skippedRows = 0 for row in range(dataFrame.shape[0]): if 'Gebühr' in dataFrame[headernames[1]][row]: # fees date = convertDate(dataFrame[headernames[0]][row]) if headernames[7]: externId = str(dataFrame[headernames[7]][row]) fees = dataFrame[headernames[8]][row] coin = str(dataFrame[headernames[2]][row]) fee = createFee(date=date, amountStr=fees, maincoin=coin, exchange='bitcoinde', externId=externId) fee.generateID() feeList.addTrade(fee) else: tempTrade_sub = core.Trade() # sub tempTrade_main = core.Trade() # main # get id if headernames[7]: tempTrade_sub.externId = str(dataFrame[headernames[7]][row]) tempTrade_main.externId = str(dataFrame[headernames[7]][row]) # get date tempTrade_sub.date = convertDate(dataFrame[headernames[0]][row]) tempTrade_main.date = convertDate(dataFrame[headernames[0]][row]) # set type tempTrade_sub.tradeType = 'trade' tempTrade_main.tradeType = 'trade' # check content type isBuy = False if re.match(r'^.*?(Kauf).*?$', dataFrame[headernames[1]][row], re.IGNORECASE): isBuy = True elif re.match(r'^.*?(Verkauf).*?$', dataFrame[headernames[1]][row], re.IGNORECASE): isBuy = False else: # invalid type skippedRows += 1 continue # get coin if headernames[12]: coinPairIndex = 12 else: coinPairIndex = 2 tempTrade_sub.coin = re.match(r'^(.*) / .*$', dataFrame[headernames[coinPairIndex]][row]).group(1).upper() tempTrade_main.coin = re.match(r'^.* / (.*)$', dataFrame[headernames[coinPairIndex]][row]).group(1).upper() # swap Coin Name swapCoinName(tempTrade_sub) swapCoinName(tempTrade_main) # get amount wo fees if isinstance(dataFrame[headernames[4]][row], numbers.Number): # if amount is number amount_wo_fees = abs(dataFrame[headernames[4]][row]) else: # now number so use regex to extract the number amount_wo_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[4]][row]).group(1))) # get price wo fees if isinstance(dataFrame[headernames[3]][row], numbers.Number): # if price is number price_wo_fees = abs(dataFrame[headernames[3]][row]) else: # no number so use regex price_wo_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[3]][row]).group(1))) # get amount w fees if isinstance(dataFrame[headernames[6]][row], numbers.Number): # if amount is number amount_w_fees = abs(dataFrame[headernames[6]][row]) else: # now number so use regex to extract the number amount_w_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[6]][row]).group(1))) # get price w fees if headernames[9]: # if fidor fee included mainFeeIndex = 9 else: mainFeeIndex = 5 if isinstance(dataFrame[headernames[mainFeeIndex]][row], numbers.Number): # if price is number price_w_fees = abs(dataFrame[headernames[mainFeeIndex]][row]) else: # no number so use regex price_w_fees = abs(float(pat.NUMBER_REGEX.match(dataFrame[headernames[mainFeeIndex]][row]).group(1))) # get sign if isBuy == True: tempTrade_sub.amount = amount_wo_fees tempTrade_main.amount = -price_w_fees fees_sub = amount_wo_fees - amount_w_fees fees_main = price_wo_fees - price_w_fees else: tempTrade_sub.amount = amount_w_fees tempTrade_main.amount = -price_wo_fees fees_sub = -(amount_wo_fees - amount_w_fees) fees_main = -(price_wo_fees - price_w_fees) # set exchange tempTrade_main.exchange = 'bitcoinde' tempTrade_sub.exchange = 'bitcoinde' # set id if not tempTrade_sub.tradeID: tempTrade_sub.generateID() if not tempTrade_main.tradeID: tempTrade_main.generateID() # add trades to tradeList if not tradeList.addTradePair(tempTrade_sub, tempTrade_main): skippedRows += 1 # fees try: # get fee feeMain = createFee(date=tempTrade_main.date, amountStr=fees_main, maincoin=tempTrade_main.coin, exchange='bitcoinde', externId=tempTrade_main.externId) feeSub = createFee(date=tempTrade_sub.date, amountStr=fees_sub, maincoin=tempTrade_sub.coin, exchange='bitcoinde', externId=tempTrade_sub.externId) feeMain.generateID() feeSub.generateID() feeList.addTrade(feeMain) feeList.addTrade(feeSub) except Exception as ex: # do not skip line if error, just ignore fee localLogger.warning('error in Converter: ' + str(ex)) return tradeList, feeList, skippedRows
def __init__(self): super(WebApiInterface, self).__init__() # self.histPrices = {} self.tradeBuffer = core.TradeList()
# %% import data mypath = r'D:\workspace\python\PyCryptoPortfolio\importdata' # mypath = r'D:\workspace\spyder\PyCryptoPortfolio\data\kaj' files, content = importer.loadTrades(mypath) settings.mySettings.setPath('D:\workspace\python\PyCryptoPortfolio\Data') headings = [frame.columns.tolist() for frame in content] # fileheadings = [] # for i in range(len(files)): # fileheadings.append([files[i],headings[i]]) # %% convert data # tradeList, matches = importer.convertTrades(models.IMPORT_MODEL_LIST, content) tradeList = core.TradeList() tradeList.fromCsv(r'D:\workspace\python\PyCryptoPortfolio\Data\Trades.csv') # tradeList.updateValues() tradeFrame = tradeList.toDataFrameComplete() coinList = core.CoinList() coinList.addTrades(tradeList) coinList.updateCurrentValues() coinFrame = coinList.toDataFrame() coinDict = coinFrame.to_dict('index') coinIndex = 3 # tradeMatcher = coinList.coins[coinIndex].tradeMatcher # matchedFrame = coinList.coins[coinIndex].tradeMatcher.toDataFrame('EUR')