def getSalesGrowth(fundamental_data, instrument, fundamental_period):
# return Ratio dataframe
import numpy as np
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio = Ratio.fundamental_revenue_Y
else:
ratio = Ratio.fundamental_revenue_Q
sales = fundamental_data.download(instrument, ratio)
if sales is None or len(sales) == 0:
logger.error(
'error downloading %s for %s => return sales growth none' %
(ratio, instrument.symbol))
return None
currentSales = sales[Ratio.ratio]
salesGrowth = sales.copy()
salesGrowth[Ratio.ratio] = np.divide(
currentSales - currentSales.shift(periodsToShift),
currentSales.shift(periodsToShift)) * 100
except Exception as e:
logger.error('Error getting sales growth for %s %s => return None' %
(instrument.symbol, str(e)))
return None
return salesGrowth
def getTurnDiff(fundamental_data, instrument, fundamental_period, fromDate,
toDate):
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_revenue_Y
ratio_2 = Ratio.fundamental_assets_Y
periodsToShift = 1
else:
ratio_1 = Ratio.fundamental_revenue_Q
ratio_2 = Ratio.fundamental_assets_Q
periodsToShift = 4
fromDate = fromDate - datetime.timedelta(days=2 * 365)
sales = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
assets = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
turnover = sales / assets
turnoverDiff = turnover - turnover.shift(periodsToShift)
return turnoverDiff
except Exception as e:
logger.error('Error getting turnoverDiff for %s %s => return None' %
(instrument.symbol, str(e)))
return None
def getLeverage(fundamental_data, instrument, fundamental_period, fromDate,
toDate):
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_debt_Y
ratio_2 = Ratio.fundamental_assets_Y
periodsToShift = 1
else:
ratio_1 = Ratio.fundamental_debt_Q
ratio_2 = Ratio.fundamental_assets_Q
periodsToShift = 4
debt = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
assets = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
leverage = debt / assets
return leverage
except Exception as e:
logger.error('Error getting gross margin for %s %s => return None' %
(instrument.symbol, str(e)))
return None
def getFCFPerShare(fundamental_data, instrument, fundamental_period, fromDate,
toDate):
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_free_cashflow_Y
ratio_2 = Ratio.fundamental_shares_Y
periodsToShift = 1
else:
ratio_1 = Ratio.fundamental_free_cashflow_Q
ratio_2 = Ratio.fundamental_shares_Q
periodsToShift = 4
fcff = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
shares = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
output = fcff / shares
return output
except Exception as e:
logger.error(
'Error getting free cashflow per share for %s %s => return None' %
(instrument, str(e)))
return None
def importExcel(self, file, instrument):
reader = pd.ExcelFile(file)
sheetNames = reader.sheet_names
output = True
for sheet in sheetNames:
ratio = self.getRatioFromSheet(sheet)
logger.debug('%s_%s reading %s' %
(instrument.symbol, instrument.currency, ratio))
dataDownloaded = reader.parse(sheet)
df_to_save = self.formatFundamental(dataDownloaded)
# save it to database
if df_to_save is not None:
logger.debug('%s_%s saving to database %s' %
(instrument.symbol, instrument.currency, ratio))
try:
self.ratioDataService.saveRatioDataFrame(
df_to_save, instrument, ratio)
except Exception as e:
logger.error('Error saving Ratio %s of %s_%s :%s' %
(ratio, instrument.symbol,
instrument.currency, str(e)))
output = False
else:
logger.error('Error getting Ratio %s of %s_%s' %
(ratio, instrument.symbol, instrument.currency))
output = False
return output
def getCurrentRatio(fundamental_data, instrument, fundamental_period, fromDate,
toDate):
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_current_assets_Y
else:
ratio_1 = Ratio.fundamental_current_assets_Q
if fundamental_period == FundamentalPeriod.yearly:
ratio_2 = Ratio.fundamental_current_liabilities_Y
else:
ratio_2 = Ratio.fundamental_current_liabilities_Q
fromDate = fromDate - datetime.timedelta(days=2 * 365)
currentAssetsassets = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
currentLiabilities = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
currentRatio = currentAssetsassets / currentLiabilities
return currentRatio
except Exception as e:
logger.error('Error getting current Ratio for %s %s => return None' %
(instrument.symbol, str(e)))
return None
def getFCFA(fundamental_data, instrument, fundamental_period, fromDate,
toDate):
# Sum(eight - yearFCF) / total assets(t)
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_free_cashflow_Y
ratio_2 = Ratio.fundamental_assets_Y
periodsToShift = 1
else:
ratio_1 = Ratio.fundamental_free_cashflow_Q
ratio_2 = Ratio.fundamental_assets_Q
periodsToShift = 4
fromDate = fromDate - datetime.timedelta(days=10 * 365)
fcff = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
assets = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
sum_fcff = fcff.rolling(8 * periodsToShift).sum()
fcfa = sum_fcff / assets
return fcfa
except Exception as e:
logger.error('Error getting fcfa for %s %s => return None' %
(instrument.symbol, str(e)))
return None
def saveTickDataFrame(self, dataframe, instrument, period):
if period == Period.tick:
dao = TickDao(self.user_settings)
return dao.save(dataframe, instrument=instrument)
else:
logger.error("Trying to save not tick period as tick ")
return False
def getFCFTADiff(fundamental_data, instrument, fundamental_period, fromDate,
toDate):
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_free_cashflow_Y
ratio_2 = Ratio.fundamental_assets_Y
periodsToShift = 1
else:
ratio_1 = Ratio.fundamental_free_cashflow_Q
ratio_2 = Ratio.fundamental_assets_Q
periodsToShift = 4
fromDate = fromDate - datetime.timedelta(days=2 * 365)
fcff = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
assets = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
fcfta = fcff / assets
fcftaDiff = fcfta - fcfta.shift(periodsToShift)
return fcftaDiff
except Exception as e:
logger.error('Error getting fcftaDiff for %s %s => return None' %
(instrument.symbol, str(e)))
return None
def cleanOutliersData(self, instrument, period, df):
if df is None:
return None
# TODO improve to get average of surrounded instead of delete
output = df.copy()
if period == Period.tick:
wrongBid = mad_outlier(df[Tick.bid].values.reshape(-1, 1))
output = df.loc[~wrongBid]
wrongAsk = mad_outlier(df[Tick.ask].values.reshape(-1, 1))
output = df.loc[~wrongAsk]
else:
wrongClose = mad_outlier(df[Bar.close].values.reshape(-1, 1))
output = df.loc[~wrongClose]
wrongOpen = mad_outlier(df[Bar.open].values.reshape(-1, 1))
output = df.loc[~wrongOpen]
if len(output) == 0:
logger.error("All data was wrong for %s_%s when cleaning" % (instrument.symbol, instrument.currency))
else:
errorsFind = len(df) - len(output)
if errorsFind is not 0:
logger.debug('Find %i outliers to remove in %s_%s when cleaning' % (errorsFind,
instrument.symbol,
instrument.currency))
return output
def importAllFiles(self):
filesInInput = self.market_data_provider.filesInDirectory
for file in filesInInput:
instrument, fromDate, toDate, period, number_periods = self.market_data_provider.getDataFile_instrument_from_to_period_numberPeriods(
file)
dataDB = self.historical_market_data_service.getHistoricalData(
instrument=instrument,
number_of_periods=number_periods,
fromDate=fromDate,
toDate=toDate,
period=period)
if dataDB is not None:
dataDB = dataDB[fromDate:toDate]
if dataDB is not None and (not fromDate in dataDB.index
or toDate not in dataDB.index):
dataDB = None
if dataDB is None:
# if dataDB is None:
dataframe = self.market_data_provider.download(
instrument, period, number_periods, fromDate, toDate)
if dataframe is not None:
isSaved = self.historical_market_data_service.saveBarDataFrame(
dataframe, instrument, period, number_periods)
if isSaved:
self.market_data_provider.markFileAsProcessed(file)
else:
# mark as proccessed if some errroes BID candlesstick for ex
logger.error('Cant read %s => mark as procesed' % file)
self.market_data_provider.markFileAsProcessed(file)
else:
# marked to avoid reading it next time
self.market_data_provider.markFileAsProcessed(file)
def __downloadHistoricalPrices__(self, wrongInstrumentsSymbolLists,
outputDict, instrument, fromDate, toDate):
# self.lock.acquire()
# outputDict['wrong'] = wrongInstrumentsSymbolLists
historicalData = self.historicalMarketDataService.getHistoricalData(
instrument,
period=Period.day,
number_of_periods=1,
fromDate=fromDate,
toDate=toDate,
bar_type=BarType.time_bar,
force_download=self.force_download,
endOfDayData=True
# force to download it with index time to zero
)
if historicalData is None:
logger.error(
'%s_%s has no historical Data to vectorized => removing column'
% (instrument.symbol, instrument.currency))
wrongInstrumentsSymbolLists.append(instrument.symbol)
if 'wrong' in outputDict.keys():
outputDict['wrong'] += wrongInstrumentsSymbolLists
else:
outputDict['wrong'] = wrongInstrumentsSymbolLists
# self.lock.release()
return
self.__formatPrices__(outputDict, instrument, historicalData)
def formatFundamental(self, input):
import pandas as pd
input_df = input.copy().reset_index()
columnsRatio = list(input_df.columns)
columnsRatio.remove(self.columns_ratio_dict[Ratio.time])
if len(columnsRatio) > 1:
logger.error('Error more 1 column in morningstar: %s' %
str(columnsRatio))
return None
columnName = columnsRatio[0]
output = pd.DataFrame(input,
columns=Ratio.columns,
index=input_df.index)
for column in output.columns:
if column == Ratio.time:
timeProcessed = self.formatTime(
input_df[self.columns_ratio_dict[column]])
output[column] = timeProcessed
else:
output[column] = input_df[columnName]
output.set_index(Ratio.index, inplace=True)
return output
def getAssetQualityIndex(fundamental_data, instrument, fundamental_period,
fromDate, toDate):
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio_1 = Ratio.fundamental_non_current_assets_Y
ratio_2 = Ratio.fundamental_assets_Y
periodsToShift = 1
else:
ratio_1 = Ratio.fundamental_non_current_assets_Q
ratio_2 = Ratio.fundamental_assets_Q
periodsToShift = 4
fromDate = fromDate - datetime.timedelta(days=2 * 365)
non_current_assets = fundamental_data.download(instrument,
ratio_1,
fromDate=fromDate)
assets = fundamental_data.download(instrument,
ratio_2,
fromDate=fromDate)
aqi_t = non_current_assets / assets # measure attempts cost deferral as intangible assets
aqi_t_1 = non_current_assets.shift(periodsToShift) / assets.shift(
periodsToShift)
return aqi_t / aqi_t_1
except Exception as e:
logger.error('Error getting gross margin for %s %s => return None' %
(instrument.symbol, str(e)))
return None
def __downloadAllRatiosBatch__(self, outputDict, instrument, fromDate,
toDate, ratioList,
wrongInstrumentsSymbolLists):
import datetime
ratio = ratioList[0]
if ratio.endswith('Y'):
fromDateRatio = fromDate - datetime.timedelta(days=2 * 365)
else:
fromDateRatio = fromDate - datetime.timedelta(days=1 * 365)
ratioDataBatch = self.ratioDataService.getRatioDataBatch(
instrument,
ratioList,
fromDateRatio,
toDate,
force_download=self.force_download)
if ratioDataBatch is None:
logger.error(
'%s_%s has no ratios %s Data to vectorized => removing column'
% (instrument.symbol, instrument.currency, ratioList))
wrongInstrumentsSymbolLists.append(instrument.symbol)
if (DataDictKeys.close in outputDict.keys()):
indexSelected = outputDict[DataDictKeys.close].index
nanDF = self.__createDataframe__([instrument], index=indexSelected)
else:
nanDF = self.__createDataframe__([instrument],
fromDate=fromDate,
toDate=toDate)
for ratio in ratioList:
if ratio not in outputDict.keys():
outputDict[ratio] = nanDF.copy()
if ratioDataBatch is not None:
ratioIndexBar = nanDF.copy().index.searchsorted(
ratioDataBatch.index)
else:
if ratioDataBatch is not None:
ratioIndexBar = outputDict[ratio].index.searchsorted(
ratioDataBatch.index)
if ratioDataBatch is not None:
ratioIndexBar[ratioIndexBar > 0] = ratioIndexBar[
ratioIndexBar > 0] - 1 # Put first value
# SettingWithCopyWarning:
# A value is trying to be set on a copy of a slice from a DataFrame
numpyValues = ratioDataBatch[ratio].values.copy()
if instrument.symbol not in outputDict[ratio].columns:
outputDict[ratio][instrument.symbol] = None
outputDict[ratio][
instrument.symbol][ratioIndexBar] = numpyValues
# outputDict[ratio].fillna(method='ffill',inplace=True)
if 'wrong' in outputDict.keys():
outputDict['wrong'] += wrongInstrumentsSymbolLists
else:
outputDict['wrong'] = wrongInstrumentsSymbolLists
outputDict['wrong'] = list(set(outputDict['wrong']))
def _pricing_iter():
sid = 0
with maybe_show_progress(
symbols,
show_progress,
label='Downloading Tradea Database pricing data ') as it, \
requests.Session() as session:
for symbol in it:
logger.debug('zipline bundle downloading %s' % symbol)
try:
instrument = Instrument(
symbol=symbol, asset_type=AssetType.us_equity)
df = self.historical_market_data_service.getHistoricalData(
instrument,
period=Period.day,
number_of_periods=1,
fromDate=start,
toDate=end,
bar_type=BarType.time_bar,
force_download=False,
cleanOutliers=False)
except Exception as e:
logger.error(
'Error downloading bundle zipline %s : %s' %
(symbol, str(e)))
print('Error downloading bundle zipline %s : %s' %
(symbol, str(e)))
df = None
continue
# the start date is the date of the first trade and
# the end date is the date of the last trade
indexSet = df.index.copy()
indexSet = (indexSet + pd.DateOffset(hours=3)
) - pd.DateOffset(days=1)
df.index = indexSet
start_date = df.index[0]
end_date = df.index[-1]
# The auto_close date is the day after the last trade.
ac_date = end_date + pd.Timedelta(days=1)
metadata.iloc[
sid] = start_date, end_date, ac_date, symbol
df.rename(
columns={
Bar.open: 'open',
Bar.high: 'high',
Bar.low: 'low',
Bar.close: 'close',
Bar.volume: 'volume',
},
inplace=True,
)
yield sid, df
sid += 1
def getRatioDataBatch(self, instrument, ratio_list, fromDate, toDate=None, force_download=False):
fromDate = convert_date(fromDate)
toDate = convert_date(toDate)
df_temp = None
df = self.__getRatioDataDBBatch__(instrument, ratio_list, fromDate, toDate)
fromDateDictTemp = None
if df is not None:
if (df.index[0] - fromDate).days > self.tresholdDaysUpdate and force_download:
if fromDateDictTemp is not None and fromDateDictTemp <= fromDate:
logger.debug('max downloaded %s ,cant download more to %s' % (ratio_list, instrument))
else:
# messageError = 'some data missing begining ratio %s in %s fromDateDictTemp=%s fromDate=%s' % (
# ratio, instrument.symbol, str(fromDateDictTemp), str(fromDate))
# logger.error(messageError)
df_temp = df.copy()
df = None
elif (toDate - df.index[-1]).days > self.tresholdDaysUpdate and force_download:
logger.debug('some data missing the end ')
df_temp = df.copy()
df = None
elif (len(df.columns) < len(ratio_list) and force_download):
logger.debug('some ratio missing in database')
df_temp = df.copy()
df = None
if df is None and force_download:
logger.debug('data %s %s_%s not found in database trying to download and save it!' % (
ratio_list, instrument.symbol, instrument.currency))
df = self.getRatioDataProviderBatch(instrument, ratio_list, fromDate, toDate)
# if df_to_save is not None:
# self.saveRatioDataFrameBatch(df_to_save, instrument, ratio_list)
# df = self.__getRatioDataDBBatch__(instrument, ratio_list, fromDate, toDate)
# else:
# logger.error('Not pssible to download %s for %s' % (ratio_list, instrument))
# df = None
if df is None:
if df_temp is not None:
logger.debug('ratioData: Cant download more , restoring DDBB files ')
df = df_temp
else:
logger.error(
"Error getting ratio data for %s %s_%s" % (ratio_list, instrument.symbol, instrument.currency))
return None
if fromDateDictTemp is not None:
fromDateSave = min(fromDate, fromDateDictTemp)
else:
fromDateSave = fromDate
# if fromDateSave != fromDateDictTemp:
# self.ratio_instrument_from_dict_date[
# self.__getKeyDict__(ratio, instrument)] = fromDateSave # add to dict and save
# self.__saveDictFromDate__() # save from date cache to file
return df
def error_handler(self, msg):
"""Handles the capturing of error messages"""
if msg.errorCode in self.notErrorsList:
return
logger.error("IB error received %s :code[%i] %s" %
(msg, msg.errorCode, msg.errorMsg))
print("Server Error: %s" % msg)
self.receivedAllHistorical = True
def downloadAllFinancials(self, instrument):
try:
symbol = self.getMorningStarTicker(instrument)
functionTemp = self.cacher.cache(
self.financial_downloader.download, ignore=['self'])
return functionTemp(symbol)
except Exception as e:
logger.error('Error downloading morningstar data of %s:%s' %
(instrument, str(e)))
return None
def __getRatioDataProvider__(self, instrument, ratio, fromDate, toDate=None):
if ratio.startswith('fundamental'):
ratio_data = self.routerProvider.getFundamentalDataProvider(ratio, instrument)
elif ratio.startswith('quant'):
ratio_data = QuantDataImpl
else:
logger.error('uknown type of ratio: %s' % ratio)
return None
ratio_fundamental_data = ratio_data(self.user_settings)
return ratio_fundamental_data.download(instrument, ratio, fromDate, toDate)
def getHistoricalData(self, instrument, period, number_of_periods, fromDate, toDate=None,
bar_type=BarType.time_bar, force_download=False, endOfDayData=False, cleanOutliers=True):
fromDate = convert_date(fromDate)
toDate = convert_date(toDate)
df_temp = None
df = self.__getHistoricalDataDB__(instrument, period, number_of_periods, fromDate, toDate, bar_type)
fromDateDictTemp = None
# if self.__getKeyDict__(instrument) in self.instrument_from_dict_date:
# fromDateDictTemp = self.instrument_from_dict_date[self.__getKeyDict__(instrument)]
if df is not None:
if (df.index[0] - fromDate).days > self.tresholdDaysUpdate and force_download:
if fromDateDictTemp is not None and fromDateDictTemp <= fromDate:
logger.debug(
'max downloaded ,cant download more to %s getting %d rows' % (instrument.symbol, len(df)))
else:
# messageError = 'some data missing begining %s fromDateDictTemp=%s fromDate=%s' % (
# instrument.symbol, str(fromDateDictTemp), str(fromDate))
# logger.error(messageError)
df_temp = df.copy()
df = None
elif (toDate - df.index[-1]).days > self.tresholdDaysUpdate and force_download:
logger.error('some data missing the end %s' % instrument.symbol)
df_temp = df.copy()
df = None
if df is None and force_download:
logger.debug('data not found in database trying to download and save it in %s!' % instrument.symbol)
df_to_save = self.getHistoricalDataProvider(instrument, period, number_of_periods, fromDate, toDate,
bar_type)
if df_to_save is not None:
self.saveBarDataFrame(df_to_save, instrument, period, number_of_periods, bar_type)
df = self.__getHistoricalDataDB__(instrument, period, number_of_periods, fromDate, toDate, bar_type)
if df is None:
if df_temp is not None:
logger.debug('marketData: Cant download more , restoring DDBB files ')
df = df_temp
else:
logger.error("Error getting historical data for %s_%s" % (instrument.symbol, instrument.currency))
if fromDateDictTemp is not None:
fromDateSave = min(fromDate, fromDateDictTemp)
else:
fromDateSave = fromDate
# if fromDateSave != fromDateDictTemp:
# self.instrument_from_dict_date[self.__getKeyDict__(instrument)] = fromDateSave # add to dict and save
# self.__saveDictFromDate__() # save from date cache to file
if cleanOutliers is True:
df = self.cleanOutliersData(instrument, period, df) # None checked inside
if endOfDayData is True:
df = self.__formatDatesEndOfDay__(df, fromDate, toDate)
return df
def delete(self, itemName):
# if instrument is None:
# # delete a complete collection
# self.store.delete_collection(self.collectionName)
# else:
# delete a item
try:
self.collection.delete_item(itemName)
self.cacheDict = {} # Cache delete everything....
except Exception as e:
logger.error("Error deleting collection %s:%s" % (itemName, e))
def getCapitalization(fundamental_data,
instrument,
fundamental_period,
fromDate=None,
toDate=None):
import datetime
from tradeasystems_connector.service.historical_market_data_service import HistoricalMarketDataService
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio = Ratio.fundamental_shares_Y
else:
ratio = Ratio.fundamental_shares_Q
shares = fundamental_data.download(instrument, ratio, fromDate, toDate)
if shares is None or len(shares) == 0:
logger.error(
'error downloading %s for %s => return capitalization none' %
(ratio, instrument.symbol))
return None
marketDataServices = HistoricalMarketDataService(
fundamental_data.user_settings)
if fromDate is None:
fromDate = shares.index[0].replace(
tzinfo=None) - datetime.timedelta(days=3)
else:
fromDate = fromDate - datetime.timedelta(days=3)
if toDate is None:
toDate = shares.index[-1].replace(
tzinfo=None) + datetime.timedelta(days=3)
else:
toDate = toDate + datetime.timedelta(days=3)
barPrices = marketDataServices.getHistoricalData(instrument,
period=Period.day,
number_of_periods=1,
force_download=True,
fromDate=fromDate,
toDate=toDate)
capitalization = shares.copy()
closesIndexBar = barPrices.index.searchsorted(capitalization.index) - 1
closesIndexBar = closesIndexBar[closesIndexBar < barPrices.shape[0]]
shares[Bar.close] = barPrices[Bar.close][closesIndexBar].values
capitalization[Ratio.ratio] = shares[Bar.close] * shares[Bar.close]
except Exception as e:
logger.error('Error getting capitalization for %s %s => return None' %
(instrument.symbol, str(e)))
capitalization = None
return capitalization
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
import pandas as pd
self.filesInDirectory = glob.glob(self.inputPath + os.sep + "*.csv")
logger.debug(
"Downloading from dukascopy file %s_%s %i_%s from %s to %s" %
(instrument.symbol, instrument.currency, number_of_periods, period,
fromDate, toDate))
toDateFile = self.getToDateFromFile(toDate, period)
if period == Period.tick:
# '%s%s_Ticks_%s-%s.csv'
file = self.tick_filename_format % (
instrument.symbol, instrument.currency,
fromDate.strftime(self.filenameDateFormat),
toDateFile.strftime(
self.filenameDateFormat)) # '%s%s_Candlestick_%i_%s_%s-%s
else:
# '%s%s_Candlestick_%i_%s_%a_%s-%s.csv'
file = self.bar_filename_format % (
instrument.symbol, instrument.currency, number_of_periods,
self.period_dict[period], 'ASK',
fromDate.strftime(self.filenameDateFormat),
toDateFile.strftime(
self.filenameDateFormat)) # '%s%s_Candlestick_%i_%s_%s-%s
for fileIterate in self.filesInDirectory:
fileIterateName = fileIterate.split(os.sep)[-1]
if fileIterateName == file:
logger.debug("File %s found , processing" % file)
try:
# data_downloaded = pd.DataFrame.from_csv(fileIterate, dayfirst=True) # date_format=self.dateFormat) date_parser=lambda x: datetime.strptime(x, '%d.%m.%Y %H:%M:%S.%f'))
data_downloaded = pd.read_csv(fileIterate)
except Exception as e:
logger.error('Cant read file %s some errors there' % file)
return None
outputComplete = self.formatHistorical(data_downloaded,
period=period)
# if fromDate is not None and toDate is not None:
# outputComplete = outputComplete[fromDate: toDate]
# elif fromDate is not None:
# outputComplete = outputComplete[fromDate:]
# elif toDate is not None:
# outputComplete = outputComplete[:toDate]
# self.markFileAsProcessed(file)
return outputComplete
def load(self, instrument, columnList=None, startTime=None, endTime=None):
collectionName = self.collectionName
super().setCollectionVariables(collectionName)
data = super().load(self.getItemName(instrument), startTime, endTime)
if data is not None and columnList is not None:
try:
data = data[columnList]
return data
except Exception as e:
logger.error('Error loading vector of %s columns %s : %s' %
(instrument, columnList, str(e)))
data = None
return data
def setTimeCorrect(self, outputComplete, instrument):
import pandas as pd
# must be in UTC directly
try:
if outputComplete is not None and outputComplete.index[0].hour == 0:
hour, minute = getCloseTimeHourMinInUTC(instrument)
outputComplete.index = outputComplete.index + pd.DateOffset(
hours=hour, minute=minute)
# outputComplete.index=outputComplete.index.tz_convert( timezone_setting)
except Exception as e:
logger.error('Error setting time of fundamental data of %s:%s' %
(instrument.symbol, str(e)))
outputComplete = None
return outputComplete
def getBookValueLiabilities(fundamental_data, instrument, fundamental_period):
# TODO review if something better cand do
try:
if fundamental_period == FundamentalPeriod.yearly:
ratio = Ratio.fundamental_liabilities_Y
else:
ratio = Ratio.fundamental_liabilities_Q
liabilities = fundamental_data.download(instrument, ratio)
except Exception as e:
logger.error('Error getting bv liabilities for %s %s => return None' %
(instrument.symbol, str(e)))
return None
return liabilities
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
self.lock.acquire()
logger.debug("Downloading %s %s from %s to %s" %
(instrument.symbol, period, str(fromDate), str(toDate)))
if instrument.asset_type == AssetType.forex:
self.whatToShow = 'MIDPOINT'
else:
self.whatToShow = 'TRADES'
contract = self.__makeContract__(instrument)
period = self.period_dict[period]
barSizeSetting = '%i %s' % (number_of_periods, period)
if number_of_periods > 1 or period == Period.second:
barSizeSetting = barSizeSetting + 's'
outputAppended = None
durationStr_list, toDateString_list = self.getPeriodsRequests(
period, fromDate, toDate)
for i in range(len(durationStr_list)):
durationStr = durationStr_list[i]
toDateString = toDateString_list[i]
output_request = self.__makeRequestSingle__(
contract, durationStr, toDateString, barSizeSetting)
if outputAppended is None:
outputAppended = output_request
else:
outputAppended = outputAppended.append(output_request)
if outputAppended is None:
logger.error("Couldnt download anything=> release return None")
else:
# clean unique index and sort
outputAppended = outputAppended[~outputAppended.index.duplicated(
keep='first')]
outputAppended.sort_index(inplace=True)
if fromDate is not None and toDate is not None:
outputAppended = outputAppended[fromDate:toDate]
elif fromDate is not None:
outputAppended = outputAppended[fromDate:]
elif toDate is not None:
outputAppended = outputAppended[:toDate]
self.lock.release()
return outputAppended
def load(self, itemName, startTime=None, endTime=None):
# Cache
key = self.__getMD5_key__(itemName, startTime, endTime)
if key in self.cacheDict.keys():
return self.cacheDict[key]
try:
item = self.collection.item(itemName)
except Exception as e:
logger.error("symbol %s not found on database to load => return None %s" % (itemName, e))
return None
df = item.to_pandas()
if startTime is not None:
startTime = np.datetime64(startTime)
if endTime is not None:
endTime = np.datetime64(endTime)
try:
output = df
mask = None
if startTime is not None and endTime is not None:
mask = (df.index >= startTime) & (df.index <= endTime)
# output = df[startTime:endTime]
elif startTime is not None:
mask = (df.index >= startTime)
# output = df[startTime:]
elif endTime is not None:
mask = (df.index <= endTime)
if mask is not None:
output = df.loc[mask]
if len(output) == 0:
output = None
else:
output = self.cleanDataframeDatetime(output)
logger.debug("Successfully load from %s: %s a dataframe of %d rows %d columns" % (
self.collectionName, itemName, output.shape[0], output.shape[1]))
self.cacheDict[key] = output # Cache
except Exception as e:
logger.error('Error loading %s from %s return None :%s' % (itemName, item._path, str(e)))
output = None
return output
def download(self):
filesInDirectory = self.getFilesInPath()
output = True
for file in filesInDirectory:
if 'empty' in file:
logger.debug('file: %s skipped=> empty is a FW' % file)
continue
instrument = self.getInstrument(file)
logger.debug('Importing ratios of %s_%s' %
(instrument.symbol, instrument.currency))
isImported = self.importExcel(file, instrument)
if isImported is False:
logger.error('Error ratios of %s_%s :%s' %
(instrument.symbol, instrument.currency, file))
output = output and isImported
return output