def dictToDataframe(outputDict, symbol, columnList):
dataframeSave = None
for key in columnList:
if key in outputDict.keys():
if symbol in list(outputDict[key].columns):
seriesFromDict = outputDict[key][symbol]
seriesFromDict = seriesFromDict[~seriesFromDict.index.
duplicated(keep='last')]
else:
# data not downloaded
logger.debug('%s not found in %s => none' % (key, symbol))
seriesFromDict = None
if seriesFromDict is not None:
if dataframeSave is None:
dataframeSave = pd.DataFrame(seriesFromDict.values,
index=seriesFromDict.index,
columns=[key])
else:
dataframeSave[
key] = seriesFromDict # pd.Series(seriesFromDict.values,index = dataframeSave.index)
if dataframeSave is not None:
dataframeSave.fillna(method='ffill', inplace=True)
# remove duplicated
dataframeSave = dataframeSave[~dataframeSave.index.duplicated(
keep='last')]
return dataframeSave
def cleanOutliersData(self, instrument, period, df):
if df is None:
return None
# TODO improve to get average of surrounded instead of delete
output = df.copy()
if period == Period.tick:
wrongBid = mad_outlier(df[Tick.bid].values.reshape(-1, 1))
output = df.loc[~wrongBid]
wrongAsk = mad_outlier(df[Tick.ask].values.reshape(-1, 1))
output = df.loc[~wrongAsk]
else:
wrongClose = mad_outlier(df[Bar.close].values.reshape(-1, 1))
output = df.loc[~wrongClose]
wrongOpen = mad_outlier(df[Bar.open].values.reshape(-1, 1))
output = df.loc[~wrongOpen]
if len(output) == 0:
logger.error("All data was wrong for %s_%s when cleaning" % (instrument.symbol, instrument.currency))
else:
errorsFind = len(df) - len(output)
if errorsFind is not 0:
logger.debug('Find %i outliers to remove in %s_%s when cleaning' % (errorsFind,
instrument.symbol,
instrument.currency))
return output
def __cleanData__(self, outputDict, assetType, persistTempFile=None):
logger.debug('all instruments processed => cleaning ')
if 'wrong' in outputDict.keys():
wrongInstrumentsSymbolLists = outputDict['wrong'].copy()
if 'wrong' in outputDict:
del outputDict['wrong']
wrongInstrumentsSymbolLists = self.vectorizedDataService.__formatWrongInstrumentList__(
wrongInstrumentsSymbolLists, outputDict)
outputDictFinal = self.vectorizedDataService.__cleanOutputDict__(
outputDict, wrongInstrumentsSymbolLists)
else:
outputDictFinal = outputDict
if assetType != AssetType.forex and assetType != AssetType.crypto:
outputDictFinal = self.vectorizedDataService.__cleanBankHolidays__(
outputDictFinal)
df = outputDictFinal[DataDictKeys.close]
logger.debug(
'all dictOfMatrix cleaned => finished %d matrixes of %d columns' %
(len(outputDictFinal), df.shape[1]))
if persistTempFile is not None:
if persistTempFile is False:
persistTempFile = None
if persistTempFile is True:
persistTempFile = 'default.xlsx'
if ~persistTempFile.endswith('.xlsx'):
persistTempFile += '.xlsx'
self.vectorizedDataService.__createTempExcelFile__(
outputDictFinal, filenameWithExtension=persistTempFile)
return outputDictFinal
def importExcel(self, file, instrument):
reader = pd.ExcelFile(file)
sheetNames = reader.sheet_names
output = True
for sheet in sheetNames:
ratio = self.getRatioFromSheet(sheet)
logger.debug('%s_%s reading %s' %
(instrument.symbol, instrument.currency, ratio))
dataDownloaded = reader.parse(sheet)
df_to_save = self.formatFundamental(dataDownloaded)
# save it to database
if df_to_save is not None:
logger.debug('%s_%s saving to database %s' %
(instrument.symbol, instrument.currency, ratio))
try:
self.ratioDataService.saveRatioDataFrame(
df_to_save, instrument, ratio)
except Exception as e:
logger.error('Error saving Ratio %s of %s_%s :%s' %
(ratio, instrument.symbol,
instrument.currency, str(e)))
output = False
else:
logger.error('Error getting Ratio %s of %s_%s' %
(ratio, instrument.symbol, instrument.currency))
output = False
return output
def download(self, instrument, period, number_of_periods, fromDate, toDate=None):
import datetime
logger.debug("Downloading %s" % instrument.symbol)
try:
if period != Period.day:
# '1min', '5min', '15min', '30min', '60min'
if number_of_periods in [1, 5, 15, 30, 60]:
periodAlpha = '%i%s' % (number_of_periods, self.period_dict[period])
data_downloaded, meta_data = self.timeseries.get_intraday(instrument.symbol, interval=periodAlpha)
else:
logger.error(
"AlphaVantage can only download intradaily! '1min', '5min', '15min', '30min', '60min' not %s" % period)
return None
else:
# dateFormat = "%Y-%m-%d"
if toDate is None:
toDate = datetime.datetime.today()
# download dataframe
try:
data_downloaded, meta_data = self.timeseries.get_daily_adjusted(instrument.symbol)
# data_downloaded = pdr.get_data_yahoo(instrument.symbol, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat))
except Exception as e:
logger.error("Cand download from alphavantage %s => return None %s" % (instrument.symbol, e))
return None
except Exception as e:
logger.error('Error downloading from alphavantage %s %s return None :%s' % (instrument.symbol, period, e))
return None
outputComplete = self.formatHistorical(data_downloaded, period=period)
outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument)
return outputComplete
def __getAllInstrumentData__(self, instrumentList, ratioList, fromDate,
toDate):
wrongInstrumentsSymbolLists = []
# nanDF = self.vectorizedDataService.__createDataframe__(instrumentList, fromDate, toDate)
columnList = DataDictKeys.keys + ratioList
outputDict = {}
if len(instrumentList) < self.threads:
self.threads = int(np.ceil(len(instrumentList) / 2))
logger.debug('Modified threads of __getAllInstrumentData__ to %d' %
self.threads)
if self.threads > 1:
mpBatches = 1
linMols = True
if self.threads > 3:
mpBatches = float(len(instrumentList)) / float(self.threads)
mpBatches = int(min(int(mpBatches / 5), 50))
if mpBatches < 1:
mpBatches = 1
logger.debug('mpBatches of __getAllInstrumentData__ to %d' %
mpBatches)
# mpBatches = 1
outputDict = mpPandasObj(
func=self.__getAllInstrumentSerial__,
pdObj=('instrumentList', instrumentList),
numThreads=self.threads,
mpBatches=mpBatches,
isVerticalParallel=True,
linMols=linMols,
columnList=columnList,
fromDate=fromDate,
toDate=toDate,
outputDict=outputDict,
wrongInstrumentsSymbolLists=wrongInstrumentsSymbolLists,
)
else:
logger.info('Downloading data serialized')
outputDict = self.__getAllInstrumentSerial__(
instrumentList, columnList, fromDate, toDate, outputDict,
wrongInstrumentsSymbolLists)
for key in outputDict.keys():
if isinstance(outputDict[key], pd.DataFrame):
outputDict[key].fillna(method='ffill', inplace=True)
outputDict[key].fillna(0, inplace=True) # 1st row
# filter to asked data
if outputDict is not None:
outputDictFinal = {}
for ratioAsked in columnList:
outputDictFinal[ratioAsked] = outputDict[ratioAsked]
else:
outputDictFinal = None
return outputDictFinal
def getRatioDataBatch(self, instrument, ratio_list, fromDate, toDate=None, force_download=False):
fromDate = convert_date(fromDate)
toDate = convert_date(toDate)
df_temp = None
df = self.__getRatioDataDBBatch__(instrument, ratio_list, fromDate, toDate)
fromDateDictTemp = None
if df is not None:
if (df.index[0] - fromDate).days > self.tresholdDaysUpdate and force_download:
if fromDateDictTemp is not None and fromDateDictTemp <= fromDate:
logger.debug('max downloaded %s ,cant download more to %s' % (ratio_list, instrument))
else:
# messageError = 'some data missing begining ratio %s in %s fromDateDictTemp=%s fromDate=%s' % (
# ratio, instrument.symbol, str(fromDateDictTemp), str(fromDate))
# logger.error(messageError)
df_temp = df.copy()
df = None
elif (toDate - df.index[-1]).days > self.tresholdDaysUpdate and force_download:
logger.debug('some data missing the end ')
df_temp = df.copy()
df = None
elif (len(df.columns) < len(ratio_list) and force_download):
logger.debug('some ratio missing in database')
df_temp = df.copy()
df = None
if df is None and force_download:
logger.debug('data %s %s_%s not found in database trying to download and save it!' % (
ratio_list, instrument.symbol, instrument.currency))
df = self.getRatioDataProviderBatch(instrument, ratio_list, fromDate, toDate)
# if df_to_save is not None:
# self.saveRatioDataFrameBatch(df_to_save, instrument, ratio_list)
# df = self.__getRatioDataDBBatch__(instrument, ratio_list, fromDate, toDate)
# else:
# logger.error('Not pssible to download %s for %s' % (ratio_list, instrument))
# df = None
if df is None:
if df_temp is not None:
logger.debug('ratioData: Cant download more , restoring DDBB files ')
df = df_temp
else:
logger.error(
"Error getting ratio data for %s %s_%s" % (ratio_list, instrument.symbol, instrument.currency))
return None
if fromDateDictTemp is not None:
fromDateSave = min(fromDate, fromDateDictTemp)
else:
fromDateSave = fromDate
# if fromDateSave != fromDateDictTemp:
# self.ratio_instrument_from_dict_date[
# self.__getKeyDict__(ratio, instrument)] = fromDateSave # add to dict and save
# self.__saveDictFromDate__() # save from date cache to file
return df
def _pricing_iter():
sid = 0
with maybe_show_progress(
symbols,
show_progress,
label='Downloading Tradea Database pricing data ') as it, \
requests.Session() as session:
for symbol in it:
logger.debug('zipline bundle downloading %s' % symbol)
try:
instrument = Instrument(
symbol=symbol, asset_type=AssetType.us_equity)
df = self.historical_market_data_service.getHistoricalData(
instrument,
period=Period.day,
number_of_periods=1,
fromDate=start,
toDate=end,
bar_type=BarType.time_bar,
force_download=False,
cleanOutliers=False)
except Exception as e:
logger.error(
'Error downloading bundle zipline %s : %s' %
(symbol, str(e)))
print('Error downloading bundle zipline %s : %s' %
(symbol, str(e)))
df = None
continue
# the start date is the date of the first trade and
# the end date is the date of the last trade
indexSet = df.index.copy()
indexSet = (indexSet + pd.DateOffset(hours=3)
) - pd.DateOffset(days=1)
df.index = indexSet
start_date = df.index[0]
end_date = df.index[-1]
# The auto_close date is the day after the last trade.
ac_date = end_date + pd.Timedelta(days=1)
metadata.iloc[
sid] = start_date, end_date, ac_date, symbol
df.rename(
columns={
Bar.open: 'open',
Bar.high: 'high',
Bar.low: 'low',
Bar.close: 'close',
Bar.volume: 'volume',
},
inplace=True,
)
yield sid, df
sid += 1
def historical_data_handler(self, msg):
logger.debug("IB historical received %s" % msg)
if self.messageCounter == 0:
logger.info("IB historical started! %s" % msg.date)
self.messageCounter += 1
if ('finished' in str(msg.date)) == True:
logger.info("IB historical finished! %s" % msg.date)
self.receivedAllHistorical = True
else:
self.appendData(msg)
def __formatDatesEndOfDay__(self, df, fromDate, toDate):
if df is None:
return None
dateRange = pd.date_range(start=fromDate, end=toDate, freq='D')
output = pd.DataFrame(None, columns=df.columns, index=dateRange)
logger.debug('getHistoricalData [endOfDayData]: Setting time to EOD ')
for column in df.columns:
closesIndexBar = output.index.searchsorted(df.index) - 1
output[column][closesIndexBar] = df[column].values
output.dropna(inplace=True)
return output
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
import pandas as pd
self.filesInDirectory = glob.glob(self.inputPath + os.sep + "*.csv")
logger.debug(
"Downloading from dukascopy file %s_%s %i_%s from %s to %s" %
(instrument.symbol, instrument.currency, number_of_periods, period,
fromDate, toDate))
toDateFile = self.getToDateFromFile(toDate, period)
if period == Period.tick:
# '%s%s_Ticks_%s-%s.csv'
file = self.tick_filename_format % (
instrument.symbol, instrument.currency,
fromDate.strftime(self.filenameDateFormat),
toDateFile.strftime(
self.filenameDateFormat)) # '%s%s_Candlestick_%i_%s_%s-%s
else:
# '%s%s_Candlestick_%i_%s_%a_%s-%s.csv'
file = self.bar_filename_format % (
instrument.symbol, instrument.currency, number_of_periods,
self.period_dict[period], 'ASK',
fromDate.strftime(self.filenameDateFormat),
toDateFile.strftime(
self.filenameDateFormat)) # '%s%s_Candlestick_%i_%s_%s-%s
for fileIterate in self.filesInDirectory:
fileIterateName = fileIterate.split(os.sep)[-1]
if fileIterateName == file:
logger.debug("File %s found , processing" % file)
try:
# data_downloaded = pd.DataFrame.from_csv(fileIterate, dayfirst=True) # date_format=self.dateFormat) date_parser=lambda x: datetime.strptime(x, '%d.%m.%Y %H:%M:%S.%f'))
data_downloaded = pd.read_csv(fileIterate)
except Exception as e:
logger.error('Cant read file %s some errors there' % file)
return None
outputComplete = self.formatHistorical(data_downloaded,
period=period)
# if fromDate is not None and toDate is not None:
# outputComplete = outputComplete[fromDate: toDate]
# elif fromDate is not None:
# outputComplete = outputComplete[fromDate:]
# elif toDate is not None:
# outputComplete = outputComplete[:toDate]
# self.markFileAsProcessed(file)
return outputComplete
def __loadDao__(self, instrument, outputDict, columnList, fromDate,
toDate):
toDate = toDate + datetime.timedelta(days=1)
dataframe = self.vectorDao.load(instrument,
columnList=columnList,
startTime=fromDate,
endTime=toDate)
if dataframe is None:
logger.debug('%s not found in dao' % instrument)
return outputDict, dataframe
if dataframe.dropna(axis=0).empty:
logger.debug('%s is empty => update with online data' %
(instrument))
return outputDict, None
dataframeFrom = dataframe.index[0]
dataframeTo = dataframe.index[-1]
if (dataframeFrom - fromDate).days > self.maxDaysUpdate:
logger.debug('fromDate = %s and get from=%s' %
(fromDate, dataframeFrom))
if (toDate - dataframeTo).days > self.maxDaysUpdate:
logger.debug('toDate = %s and get from=%s => update it' %
(toDate, dataframeTo))
return outputDict, None
outputDict = dataframeToDict(dataframe, instrument.symbol, outputDict,
columnList)
return outputDict, dataframe
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
self.lock.acquire()
logger.debug("Downloading %s %s from %s to %s" %
(instrument.symbol, period, str(fromDate), str(toDate)))
if instrument.asset_type == AssetType.forex:
self.whatToShow = 'MIDPOINT'
else:
self.whatToShow = 'TRADES'
contract = self.__makeContract__(instrument)
period = self.period_dict[period]
barSizeSetting = '%i %s' % (number_of_periods, period)
if number_of_periods > 1 or period == Period.second:
barSizeSetting = barSizeSetting + 's'
outputAppended = None
durationStr_list, toDateString_list = self.getPeriodsRequests(
period, fromDate, toDate)
for i in range(len(durationStr_list)):
durationStr = durationStr_list[i]
toDateString = toDateString_list[i]
output_request = self.__makeRequestSingle__(
contract, durationStr, toDateString, barSizeSetting)
if outputAppended is None:
outputAppended = output_request
else:
outputAppended = outputAppended.append(output_request)
if outputAppended is None:
logger.error("Couldnt download anything=> release return None")
else:
# clean unique index and sort
outputAppended = outputAppended[~outputAppended.index.duplicated(
keep='first')]
outputAppended.sort_index(inplace=True)
if fromDate is not None and toDate is not None:
outputAppended = outputAppended[fromDate:toDate]
elif fromDate is not None:
outputAppended = outputAppended[fromDate:]
elif toDate is not None:
outputAppended = outputAppended[:toDate]
self.lock.release()
return outputAppended
def __save_ratio_list__(self, instrumentList, ratioList, fromDate, toDate):
pk = self.__getInputKeys__(instrumentList, fromDate, toDate)
if pk not in self.ratio_list_from_input_vectorized.keys():
logger.debug('Saving cacheRatioList %s' % str(ratioList))
self.ratio_list_from_input_vectorized[pk] = ratioList
joblib.dump(self.ratio_list_from_input_vectorized,
self.temp_dir_data_downloaded)
else:
previous_ratioList = self.ratio_list_from_input_vectorized[pk]
if len(ratioList) > len(previous_ratioList):
logger.debug('Saving/updating cacheRatioList %s' %
str(ratioList))
self.ratio_list_from_input_vectorized[pk] = ratioList
joblib.dump(self.ratio_list_from_input_vectorized,
self.temp_dir_data_downloaded)
def load(self, itemName, startTime=None, endTime=None):
# Cache
key = self.__getMD5_key__(itemName, startTime, endTime)
if key in self.cacheDict.keys():
return self.cacheDict[key]
try:
item = self.collection.item(itemName)
except Exception as e:
logger.error("symbol %s not found on database to load => return None %s" % (itemName, e))
return None
df = item.to_pandas()
if startTime is not None:
startTime = np.datetime64(startTime)
if endTime is not None:
endTime = np.datetime64(endTime)
try:
output = df
mask = None
if startTime is not None and endTime is not None:
mask = (df.index >= startTime) & (df.index <= endTime)
# output = df[startTime:endTime]
elif startTime is not None:
mask = (df.index >= startTime)
# output = df[startTime:]
elif endTime is not None:
mask = (df.index <= endTime)
if mask is not None:
output = df.loc[mask]
if len(output) == 0:
output = None
else:
output = self.cleanDataframeDatetime(output)
logger.debug("Successfully load from %s: %s a dataframe of %d rows %d columns" % (
self.collectionName, itemName, output.shape[0], output.shape[1]))
self.cacheDict[key] = output # Cache
except Exception as e:
logger.error('Error loading %s from %s return None :%s' % (itemName, item._path, str(e)))
output = None
return output
def download(self):
filesInDirectory = self.getFilesInPath()
output = True
for file in filesInDirectory:
if 'empty' in file:
logger.debug('file: %s skipped=> empty is a FW' % file)
continue
instrument = self.getInstrument(file)
logger.debug('Importing ratios of %s_%s' %
(instrument.symbol, instrument.currency))
isImported = self.importExcel(file, instrument)
if isImported is False:
logger.error('Error ratios of %s_%s :%s' %
(instrument.symbol, instrument.currency, file))
output = output and isImported
return output
def downloadBatch(self,
instrument,
fundamental_ratio_list,
fromDate,
toDate=None):
logger.debug('Downloading batch iterating... not own implementation')
output = None
for fundamental_ratio in fundamental_ratio_list:
ratioDF = self.download(instrument, fundamental_ratio, fromDate,
toDate)
if output is None:
output = pd.DataFrame(None,
columns=fundamental_ratio_list,
index=ratioDF.index)
output[fundamental_ratio] = ratioDF
return output
def __cleanOutputDict__(self, outputDict, wrongInstrumentsSymbolLists):
if len(wrongInstrumentsSymbolLists) == 0:
return outputDict
logger.debug('Some errors detected in %s => cleaning' %
str(wrongInstrumentsSymbolLists))
keysInDict = outputDict.keys()
for key in keysInDict:
dataframeToRemoveColumns = outputDict[key]
try:
dataframeToRemoveColumns.drop(wrongInstrumentsSymbolLists,
axis=1,
inplace=True)
outputDict[key] = dataframeToRemoveColumns
except:
pass
return outputDict
def cleanPricesFactor(factorDf, prices, cleanRows=True):
import numpy as np
logger.debug('Cleaning factorDF+prices to alphalens')
# remove symbols all prices to zero,
columnsStart = list(factorDf.index.levels[1])
columnsRemove = list(prices.sum(axis=0)[prices.sum(axis=0) == 0].index)
if len(columnsRemove) > 0:
prices = prices.drop(columnsRemove, axis=1)
factorDf = factorDf.drop(columnsRemove, axis=1, level=1)
factorDf = __resetMultiIndexDF__(factorDf)
columnsFinal = list(factorDf.index.levels[1])
logger.debug(
'Cleaned columns factor/prices from %d to %d, removed :%s' %
(len(columnsStart), len(columnsFinal), str(columnsRemove)))
# rows with zero prices => remove it from factorDF
if cleanRows:
originalIndex = len(prices.index)
originalFrom = str(prices.index[0])
prices = prices.replace([np.inf, -np.inf, 0], np.nan)
prices.dropna(axis=0, inplace=True)
if prices.empty:
logger.error('Prices are empty after cleaning')
newIndexTime = prices.index
newIndexFrom = str(newIndexTime[0])
factorDf = factorDf.T[newIndexTime].T
factorDf = __resetMultiIndexDF__(factorDf)
logger.debug(
'Cleaned rows factor/prices from[%d] %s to[%d] %s' %
(originalIndex, originalFrom, len(newIndexTime), newIndexFrom))
# columns with zero prices=> remove it
else:
prices = prices.replace([np.inf, -np.inf, 0], np.nan)
prices.dropna(axis=1, inplace=True)
columnsRemove = [
x for x in list(factorDf.index.levels[1])
if x not in list(prices.columns)
]
factorDf = factorDf.drop(columnsRemove, axis=0, level=1)
factorDf = __resetMultiIndexDF__(factorDf)
logger.debug('Cleaned columns factor/prices %s' % str(columnsRemove))
logger.debug('Cleaned prices shape = %d rows and %d columns' %
(prices.shape[0], prices.shape[1]))
return factorDf, prices
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
import datetime
logger.debug("Downloading %s" % instrument)
oandaInstrument = '%s_%s' % (instrument.symbol, instrument.currency)
if period == Period.day:
oandaGranularity = self.period_dict[period]
else:
oandaGranularity = '%s%i' % (self.period_dict[period],
number_of_periods)
# 2014-07-03T04:00:00.000000Z
startDate = fromDate.strftime(self.formatDate)
if toDate is None:
toDate = datetime.datetime.today()
endDate = toDate.strftime(self.formatDate)
try:
data_downloaded = self.oanda.get_history(
instrument=oandaInstrument,
granularity=oandaGranularity,
start=startDate,
end=endDate,
candleFormat=self.candleFormat,
dailyAlignment=self.dailyAlignment,
alignmentTimezone=self.alignmentTimezone,
weeklyAlignment=self.weeklyAlignment,
)
except Exception as e:
logger.error("Cant download from oanda %s %s=> return None %s" %
(instrument.symbol, period, e))
return None
logger.info("formatting oanda data for %s" % oandaInstrument)
outputComplete = self.formatHistorical(data_downloaded, period=period)
# Already added
# outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument)
return outputComplete
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
import datetime
logger.debug("Downloading %s" % instrument)
if period != Period.day:
logger.error("Quandl can only download daily! not %s" % period)
return None
dateFormat = "%Y-%m-%d"
if toDate is None:
toDate = datetime.datetime.today()
# download dataframe
prefix = 'CHRIS'
if instrument.asset_type == AssetType.commodity_future:
prefix = 'CHRIS'
if instrument.asset_type == AssetType.future:
prefix = 'CHRIS'
quandlDatabase = '%s/%s' % (prefix, instrument.symbol)
try:
data_downloaded = quandl.get(
quandlDatabase,
start_date=fromDate.strftime(dateFormat),
end_date=toDate.strftime(dateFormat)
) # , authtoken=self.user_settings.quandl_token)
# data_downloaded = pdr.get_data_yahoo(instrument.symbol, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat))
except Exception as e:
logger.error("Cant download from quandl %s => return None %s" %
(instrument.symbol, e))
return None
outputComplete = self.formatHistorical(data_downloaded)
outputComplete = self.setTimeCorrect(outputComplete,
period=period,
instrument=instrument)
return outputComplete
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
import datetime
logger.debug("Downloading %s yahoo" % (instrument.symbol))
import fix_yahoo_finance as yf
yf.pdr_override() # <== that's all it takes :-)
if period != Period.day:
logger.error("Yahoo can only download daily! not %s" % period)
return None
dateFormat = "%Y-%m-%d"
if toDate is None:
toDate = datetime.datetime.today()
# download dataframe
try:
if instrument.asset_type == AssetType.index and not instrument.symbol.upper(
).startswith('^'):
symbolToDownload = '^' + instrument.symbol.upper()
else:
symbolToDownload = instrument.symbol.upper()
data_downloaded = yf.download(symbolToDownload,
start=fromDate.strftime(dateFormat),
end=toDate.strftime(dateFormat))
# data_downloaded = pdr.get_data_yahoo(instrument.symbol, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat))
except Exception as e:
logger.error(
"Cant download from yahoo %s %s => return None %s" %
(instrument.symbol, period, e))
return None
outputComplete = self.formatHistorical(data_downloaded)
outputComplete = self.setTimeCorrect(outputComplete,
period=period,
instrument=instrument)
return outputComplete
def download(self, instrument, period, number_of_periods, fromDate, toDate=None):
import datetime
logger.debug("Downloading %s_%s google" % (instrument.symbol, instrument.currency))
if period != Period.day:
logger.error("Google can only download daily! not %s" % period)
return None
dateFormat = "%Y-%m-%d"
if toDate is None:
toDate = datetime.datetime.today()
# download dataframe
try:
if instrument.asset_type == AssetType.index:
symbolToDownload = '^' + instrument.symbol.upper()
else:
symbolToDownload = instrument.symbol.upper()
data_downloaded = data.DataReader(symbolToDownload, data_source='google', start=fromDate, end=toDate)
except Exception as e:
logger.error("Cant download from google %s %s => return None %s" % (instrument.symbol, period, e))
return None
outputComplete = self.formatHistorical(data_downloaded)
outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument)
return outputComplete
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
import datetime
logger.debug("Downloading %s" % instrument.symbol)
if period != Period.day:
logger.error("Bloomberg can only download daily! not %s" % period)
return None
if toDate is None:
toDate = datetime.datetime.today()
try:
data_downloaded = self.bdh(
instrument.symbol +
self.suffix_assetType[instrument.asset_type],
self.period_dict[period], fromDate.strftime(self.formatDate),
toDate.strftime(self.formatDate))
except Exception as e:
logger.error(
"Cant download from bloomberg %s => return None %s" %
(instrument.symbol, e))
return None
data_downloaded = data_downloaded[instrument.symbol]
outputComplete = self.formatHistorical(data_downloaded)
if fromDate is not None and toDate is not None:
outputComplete = outputComplete[fromDate:toDate]
elif fromDate is not None:
outputComplete = outputComplete[fromDate:]
elif toDate is not None:
outputComplete = outputComplete[:toDate]
return outputComplete
def __makeRequestSingle__(self, contract, durationStr, toDateString,
barSizeSetting):
self.receivedDataObject.reset()
self.ib_object.reqHistoricalData(self.tickId, contract, toDateString,
durationStr, barSizeSetting,
self.whatToShow, 1,
self.ib_formatDate_return)
self.tickId += 1
logger.info("send req historical %s : waiting" % contract.m_symbol)
while (self.receivedDataObject.receivedAllHistorical is False):
sleep(3)
logger.debug("finished single request %s " % contract.m_symbol)
dataframeReceived = self.receivedDataObject.getDataframe()
if dataframeReceived is None:
logger.error("Some error appears on single request !! check it")
return None
# is necessary???
outputComplete = self.__formatHistorical__(dataframeReceived)
return outputComplete
def download(self,
instrument,
period,
number_of_periods,
fromDate,
toDate=None):
logger.debug("Downloading %s" % instrument)
crypto_period = self.period_dict[period]
dict_downloaded = price.get_historical_data(
instrument.symbol,
instrument.currency,
crypto_period,
aggregate=number_of_periods)
if len(dict_downloaded) == 0:
logger.error(
'Cant download cryptocompare %s_%s for %s return None' %
(instrument.symbol, instrument.currency, period))
return None
outputComplete = self.formatHistorical(dict_downloaded)
# Already added
# outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument)
return outputComplete
