def dictToDataframe(outputDict, symbol, columnList): dataframeSave = None for key in columnList: if key in outputDict.keys(): if symbol in list(outputDict[key].columns): seriesFromDict = outputDict[key][symbol] seriesFromDict = seriesFromDict[~seriesFromDict.index. duplicated(keep='last')] else: # data not downloaded logger.debug('%s not found in %s => none' % (key, symbol)) seriesFromDict = None if seriesFromDict is not None: if dataframeSave is None: dataframeSave = pd.DataFrame(seriesFromDict.values, index=seriesFromDict.index, columns=[key]) else: dataframeSave[ key] = seriesFromDict # pd.Series(seriesFromDict.values,index = dataframeSave.index) if dataframeSave is not None: dataframeSave.fillna(method='ffill', inplace=True) # remove duplicated dataframeSave = dataframeSave[~dataframeSave.index.duplicated( keep='last')] return dataframeSave
def cleanOutliersData(self, instrument, period, df): if df is None: return None # TODO improve to get average of surrounded instead of delete output = df.copy() if period == Period.tick: wrongBid = mad_outlier(df[Tick.bid].values.reshape(-1, 1)) output = df.loc[~wrongBid] wrongAsk = mad_outlier(df[Tick.ask].values.reshape(-1, 1)) output = df.loc[~wrongAsk] else: wrongClose = mad_outlier(df[Bar.close].values.reshape(-1, 1)) output = df.loc[~wrongClose] wrongOpen = mad_outlier(df[Bar.open].values.reshape(-1, 1)) output = df.loc[~wrongOpen] if len(output) == 0: logger.error("All data was wrong for %s_%s when cleaning" % (instrument.symbol, instrument.currency)) else: errorsFind = len(df) - len(output) if errorsFind is not 0: logger.debug('Find %i outliers to remove in %s_%s when cleaning' % (errorsFind, instrument.symbol, instrument.currency)) return output
def __cleanData__(self, outputDict, assetType, persistTempFile=None): logger.debug('all instruments processed => cleaning ') if 'wrong' in outputDict.keys(): wrongInstrumentsSymbolLists = outputDict['wrong'].copy() if 'wrong' in outputDict: del outputDict['wrong'] wrongInstrumentsSymbolLists = self.vectorizedDataService.__formatWrongInstrumentList__( wrongInstrumentsSymbolLists, outputDict) outputDictFinal = self.vectorizedDataService.__cleanOutputDict__( outputDict, wrongInstrumentsSymbolLists) else: outputDictFinal = outputDict if assetType != AssetType.forex and assetType != AssetType.crypto: outputDictFinal = self.vectorizedDataService.__cleanBankHolidays__( outputDictFinal) df = outputDictFinal[DataDictKeys.close] logger.debug( 'all dictOfMatrix cleaned => finished %d matrixes of %d columns' % (len(outputDictFinal), df.shape[1])) if persistTempFile is not None: if persistTempFile is False: persistTempFile = None if persistTempFile is True: persistTempFile = 'default.xlsx' if ~persistTempFile.endswith('.xlsx'): persistTempFile += '.xlsx' self.vectorizedDataService.__createTempExcelFile__( outputDictFinal, filenameWithExtension=persistTempFile) return outputDictFinal
def importExcel(self, file, instrument): reader = pd.ExcelFile(file) sheetNames = reader.sheet_names output = True for sheet in sheetNames: ratio = self.getRatioFromSheet(sheet) logger.debug('%s_%s reading %s' % (instrument.symbol, instrument.currency, ratio)) dataDownloaded = reader.parse(sheet) df_to_save = self.formatFundamental(dataDownloaded) # save it to database if df_to_save is not None: logger.debug('%s_%s saving to database %s' % (instrument.symbol, instrument.currency, ratio)) try: self.ratioDataService.saveRatioDataFrame( df_to_save, instrument, ratio) except Exception as e: logger.error('Error saving Ratio %s of %s_%s :%s' % (ratio, instrument.symbol, instrument.currency, str(e))) output = False else: logger.error('Error getting Ratio %s of %s_%s' % (ratio, instrument.symbol, instrument.currency)) output = False return output
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import datetime logger.debug("Downloading %s" % instrument.symbol) try: if period != Period.day: # '1min', '5min', '15min', '30min', '60min' if number_of_periods in [1, 5, 15, 30, 60]: periodAlpha = '%i%s' % (number_of_periods, self.period_dict[period]) data_downloaded, meta_data = self.timeseries.get_intraday(instrument.symbol, interval=periodAlpha) else: logger.error( "AlphaVantage can only download intradaily! '1min', '5min', '15min', '30min', '60min' not %s" % period) return None else: # dateFormat = "%Y-%m-%d" if toDate is None: toDate = datetime.datetime.today() # download dataframe try: data_downloaded, meta_data = self.timeseries.get_daily_adjusted(instrument.symbol) # data_downloaded = pdr.get_data_yahoo(instrument.symbol, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat)) except Exception as e: logger.error("Cand download from alphavantage %s => return None %s" % (instrument.symbol, e)) return None except Exception as e: logger.error('Error downloading from alphavantage %s %s return None :%s' % (instrument.symbol, period, e)) return None outputComplete = self.formatHistorical(data_downloaded, period=period) outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument) return outputComplete
def __getAllInstrumentData__(self, instrumentList, ratioList, fromDate, toDate): wrongInstrumentsSymbolLists = [] # nanDF = self.vectorizedDataService.__createDataframe__(instrumentList, fromDate, toDate) columnList = DataDictKeys.keys + ratioList outputDict = {} if len(instrumentList) < self.threads: self.threads = int(np.ceil(len(instrumentList) / 2)) logger.debug('Modified threads of __getAllInstrumentData__ to %d' % self.threads) if self.threads > 1: mpBatches = 1 linMols = True if self.threads > 3: mpBatches = float(len(instrumentList)) / float(self.threads) mpBatches = int(min(int(mpBatches / 5), 50)) if mpBatches < 1: mpBatches = 1 logger.debug('mpBatches of __getAllInstrumentData__ to %d' % mpBatches) # mpBatches = 1 outputDict = mpPandasObj( func=self.__getAllInstrumentSerial__, pdObj=('instrumentList', instrumentList), numThreads=self.threads, mpBatches=mpBatches, isVerticalParallel=True, linMols=linMols, columnList=columnList, fromDate=fromDate, toDate=toDate, outputDict=outputDict, wrongInstrumentsSymbolLists=wrongInstrumentsSymbolLists, ) else: logger.info('Downloading data serialized') outputDict = self.__getAllInstrumentSerial__( instrumentList, columnList, fromDate, toDate, outputDict, wrongInstrumentsSymbolLists) for key in outputDict.keys(): if isinstance(outputDict[key], pd.DataFrame): outputDict[key].fillna(method='ffill', inplace=True) outputDict[key].fillna(0, inplace=True) # 1st row # filter to asked data if outputDict is not None: outputDictFinal = {} for ratioAsked in columnList: outputDictFinal[ratioAsked] = outputDict[ratioAsked] else: outputDictFinal = None return outputDictFinal
def getRatioDataBatch(self, instrument, ratio_list, fromDate, toDate=None, force_download=False): fromDate = convert_date(fromDate) toDate = convert_date(toDate) df_temp = None df = self.__getRatioDataDBBatch__(instrument, ratio_list, fromDate, toDate) fromDateDictTemp = None if df is not None: if (df.index[0] - fromDate).days > self.tresholdDaysUpdate and force_download: if fromDateDictTemp is not None and fromDateDictTemp <= fromDate: logger.debug('max downloaded %s ,cant download more to %s' % (ratio_list, instrument)) else: # messageError = 'some data missing begining ratio %s in %s fromDateDictTemp=%s fromDate=%s' % ( # ratio, instrument.symbol, str(fromDateDictTemp), str(fromDate)) # logger.error(messageError) df_temp = df.copy() df = None elif (toDate - df.index[-1]).days > self.tresholdDaysUpdate and force_download: logger.debug('some data missing the end ') df_temp = df.copy() df = None elif (len(df.columns) < len(ratio_list) and force_download): logger.debug('some ratio missing in database') df_temp = df.copy() df = None if df is None and force_download: logger.debug('data %s %s_%s not found in database trying to download and save it!' % ( ratio_list, instrument.symbol, instrument.currency)) df = self.getRatioDataProviderBatch(instrument, ratio_list, fromDate, toDate) # if df_to_save is not None: # self.saveRatioDataFrameBatch(df_to_save, instrument, ratio_list) # df = self.__getRatioDataDBBatch__(instrument, ratio_list, fromDate, toDate) # else: # logger.error('Not pssible to download %s for %s' % (ratio_list, instrument)) # df = None if df is None: if df_temp is not None: logger.debug('ratioData: Cant download more , restoring DDBB files ') df = df_temp else: logger.error( "Error getting ratio data for %s %s_%s" % (ratio_list, instrument.symbol, instrument.currency)) return None if fromDateDictTemp is not None: fromDateSave = min(fromDate, fromDateDictTemp) else: fromDateSave = fromDate # if fromDateSave != fromDateDictTemp: # self.ratio_instrument_from_dict_date[ # self.__getKeyDict__(ratio, instrument)] = fromDateSave # add to dict and save # self.__saveDictFromDate__() # save from date cache to file return df
def _pricing_iter(): sid = 0 with maybe_show_progress( symbols, show_progress, label='Downloading Tradea Database pricing data ') as it, \ requests.Session() as session: for symbol in it: logger.debug('zipline bundle downloading %s' % symbol) try: instrument = Instrument( symbol=symbol, asset_type=AssetType.us_equity) df = self.historical_market_data_service.getHistoricalData( instrument, period=Period.day, number_of_periods=1, fromDate=start, toDate=end, bar_type=BarType.time_bar, force_download=False, cleanOutliers=False) except Exception as e: logger.error( 'Error downloading bundle zipline %s : %s' % (symbol, str(e))) print('Error downloading bundle zipline %s : %s' % (symbol, str(e))) df = None continue # the start date is the date of the first trade and # the end date is the date of the last trade indexSet = df.index.copy() indexSet = (indexSet + pd.DateOffset(hours=3) ) - pd.DateOffset(days=1) df.index = indexSet start_date = df.index[0] end_date = df.index[-1] # The auto_close date is the day after the last trade. ac_date = end_date + pd.Timedelta(days=1) metadata.iloc[ sid] = start_date, end_date, ac_date, symbol df.rename( columns={ Bar.open: 'open', Bar.high: 'high', Bar.low: 'low', Bar.close: 'close', Bar.volume: 'volume', }, inplace=True, ) yield sid, df sid += 1
def historical_data_handler(self, msg): logger.debug("IB historical received %s" % msg) if self.messageCounter == 0: logger.info("IB historical started! %s" % msg.date) self.messageCounter += 1 if ('finished' in str(msg.date)) == True: logger.info("IB historical finished! %s" % msg.date) self.receivedAllHistorical = True else: self.appendData(msg)
def __formatDatesEndOfDay__(self, df, fromDate, toDate): if df is None: return None dateRange = pd.date_range(start=fromDate, end=toDate, freq='D') output = pd.DataFrame(None, columns=df.columns, index=dateRange) logger.debug('getHistoricalData [endOfDayData]: Setting time to EOD ') for column in df.columns: closesIndexBar = output.index.searchsorted(df.index) - 1 output[column][closesIndexBar] = df[column].values output.dropna(inplace=True) return output
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import pandas as pd self.filesInDirectory = glob.glob(self.inputPath + os.sep + "*.csv") logger.debug( "Downloading from dukascopy file %s_%s %i_%s from %s to %s" % (instrument.symbol, instrument.currency, number_of_periods, period, fromDate, toDate)) toDateFile = self.getToDateFromFile(toDate, period) if period == Period.tick: # '%s%s_Ticks_%s-%s.csv' file = self.tick_filename_format % ( instrument.symbol, instrument.currency, fromDate.strftime(self.filenameDateFormat), toDateFile.strftime( self.filenameDateFormat)) # '%s%s_Candlestick_%i_%s_%s-%s else: # '%s%s_Candlestick_%i_%s_%a_%s-%s.csv' file = self.bar_filename_format % ( instrument.symbol, instrument.currency, number_of_periods, self.period_dict[period], 'ASK', fromDate.strftime(self.filenameDateFormat), toDateFile.strftime( self.filenameDateFormat)) # '%s%s_Candlestick_%i_%s_%s-%s for fileIterate in self.filesInDirectory: fileIterateName = fileIterate.split(os.sep)[-1] if fileIterateName == file: logger.debug("File %s found , processing" % file) try: # data_downloaded = pd.DataFrame.from_csv(fileIterate, dayfirst=True) # date_format=self.dateFormat) date_parser=lambda x: datetime.strptime(x, '%d.%m.%Y %H:%M:%S.%f')) data_downloaded = pd.read_csv(fileIterate) except Exception as e: logger.error('Cant read file %s some errors there' % file) return None outputComplete = self.formatHistorical(data_downloaded, period=period) # if fromDate is not None and toDate is not None: # outputComplete = outputComplete[fromDate: toDate] # elif fromDate is not None: # outputComplete = outputComplete[fromDate:] # elif toDate is not None: # outputComplete = outputComplete[:toDate] # self.markFileAsProcessed(file) return outputComplete
def __loadDao__(self, instrument, outputDict, columnList, fromDate, toDate): toDate = toDate + datetime.timedelta(days=1) dataframe = self.vectorDao.load(instrument, columnList=columnList, startTime=fromDate, endTime=toDate) if dataframe is None: logger.debug('%s not found in dao' % instrument) return outputDict, dataframe if dataframe.dropna(axis=0).empty: logger.debug('%s is empty => update with online data' % (instrument)) return outputDict, None dataframeFrom = dataframe.index[0] dataframeTo = dataframe.index[-1] if (dataframeFrom - fromDate).days > self.maxDaysUpdate: logger.debug('fromDate = %s and get from=%s' % (fromDate, dataframeFrom)) if (toDate - dataframeTo).days > self.maxDaysUpdate: logger.debug('toDate = %s and get from=%s => update it' % (toDate, dataframeTo)) return outputDict, None outputDict = dataframeToDict(dataframe, instrument.symbol, outputDict, columnList) return outputDict, dataframe
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): self.lock.acquire() logger.debug("Downloading %s %s from %s to %s" % (instrument.symbol, period, str(fromDate), str(toDate))) if instrument.asset_type == AssetType.forex: self.whatToShow = 'MIDPOINT' else: self.whatToShow = 'TRADES' contract = self.__makeContract__(instrument) period = self.period_dict[period] barSizeSetting = '%i %s' % (number_of_periods, period) if number_of_periods > 1 or period == Period.second: barSizeSetting = barSizeSetting + 's' outputAppended = None durationStr_list, toDateString_list = self.getPeriodsRequests( period, fromDate, toDate) for i in range(len(durationStr_list)): durationStr = durationStr_list[i] toDateString = toDateString_list[i] output_request = self.__makeRequestSingle__( contract, durationStr, toDateString, barSizeSetting) if outputAppended is None: outputAppended = output_request else: outputAppended = outputAppended.append(output_request) if outputAppended is None: logger.error("Couldnt download anything=> release return None") else: # clean unique index and sort outputAppended = outputAppended[~outputAppended.index.duplicated( keep='first')] outputAppended.sort_index(inplace=True) if fromDate is not None and toDate is not None: outputAppended = outputAppended[fromDate:toDate] elif fromDate is not None: outputAppended = outputAppended[fromDate:] elif toDate is not None: outputAppended = outputAppended[:toDate] self.lock.release() return outputAppended
def __save_ratio_list__(self, instrumentList, ratioList, fromDate, toDate): pk = self.__getInputKeys__(instrumentList, fromDate, toDate) if pk not in self.ratio_list_from_input_vectorized.keys(): logger.debug('Saving cacheRatioList %s' % str(ratioList)) self.ratio_list_from_input_vectorized[pk] = ratioList joblib.dump(self.ratio_list_from_input_vectorized, self.temp_dir_data_downloaded) else: previous_ratioList = self.ratio_list_from_input_vectorized[pk] if len(ratioList) > len(previous_ratioList): logger.debug('Saving/updating cacheRatioList %s' % str(ratioList)) self.ratio_list_from_input_vectorized[pk] = ratioList joblib.dump(self.ratio_list_from_input_vectorized, self.temp_dir_data_downloaded)
def load(self, itemName, startTime=None, endTime=None): # Cache key = self.__getMD5_key__(itemName, startTime, endTime) if key in self.cacheDict.keys(): return self.cacheDict[key] try: item = self.collection.item(itemName) except Exception as e: logger.error("symbol %s not found on database to load => return None %s" % (itemName, e)) return None df = item.to_pandas() if startTime is not None: startTime = np.datetime64(startTime) if endTime is not None: endTime = np.datetime64(endTime) try: output = df mask = None if startTime is not None and endTime is not None: mask = (df.index >= startTime) & (df.index <= endTime) # output = df[startTime:endTime] elif startTime is not None: mask = (df.index >= startTime) # output = df[startTime:] elif endTime is not None: mask = (df.index <= endTime) if mask is not None: output = df.loc[mask] if len(output) == 0: output = None else: output = self.cleanDataframeDatetime(output) logger.debug("Successfully load from %s: %s a dataframe of %d rows %d columns" % ( self.collectionName, itemName, output.shape[0], output.shape[1])) self.cacheDict[key] = output # Cache except Exception as e: logger.error('Error loading %s from %s return None :%s' % (itemName, item._path, str(e))) output = None return output
def download(self): filesInDirectory = self.getFilesInPath() output = True for file in filesInDirectory: if 'empty' in file: logger.debug('file: %s skipped=> empty is a FW' % file) continue instrument = self.getInstrument(file) logger.debug('Importing ratios of %s_%s' % (instrument.symbol, instrument.currency)) isImported = self.importExcel(file, instrument) if isImported is False: logger.error('Error ratios of %s_%s :%s' % (instrument.symbol, instrument.currency, file)) output = output and isImported return output
def downloadBatch(self, instrument, fundamental_ratio_list, fromDate, toDate=None): logger.debug('Downloading batch iterating... not own implementation') output = None for fundamental_ratio in fundamental_ratio_list: ratioDF = self.download(instrument, fundamental_ratio, fromDate, toDate) if output is None: output = pd.DataFrame(None, columns=fundamental_ratio_list, index=ratioDF.index) output[fundamental_ratio] = ratioDF return output
def __cleanOutputDict__(self, outputDict, wrongInstrumentsSymbolLists): if len(wrongInstrumentsSymbolLists) == 0: return outputDict logger.debug('Some errors detected in %s => cleaning' % str(wrongInstrumentsSymbolLists)) keysInDict = outputDict.keys() for key in keysInDict: dataframeToRemoveColumns = outputDict[key] try: dataframeToRemoveColumns.drop(wrongInstrumentsSymbolLists, axis=1, inplace=True) outputDict[key] = dataframeToRemoveColumns except: pass return outputDict
def cleanPricesFactor(factorDf, prices, cleanRows=True): import numpy as np logger.debug('Cleaning factorDF+prices to alphalens') # remove symbols all prices to zero, columnsStart = list(factorDf.index.levels[1]) columnsRemove = list(prices.sum(axis=0)[prices.sum(axis=0) == 0].index) if len(columnsRemove) > 0: prices = prices.drop(columnsRemove, axis=1) factorDf = factorDf.drop(columnsRemove, axis=1, level=1) factorDf = __resetMultiIndexDF__(factorDf) columnsFinal = list(factorDf.index.levels[1]) logger.debug( 'Cleaned columns factor/prices from %d to %d, removed :%s' % (len(columnsStart), len(columnsFinal), str(columnsRemove))) # rows with zero prices => remove it from factorDF if cleanRows: originalIndex = len(prices.index) originalFrom = str(prices.index[0]) prices = prices.replace([np.inf, -np.inf, 0], np.nan) prices.dropna(axis=0, inplace=True) if prices.empty: logger.error('Prices are empty after cleaning') newIndexTime = prices.index newIndexFrom = str(newIndexTime[0]) factorDf = factorDf.T[newIndexTime].T factorDf = __resetMultiIndexDF__(factorDf) logger.debug( 'Cleaned rows factor/prices from[%d] %s to[%d] %s' % (originalIndex, originalFrom, len(newIndexTime), newIndexFrom)) # columns with zero prices=> remove it else: prices = prices.replace([np.inf, -np.inf, 0], np.nan) prices.dropna(axis=1, inplace=True) columnsRemove = [ x for x in list(factorDf.index.levels[1]) if x not in list(prices.columns) ] factorDf = factorDf.drop(columnsRemove, axis=0, level=1) factorDf = __resetMultiIndexDF__(factorDf) logger.debug('Cleaned columns factor/prices %s' % str(columnsRemove)) logger.debug('Cleaned prices shape = %d rows and %d columns' % (prices.shape[0], prices.shape[1])) return factorDf, prices
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import datetime logger.debug("Downloading %s" % instrument) oandaInstrument = '%s_%s' % (instrument.symbol, instrument.currency) if period == Period.day: oandaGranularity = self.period_dict[period] else: oandaGranularity = '%s%i' % (self.period_dict[period], number_of_periods) # 2014-07-03T04:00:00.000000Z startDate = fromDate.strftime(self.formatDate) if toDate is None: toDate = datetime.datetime.today() endDate = toDate.strftime(self.formatDate) try: data_downloaded = self.oanda.get_history( instrument=oandaInstrument, granularity=oandaGranularity, start=startDate, end=endDate, candleFormat=self.candleFormat, dailyAlignment=self.dailyAlignment, alignmentTimezone=self.alignmentTimezone, weeklyAlignment=self.weeklyAlignment, ) except Exception as e: logger.error("Cant download from oanda %s %s=> return None %s" % (instrument.symbol, period, e)) return None logger.info("formatting oanda data for %s" % oandaInstrument) outputComplete = self.formatHistorical(data_downloaded, period=period) # Already added # outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument) return outputComplete
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import datetime logger.debug("Downloading %s" % instrument) if period != Period.day: logger.error("Quandl can only download daily! not %s" % period) return None dateFormat = "%Y-%m-%d" if toDate is None: toDate = datetime.datetime.today() # download dataframe prefix = 'CHRIS' if instrument.asset_type == AssetType.commodity_future: prefix = 'CHRIS' if instrument.asset_type == AssetType.future: prefix = 'CHRIS' quandlDatabase = '%s/%s' % (prefix, instrument.symbol) try: data_downloaded = quandl.get( quandlDatabase, start_date=fromDate.strftime(dateFormat), end_date=toDate.strftime(dateFormat) ) # , authtoken=self.user_settings.quandl_token) # data_downloaded = pdr.get_data_yahoo(instrument.symbol, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat)) except Exception as e: logger.error("Cant download from quandl %s => return None %s" % (instrument.symbol, e)) return None outputComplete = self.formatHistorical(data_downloaded) outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument) return outputComplete
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import datetime logger.debug("Downloading %s yahoo" % (instrument.symbol)) import fix_yahoo_finance as yf yf.pdr_override() # <== that's all it takes :-) if period != Period.day: logger.error("Yahoo can only download daily! not %s" % period) return None dateFormat = "%Y-%m-%d" if toDate is None: toDate = datetime.datetime.today() # download dataframe try: if instrument.asset_type == AssetType.index and not instrument.symbol.upper( ).startswith('^'): symbolToDownload = '^' + instrument.symbol.upper() else: symbolToDownload = instrument.symbol.upper() data_downloaded = yf.download(symbolToDownload, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat)) # data_downloaded = pdr.get_data_yahoo(instrument.symbol, start=fromDate.strftime(dateFormat), end=toDate.strftime(dateFormat)) except Exception as e: logger.error( "Cant download from yahoo %s %s => return None %s" % (instrument.symbol, period, e)) return None outputComplete = self.formatHistorical(data_downloaded) outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument) return outputComplete
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import datetime logger.debug("Downloading %s_%s google" % (instrument.symbol, instrument.currency)) if period != Period.day: logger.error("Google can only download daily! not %s" % period) return None dateFormat = "%Y-%m-%d" if toDate is None: toDate = datetime.datetime.today() # download dataframe try: if instrument.asset_type == AssetType.index: symbolToDownload = '^' + instrument.symbol.upper() else: symbolToDownload = instrument.symbol.upper() data_downloaded = data.DataReader(symbolToDownload, data_source='google', start=fromDate, end=toDate) except Exception as e: logger.error("Cant download from google %s %s => return None %s" % (instrument.symbol, period, e)) return None outputComplete = self.formatHistorical(data_downloaded) outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument) return outputComplete
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): import datetime logger.debug("Downloading %s" % instrument.symbol) if period != Period.day: logger.error("Bloomberg can only download daily! not %s" % period) return None if toDate is None: toDate = datetime.datetime.today() try: data_downloaded = self.bdh( instrument.symbol + self.suffix_assetType[instrument.asset_type], self.period_dict[period], fromDate.strftime(self.formatDate), toDate.strftime(self.formatDate)) except Exception as e: logger.error( "Cant download from bloomberg %s => return None %s" % (instrument.symbol, e)) return None data_downloaded = data_downloaded[instrument.symbol] outputComplete = self.formatHistorical(data_downloaded) if fromDate is not None and toDate is not None: outputComplete = outputComplete[fromDate:toDate] elif fromDate is not None: outputComplete = outputComplete[fromDate:] elif toDate is not None: outputComplete = outputComplete[:toDate] return outputComplete
def __makeRequestSingle__(self, contract, durationStr, toDateString, barSizeSetting): self.receivedDataObject.reset() self.ib_object.reqHistoricalData(self.tickId, contract, toDateString, durationStr, barSizeSetting, self.whatToShow, 1, self.ib_formatDate_return) self.tickId += 1 logger.info("send req historical %s : waiting" % contract.m_symbol) while (self.receivedDataObject.receivedAllHistorical is False): sleep(3) logger.debug("finished single request %s " % contract.m_symbol) dataframeReceived = self.receivedDataObject.getDataframe() if dataframeReceived is None: logger.error("Some error appears on single request !! check it") return None # is necessary??? outputComplete = self.__formatHistorical__(dataframeReceived) return outputComplete
def download(self, instrument, period, number_of_periods, fromDate, toDate=None): logger.debug("Downloading %s" % instrument) crypto_period = self.period_dict[period] dict_downloaded = price.get_historical_data( instrument.symbol, instrument.currency, crypto_period, aggregate=number_of_periods) if len(dict_downloaded) == 0: logger.error( 'Cant download cryptocompare %s_%s for %s return None' % (instrument.symbol, instrument.currency, period)) return None outputComplete = self.formatHistorical(dict_downloaded) # Already added # outputComplete = self.setTimeCorrect(outputComplete, period=period, instrument=instrument) return outputComplete