def getIntraday(self, ticker, start, end, resolution, showUrl=False, key=None): logging.info( '======= Called Tiingo -- no practical limit, 500/hour =======') # hd = TGO_URL_INTRADAY.format(ticker=ticker, sd='2019-01-02', interval="1min", cols="date,close,high,low,open,volume") hd = TGO_URL_INTRADAY0.format(ticker=ticker) header = {'Content-Type': 'application/json'} start = pd.Timestamp(start) startsent = start - pd.Timedelta(days=14) end = pd.Timestamp(end) if resolution < 60: resolution = str(resolution) + 'min' else: resolution = (resolution // 60) + 1 resolution = str(resolution) + "hour" params = {} params['startDate'] = startsent.strftime('%Y-%m-%d') # params['endDate'] =end.strftime('%Y-%m-%d') params['resampleFreq'] = resolution params['afterHours'] = 'false' if excludeAfterHours() else 'true' params['forceFill'] = 'true' params['format'] = 'json' # params['token'] = getApiKey params['columns'] = "date,open,high,low,close,volume" r = requests.get(hd, params=params, headers=getHeaders()) meta = {'code': r.status_code} if r.status_code != 200: meta['message'] = r.content return meta, pd.DataFrame(), None result = r.json() if len(result) == 0: meta['code'] = 666 logging.error('Error: Tiingo returned no data') return meta, pd.DataFrame(), None df = self.getdf(result) maDict = movingAverage(df.close, df, start) meta, df, maDict = self.trimit(df, maDict, start, end, meta) return meta, df, maDict
def getFh_intraday(symbol, start=None, end=None, minutes=5, showUrl=True, key=None): ''' Common interface for apiChooser. :params start: Time string or naive pandas timestamp or naive datetime object. :params end: Time string or naive pandas timestamp or naive datetime object. ''' if getLimitReached('fh'): msg = 'Finnhub limit was reached' logging.info(msg) return {'code': 666, 'message': msg}, pd.DataFrame(), None logging.info( '======= Called Finnhub -- no practical limit, 60/minute =======') base = 'https://finnhub.io/api/v1/stock/candle?' start = getDaTime(start, isStart=True) end = getDaTime(end, isStart=False) if not isinstance(minutes, int): minutes = 60 resolution = ni(minutes) rstart, rend = getStartForRequest(start, end, minutes) meta, j = fh_intraday(base, symbol, rstart, rend, resolution, key=key) if meta['code'] != 200: return meta, pd.DataFrame(), None assert set(['o', 'h', 'l', 'c', 't', 'v', 's']).issubset(set(j.keys())) if len(j['o']) == 0: meta['code'] = 666 logging.error('Error-- no data') return meta, pd.DataFrame(), None df = getdf(j) df = resample(df, minutes, resolution) # remove candles that lack data df = df[df['open'] > 0] maDict = movingAverage(df.close, df, start) meta, df, maDict = trimit(df, maDict, start, end, meta) return meta, df, maDict
def getmav_intraday(symbol, start=None, end=None, minutes=None, showUrl=False, key=None): ''' Limited to getting minute data intended to chart day trades. Note that start and end are not sent to the api request. :params symb: The stock ticker :params start: A date time string or datetime object to indicate the beginning time. :params end: A datetime string or datetime object to indicate the end time. :params minutes: An int for the candle time, 5 minute, 15 minute etc. If minutes is not one of Alphavantage's accepted times, we will resample. :returns: (status, df, maDict) The DataFrame has minute data indexed by time with columns open, high, low low, close, volume and indexed by pd timestamp. If not specified, this will return a weeks data. ''' if getLimitReached('av'): msg = 'AlphaVantage limit was reached' logging.info(msg) return {'code': 666, 'message': msg}, pd.DataFrame(), None logging.info( '======= Called alpha 500 calls per day limit, 5/minute =======') start = pd.to_datetime(start) if start else None end = pd.to_datetime(end) if end else None if not minutes: minutes = 1 original_minutes = minutes resamp, (minutes, interval, original_minutes) = ni(minutes) params = {} params['function'] = FUNCTION['intraday'] if minutes: params['interval'] = minutes params['symbol'] = symbol params['outputsize'] = 'full' params['datatype'] = DATATYPES[0] # params['apikey'] = APIKEY params['apikey'] = key if key else getKey() request_url = f"{BASE_URL}" response = requests.get(request_url, params=params) if showUrl: logging.info(response.url) if response.status_code != 200: raise Exception( f"{response.status_code}: {response.content.decode('utf-8')}") result = response.json() keys = list(result.keys()) msg = f'{keys[0]}: {result[keys[0]]}' metaj = {'code': 200, 'message': msg} if len(keys) == 1: d = pd.Timestamp.now() dd = pd.Timestamp(d.year, d.month, d.day, d.hour, d.minute + 2, d.second) setLimitReached('av', dd) logging.warning(msg) return metaj, pd.DataFrame(), None dataJson = result[keys[1]] df = pd.DataFrame(dataJson).T df.index = pd.to_datetime(df.index) if df.index[0] > df.index[-1]: df.sort_index(inplace=True) if end: if end < df.index[0]: msg = 'WARNING: You have requested data that is unavailable:' msg = msg + f'\nYour end date ({end}) is before the earliest first date ({df.index[0]}).' logging.warning(msg) metaj['code'] = 666 metaj['message'] = msg return metaj, pd.DataFrame(), None df.rename(columns={ '1. open': 'open', '2. high': 'high', '3. low': 'low', '4. close': 'close', '5. volume': 'volume' }, inplace=True) df.open = pd.to_numeric(df.open) df.high = pd.to_numeric(df.high) df.low = pd.to_numeric(df.low) df.close = pd.to_numeric(df.close) df.volume = pd.to_numeric(df.volume) # Alphavantage indexes the candle ends as a time index. So the beginninng of the daay is 9:31 # I think that makes them off by one when processing forward. IB, and others, index candle # beginnings. To make the APIs harmonious, we will transalte the index time down by # one interval. I think the translation will always be the interval sent to mav. So # '1min' will translate down 1 minute etc. We saved the translation as a return # from ni(). # for i, row in df.iterrows(): delt = pd.Timedelta(minutes=interval) df.index = df.index - delt # df.index[i] = df.index[i] - delt if resamp: srate = f'{original_minutes}T' df_ohlc = df[['open']].resample(srate).first() df_ohlc['high'] = df[['high']].resample(srate).max() df_ohlc['low'] = df[['low']].resample(srate).min() df_ohlc['close'] = df[['close']].resample(srate).last() df_ohlc['volume'] = df[['volume']].resample(srate).sum() df = df_ohlc.copy() maDict = movingAverage(df.close, df, start) # Trim the data to the requested time frame. If we slice it all off set status message and return if start: # Remove preemarket hours from the start variable starttime = start.time() opening = dt.time(9, 30) if opening > starttime: start = pd.Timestamp(start.year, start.month, start.day, 9, 30) if start > df.index[0]: df = df[df.index >= start] for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index >= start] if len(df) == 0: msg = f"\nWARNING: you have sliced off all the data with the end date {start}" logging.warning(msg) metaj['code'] = 666 metaj['message'] = msg return metaj, pd.DataFrame(), maDict if end: if end < df.index[-1]: df = df[df.index <= end] for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index <= end] if len(df) < 1: msg = f"\nWARNING: you have sliced off all the data with the end date {end}" logging.warning(msg) metaj['code'] = 666 metaj['message'] = msg return metaj, pd.DataFrame(), maDict # If we don't have a full ma, delete -- Later:, implement a 'delayed start' ma in graphstuff keys = list(maDict.keys()) for key in keys: if len(df) != len(maDict[key]): del maDict[key] return metaj, df, maDict
def getib_intraday(symbol, start=None, end=None, minutes=1, showUrl='dummy', key=None): ''' An interface API to match the other getters. In this case its a substantial dumbing down of the capabilities to our one specific need. Output will be resampled if necessary to return a df with intervals 1~60 minutes :params symbol: The stock to get :params start: A timedate object or time string for when to start. Defaults to the most recent weekday at open. :params end: A timedate object or time string for when to end. Defaults to the most recent biz day at close :params minutes: The length of the candle, 1~60 minutes. Defaults to 1 minute :return (length, df):A DataFrame of the requested stuff and its length ''' apiset = QSettings('zero_substance/stockapi', 'structjour') if not apiset.value('gotibapi', type=bool): return {'message': 'ibapi is not installed', 'code': 666}, pd.DataFrame(), None logging.info('***** IB *****') biz = getLastWorkDay() if not end: end = pd.Timestamp(biz.year, biz.month, biz.day, 16, 0) if not start: start = pd.Timestamp(biz.year, biz.month, biz.day, 9, 30) start = pd.Timestamp(start) end = pd.Timestamp(end) dur = '' fullstart = end fullstart = fullstart - pd.Timedelta(days=5) if start < fullstart: delt = end - start fullstart = end - delt if (end - fullstart).days < 1: if ((end - fullstart).seconds // 3600) > 8: dur = '2 D' else: dur = f'{(end-fullstart).seconds} S' elif (end - fullstart).days < 7: dur = f'{(end-fullstart).days + 1} D' else: dur = f'{(end-fullstart).days} D' # return 0, pd.DataFrame([], []) # if the end = 9:31 and dur = 3 minutes, ib will retrieve a start of the preceding day @ 15:58 # This is unique behavior in implemeted apis. We will just let ib do whatever and cut off any # data prior to the requested data. In that we get no after hours, a request for 7 will begin # at 9:30 (instead of the previous day at 1330) symb = symbol (resamp, (interval, minutes, origminutes)) = ni(minutes) # ib = TestApp(7496, 7878, '127.0.0.1') # ib = TestApp(4002, 7979, '127.0.0.1') x = IbSettings() ibs = x.getIbSettings() if not ibs: return 0, pd.DataFrame(), None ib = TestApp(ibs['port'], ibs['id'], ibs['host']) df = ib.getHistorical(symb, end=end, dur=dur, interval=interval, exchange='NASDAQ') lendf = len(df) if lendf == 0: return 0, df, None # Normalize the date to our favorite format df.index = pd.to_datetime(df.index) if resamp: srate = f'{origminutes}T' df_ohlc = df[['open']].resample(srate).first() df_ohlc['high'] = df[['high']].resample(srate).max() df_ohlc['low'] = df[['low']].resample(srate).min() df_ohlc['close'] = df[['close']].resample(srate).last() df_ohlc['volume'] = df[['volume']].resample(srate).sum() df = df_ohlc.copy() maDict = movingAverage(df.close, df, end) if start > df.index[0]: msg = f"Cutting off beginning: {df.index[0]} to begin at {start}" logging.info(msg) df = df.loc[df.index >= start] if not df.high.any(): logging.warning('All data has been removed') return 0, pd.DataFrame(), None for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index >= start] removeMe = list() for key in maDict: # VWAP is a reference that begins at market open. If the open trade precedes VWAP # we will exclude it from the chart. Other possibilities: give VWAP a start time or # pick an arbitrary premarket time to begin it. The former would be havoc to implement # the latter probably better but inaccurate; would not reflect what the trader was using # Better suggestions? if key == 'vwap': if len(maDict['vwap']) < 1 or (df.index[0] < maDict['vwap'].index[0]): removeMe.append(key) # del maDict['vwap'] else: if len(df) != len(maDict[key]): removeMe.append(key) # del maDict[key] for key in removeMe: del maDict[key] ib.disconnect() return len(df), df, maDict
def getbc_intraday(symbol, start=None, end=None, minutes=5, showUrl=False, key=None): ''' Note that getHistory will return previous day's prices until 15 minutes after the market closes. We will generate a warning if our start or end date differ from the date of the response. Given todays date at 14:00, it will retrive the previous business days stuff. Given not start parameter, we will return data for the last weekday. Today or earlier. We will return everything we between start and end. It may be incomplete. Its now limiting yesterdays data. At 3:00, the latest I get is yesterday up to 12 noon. Retrieve candle data measured in minutes as given in the minutes parameter :params start: A datetime object or time string to indicate the begin time for the data. By default, start will be set to the most recent weekday at market open. :params end: A datetime object or time string to indicate the end time for the data :params minutes: An int for the candle time, 5 minute, 15 minute etc :return (status, data): A tuple of (status as dictionary, data as a DataFrame ) This status is seperate from request status_code. :raise: ValueError if response.status_code is not 200. ''' if getLimitReached('bc'): msg = 'BarChart limit was reached' logging.info(msg) return {'code': 666, 'message': msg}, pd.DataFrame(), None logging.info( '======= Called Barchart -- 150 call limit, data available after market close =======' ) if not end: tdy = dt.datetime.today() end = dt.datetime(tdy.year, tdy.month, tdy.day, 17, 0) # end if not start: tdy = dt.datetime.today() start = dt.datetime(tdy.year, tdy.month, tdy.day, 6, 0) start = getLastWorkDay(start) end = pd.to_datetime(end) start = pd.to_datetime(start) # startDay = start.strftime("%Y%m%d") # Get the maximum data in order to set the 200 MA on a 60 minute chart fullstart = pd.Timestamp.today() fullstart = fullstart - pd.Timedelta(days=40) fullstart = fullstart.strftime("%Y%m%d") params = setParams(symbol, minutes, fullstart, key=key) response = requests.get(BASE_URL, params=params) if showUrl: logging.info(response.url) if response.status_code != 200: raise Exception( f"{response.status_code}: {response.content.decode('utf-8')}") meta = {'code': 200} if (response.text and isinstance(response.text, str) and response.text.startswith('You have reached')): d = pd.Timestamp.now() dd = pd.Timestamp(d.year, d.month, d.day + 1, 3, 0, 0) setLimitReached('bc', dd) logging.warning(f'API max queries: {response.text}') meta['message'] = response.text return meta, pd.DataFrame(), None result = response.json() if not result['results']: logging.warning( '''Failed to retrieve any data. Barchart sends the following greeting: {result['status']}''' ) return result['status'], pd.DataFrame(), None meta['message'] = result['status']['message'] df = pd.DataFrame(result['results']) for i, row in df.iterrows(): d = pd.Timestamp(row['timestamp']) newd = pd.Timestamp(d.year, d.month, d.day, d.hour, d.minute, d.second) df.at[i, 'timestamp'] = newd df.set_index(df.timestamp, inplace=True) df.index.rename('date', inplace=True) maDict = movingAverage(df.close, df, start) if start > df.index[0]: rstart = df.index[0] rend = df.index[-1] df = df.loc[df.index >= start] for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index >= start] lendf = len(df) if lendf == 0: msg = '\nWARNING: all data has been removed.' msg = msg + f'\nThe Requested start was({start}).' msg = msg + f'\nBarchart returned data beginning {rstart} and ending {rend}' msg += '''If you are seeking a chart from today, its possible Barchart has not made''' msg += 'the data available yet. (Should be available by 4:45PM but they are occasionally late)' msg += 'You can copy the image yourself, wait, or try a different API. Open File->StockAPI' logging.warning(msg) meta['code2'] = 199 meta['message'] = meta['message'] + msg return meta, df, maDict if end < df.index[-1]: df = df.loc[df.index <= end] for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index <= end] # If we just sliced off all our data. Set warning message lendf = len(df) if lendf == 0: msg = '\nWARNING: all data has been removed.' msg = msg + f'\nThe Requested end was({end}).' meta['code2'] = 199 meta['message'] = meta['message'] + msg logging.warning(f'{meta}') return meta, df, maDict deleteMe = list() for key in maDict: if key == 'vwap': continue if len(df) != len(maDict[key]): deleteMe.append(key) for key in deleteMe: del maDict[key] # Note we are dropping columns ['symbol', 'timestamp', 'tradingDay[] in favor of ohlcv df = df[['open', 'high', 'low', 'close', 'volume']].copy(deep=True) return meta, df, maDict
def getWTD_intraday(symbol, start=None, end=None, minutes=5, showUrl=False): ''' Implement the interface to retrieve intraday data. showUrl is not part of this api. Note that the requested range will always be the maximum to get the maximum size MA. :symbol: The ticker symbol :start: Timestamp or time string. The beginning of requested data :end: Timestamp or time string. The end of requested data :minutes: int between 1 and 60 :showUrl: not used :return: meta, df, maDict :depends: On the settings of 'zero_substance/chart' for moving average settings ''' # Intraday base differs from others. It has the intrday subdomain instead of api subdomain if getLimitReached('wtd'): msg = 'World Trade Data limit was reached' logging.info(msg) return {'code': 666, 'message': msg}, pd.DataFrame(), None logging.info( '======= called WorldTradingData. 25 calls per day limit =======') base = 'https://intraday.worldtradingdata.com/api/v1/intraday?' original_minutes = minutes if not isinstance(original_minutes, int): original_minutes = 5 minutes = ni(minutes) if not isinstance(minutes, int) or minutes < 0 or minutes > 60: raise ValueError( 'Only candle intervals between 1 and 60 are supported') if not start: tdy = pd.Timestamp.now() tdy = getLastWorkDay(tdy) start = pd.Timestamp(tdy.year, tdy.month, tdy.day, 9, 25) else: start = pd.Timestamp(start) if not end: tdy = pd.Timestamp.now() tdy = getLastWorkDay(tdy) end = pd.Timestamp(tdy.year, tdy.month, tdy.day, 16, 5) else: end = pd.Timestamp(end) # Retrieve the maximum data to get the longest possible moving averages daRange = 30 if minutes == 1: daRange = 7 params = getParams(symbol, minutes, daRange) response = requests.get(base, params=params) meta = {'code': response.status_code} if response.status_code != 200: meta = {'code': 666, 'message': response.text} return meta, pd.DataFrame(), None result = response.json() if 'intraday' not in result.keys(): if 'message' in result.keys(): d = pd.Timestamp.now() dd = pd.Timestamp(d.year, d.month, d.day + 1, 3, 0, 0) setLimitReached('wtd', dd) logging.warning( f"WorldTradeData limit reached: {result['message']}") meta['message'] = result['message'] else: meta['message'] = 'Failed to retrieve data from WorldTradingData' return meta, pd.DataFrame(), None if result['timezone_name'] != 'America/New_York': msg = f'''Time zone returned a non EST zone: {result['timezone_name']}''' raise ValueError(msg) meta['message'] = result['symbol'] + ': ' + result[ 'stock_exchange_short'] + ': ' + result['timezone_name'] df = pd.DataFrame(data=result['intraday'].values(), index=result['intraday'].keys()) df.open = pd.to_numeric(df.open) df.high = pd.to_numeric(df.high) df.low = pd.to_numeric(df.low) df.close = pd.to_numeric(df.close) df.volume = pd.to_numeric(df.volume) df.index = pd.to_datetime(df.index) # resample for requested interval if necessary if original_minutes != minutes: srate = f'{original_minutes}T' df_ohlc = df[['open']].resample(srate).first() df_ohlc['high'] = df[['high']].resample(srate).max() df_ohlc['low'] = df[['low']].resample(srate).min() df_ohlc['close'] = df[['close']].resample(srate).last() df_ohlc['volume'] = df[['volume']].resample(srate).sum() df = df_ohlc.copy() # Get requested moving averages maDict = movingAverage(df.close, df, start) # Trim off times not requested if start > pd.Timestamp(df.index[0]): # rstart = df.index[0] # rend = df.index[-1] df = df.loc[df.index >= start] for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index >= start] lendf = len(df) if lendf == 0: msg = '\nWARNING: all data has been removed.' msg += f'\nThe Requested start was({start}).' meta['code2'] = 199 meta['message'] = meta['message'] + msg return meta, df, maDict if end < df.index[-1]: df = df.loc[df.index <= end] for ma in maDict: maDict[ma] = maDict[ma].loc[maDict[ma].index <= end] # If we just sliced off all our data. Set warning message lendf = len(df) if lendf == 0: msg = '\nWARNING: all data has been removed.' msg = msg + f'\nThe Requested end was({end}).' meta['code2'] = 199 meta['message'] = meta['message'] + msg logging.error(f'{meta}') return meta, df, maDict # If we don't have a full ma, delete -- Later:, implement a 'delayed start' ma in graphstuff and remove this stuff keys = list(maDict.keys()) for key in keys: if len(df) != len(maDict[key]): del maDict[key] # Note we are dropping columns ['symbol', 'timestamp', 'tradingDay[] in favor of ohlcv df = df.copy(deep=True) return meta, df, maDict