def getIntraday(self,
ticker,
start,
end,
resolution,
showUrl=False,
key=None):
logging.info(
'======= Called Tiingo -- no practical limit, 500/hour =======')
# hd = TGO_URL_INTRADAY.format(ticker=ticker, sd='2019-01-02', interval="1min", cols="date,close,high,low,open,volume")
hd = TGO_URL_INTRADAY0.format(ticker=ticker)
header = {'Content-Type': 'application/json'}
start = pd.Timestamp(start)
startsent = start - pd.Timedelta(days=14)
end = pd.Timestamp(end)
if resolution < 60:
resolution = str(resolution) + 'min'
else:
resolution = (resolution // 60) + 1
resolution = str(resolution) + "hour"
params = {}
params['startDate'] = startsent.strftime('%Y-%m-%d')
# params['endDate'] =end.strftime('%Y-%m-%d')
params['resampleFreq'] = resolution
params['afterHours'] = 'false' if excludeAfterHours() else 'true'
params['forceFill'] = 'true'
params['format'] = 'json'
# params['token'] = getApiKey
params['columns'] = "date,open,high,low,close,volume"
r = requests.get(hd, params=params, headers=getHeaders())
meta = {'code': r.status_code}
if r.status_code != 200:
meta['message'] = r.content
return meta, pd.DataFrame(), None
result = r.json()
if len(result) == 0:
meta['code'] = 666
logging.error('Error: Tiingo returned no data')
return meta, pd.DataFrame(), None
df = self.getdf(result)
maDict = movingAverage(df.close, df, start)
meta, df, maDict = self.trimit(df, maDict, start, end, meta)
return meta, df, maDict
def getFh_intraday(symbol,
start=None,
end=None,
minutes=5,
showUrl=True,
key=None):
'''
Common interface for apiChooser.
:params start: Time string or naive pandas timestamp or naive datetime object.
:params end: Time string or naive pandas timestamp or naive datetime object.
'''
if getLimitReached('fh'):
msg = 'Finnhub limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= Called Finnhub -- no practical limit, 60/minute =======')
base = 'https://finnhub.io/api/v1/stock/candle?'
start = getDaTime(start, isStart=True)
end = getDaTime(end, isStart=False)
if not isinstance(minutes, int):
minutes = 60
resolution = ni(minutes)
rstart, rend = getStartForRequest(start, end, minutes)
meta, j = fh_intraday(base, symbol, rstart, rend, resolution, key=key)
if meta['code'] != 200:
return meta, pd.DataFrame(), None
assert set(['o', 'h', 'l', 'c', 't', 'v', 's']).issubset(set(j.keys()))
if len(j['o']) == 0:
meta['code'] = 666
logging.error('Error-- no data')
return meta, pd.DataFrame(), None
df = getdf(j)
df = resample(df, minutes, resolution)
# remove candles that lack data
df = df[df['open'] > 0]
maDict = movingAverage(df.close, df, start)
meta, df, maDict = trimit(df, maDict, start, end, meta)
return meta, df, maDict
def getmav_intraday(symbol,
start=None,
end=None,
minutes=None,
showUrl=False,
key=None):
'''
Limited to getting minute data intended to chart day trades. Note that start and end are not
sent to the api request.
:params symb: The stock ticker
:params start: A date time string or datetime object to indicate the beginning time.
:params end: A datetime string or datetime object to indicate the end time.
:params minutes: An int for the candle time, 5 minute, 15 minute etc. If minutes is not one of
Alphavantage's accepted times, we will resample.
:returns: (status, df, maDict) The DataFrame has minute data indexed by time with columns open, high, low
low, close, volume and indexed by pd timestamp. If not specified, this
will return a weeks data.
'''
if getLimitReached('av'):
msg = 'AlphaVantage limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= Called alpha 500 calls per day limit, 5/minute =======')
start = pd.to_datetime(start) if start else None
end = pd.to_datetime(end) if end else None
if not minutes:
minutes = 1
original_minutes = minutes
resamp, (minutes, interval, original_minutes) = ni(minutes)
params = {}
params['function'] = FUNCTION['intraday']
if minutes:
params['interval'] = minutes
params['symbol'] = symbol
params['outputsize'] = 'full'
params['datatype'] = DATATYPES[0]
# params['apikey'] = APIKEY
params['apikey'] = key if key else getKey()
request_url = f"{BASE_URL}"
response = requests.get(request_url, params=params)
if showUrl:
logging.info(response.url)
if response.status_code != 200:
raise Exception(
f"{response.status_code}: {response.content.decode('utf-8')}")
result = response.json()
keys = list(result.keys())
msg = f'{keys[0]}: {result[keys[0]]}'
metaj = {'code': 200, 'message': msg}
if len(keys) == 1:
d = pd.Timestamp.now()
dd = pd.Timestamp(d.year, d.month, d.day, d.hour, d.minute + 2,
d.second)
setLimitReached('av', dd)
logging.warning(msg)
return metaj, pd.DataFrame(), None
dataJson = result[keys[1]]
df = pd.DataFrame(dataJson).T
df.index = pd.to_datetime(df.index)
if df.index[0] > df.index[-1]:
df.sort_index(inplace=True)
if end:
if end < df.index[0]:
msg = 'WARNING: You have requested data that is unavailable:'
msg = msg + f'\nYour end date ({end}) is before the earliest first date ({df.index[0]}).'
logging.warning(msg)
metaj['code'] = 666
metaj['message'] = msg
return metaj, pd.DataFrame(), None
df.rename(columns={
'1. open': 'open',
'2. high': 'high',
'3. low': 'low',
'4. close': 'close',
'5. volume': 'volume'
},
inplace=True)
df.open = pd.to_numeric(df.open)
df.high = pd.to_numeric(df.high)
df.low = pd.to_numeric(df.low)
df.close = pd.to_numeric(df.close)
df.volume = pd.to_numeric(df.volume)
# Alphavantage indexes the candle ends as a time index. So the beginninng of the daay is 9:31
# I think that makes them off by one when processing forward. IB, and others, index candle
# beginnings. To make the APIs harmonious, we will transalte the index time down by
# one interval. I think the translation will always be the interval sent to mav. So
# '1min' will translate down 1 minute etc. We saved the translation as a return
# from ni().
# for i, row in df.iterrows():
delt = pd.Timedelta(minutes=interval)
df.index = df.index - delt
# df.index[i] = df.index[i] - delt
if resamp:
srate = f'{original_minutes}T'
df_ohlc = df[['open']].resample(srate).first()
df_ohlc['high'] = df[['high']].resample(srate).max()
df_ohlc['low'] = df[['low']].resample(srate).min()
df_ohlc['close'] = df[['close']].resample(srate).last()
df_ohlc['volume'] = df[['volume']].resample(srate).sum()
df = df_ohlc.copy()
maDict = movingAverage(df.close, df, start)
# Trim the data to the requested time frame. If we slice it all off set status message and return
if start:
# Remove preemarket hours from the start variable
starttime = start.time()
opening = dt.time(9, 30)
if opening > starttime:
start = pd.Timestamp(start.year, start.month, start.day, 9, 30)
if start > df.index[0]:
df = df[df.index >= start]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
if len(df) == 0:
msg = f"\nWARNING: you have sliced off all the data with the end date {start}"
logging.warning(msg)
metaj['code'] = 666
metaj['message'] = msg
return metaj, pd.DataFrame(), maDict
if end:
if end < df.index[-1]:
df = df[df.index <= end]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index <= end]
if len(df) < 1:
msg = f"\nWARNING: you have sliced off all the data with the end date {end}"
logging.warning(msg)
metaj['code'] = 666
metaj['message'] = msg
return metaj, pd.DataFrame(), maDict
# If we don't have a full ma, delete -- Later:, implement a 'delayed start' ma in graphstuff
keys = list(maDict.keys())
for key in keys:
if len(df) != len(maDict[key]):
del maDict[key]
return metaj, df, maDict
def getib_intraday(symbol, start=None, end=None, minutes=1, showUrl='dummy', key=None):
'''
An interface API to match the other getters. In this case its a substantial
dumbing down of the capabilities to our one specific need. Output will be resampled
if necessary to return a df with intervals 1~60 minutes
:params symbol: The stock to get
:params start: A timedate object or time string for when to start. Defaults to the most recent
weekday at open.
:params end: A timedate object or time string for when to end. Defaults to the most recent biz
day at close
:params minutes: The length of the candle, 1~60 minutes. Defaults to 1 minute
:return (length, df):A DataFrame of the requested stuff and its length
'''
apiset = QSettings('zero_substance/stockapi', 'structjour')
if not apiset.value('gotibapi', type=bool):
return {'message': 'ibapi is not installed', 'code': 666}, pd.DataFrame(), None
logging.info('***** IB *****')
biz = getLastWorkDay()
if not end:
end = pd.Timestamp(biz.year, biz.month, biz.day, 16, 0)
if not start:
start = pd.Timestamp(biz.year, biz.month, biz.day, 9, 30)
start = pd.Timestamp(start)
end = pd.Timestamp(end)
dur = ''
fullstart = end
fullstart = fullstart - pd.Timedelta(days=5)
if start < fullstart:
delt = end - start
fullstart = end - delt
if (end - fullstart).days < 1:
if ((end - fullstart).seconds // 3600) > 8:
dur = '2 D'
else:
dur = f'{(end-fullstart).seconds} S'
elif (end - fullstart).days < 7:
dur = f'{(end-fullstart).days + 1} D'
else:
dur = f'{(end-fullstart).days} D'
# return 0, pd.DataFrame([], [])
# if the end = 9:31 and dur = 3 minutes, ib will retrieve a start of the preceding day @ 15:58
# This is unique behavior in implemeted apis. We will just let ib do whatever and cut off any
# data prior to the requested data. In that we get no after hours, a request for 7 will begin
# at 9:30 (instead of the previous day at 1330)
symb = symbol
(resamp, (interval, minutes, origminutes)) = ni(minutes)
# ib = TestApp(7496, 7878, '127.0.0.1')
# ib = TestApp(4002, 7979, '127.0.0.1')
x = IbSettings()
ibs = x.getIbSettings()
if not ibs:
return 0, pd.DataFrame(), None
ib = TestApp(ibs['port'], ibs['id'], ibs['host'])
df = ib.getHistorical(symb, end=end, dur=dur, interval=interval, exchange='NASDAQ')
lendf = len(df)
if lendf == 0:
return 0, df, None
# Normalize the date to our favorite format
df.index = pd.to_datetime(df.index)
if resamp:
srate = f'{origminutes}T'
df_ohlc = df[['open']].resample(srate).first()
df_ohlc['high'] = df[['high']].resample(srate).max()
df_ohlc['low'] = df[['low']].resample(srate).min()
df_ohlc['close'] = df[['close']].resample(srate).last()
df_ohlc['volume'] = df[['volume']].resample(srate).sum()
df = df_ohlc.copy()
maDict = movingAverage(df.close, df, end)
if start > df.index[0]:
msg = f"Cutting off beginning: {df.index[0]} to begin at {start}"
logging.info(msg)
df = df.loc[df.index >= start]
if not df.high.any():
logging.warning('All data has been removed')
return 0, pd.DataFrame(), None
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
removeMe = list()
for key in maDict:
# VWAP is a reference that begins at market open. If the open trade precedes VWAP
# we will exclude it from the chart. Other possibilities: give VWAP a start time or
# pick an arbitrary premarket time to begin it. The former would be havoc to implement
# the latter probably better but inaccurate; would not reflect what the trader was using
# Better suggestions?
if key == 'vwap':
if len(maDict['vwap']) < 1 or (df.index[0] < maDict['vwap'].index[0]):
removeMe.append(key)
# del maDict['vwap']
else:
if len(df) != len(maDict[key]):
removeMe.append(key)
# del maDict[key]
for key in removeMe:
del maDict[key]
ib.disconnect()
return len(df), df, maDict
def getbc_intraday(symbol,
start=None,
end=None,
minutes=5,
showUrl=False,
key=None):
'''
Note that getHistory will return previous day's prices until 15 minutes after the market
closes. We will generate a warning if our start or end date differ from the date of the
response. Given todays date at 14:00, it will retrive the previous business days stuff.
Given not start parameter, we will return data for the last weekday. Today or earlier.
We will return everything we between start and end. It may be incomplete.
Its now limiting yesterdays data. At 3:00, the latest I get is yesterday
up to 12 noon.
Retrieve candle data measured in minutes as given in the minutes parameter
:params start: A datetime object or time string to indicate the begin time for the data. By
default, start will be set to the most recent weekday at market open.
:params end: A datetime object or time string to indicate the end time for the data
:params minutes: An int for the candle time, 5 minute, 15 minute etc
:return (status, data): A tuple of (status as dictionary, data as a DataFrame ) This status is
seperate from request status_code.
:raise: ValueError if response.status_code is not 200.
'''
if getLimitReached('bc'):
msg = 'BarChart limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= Called Barchart -- 150 call limit, data available after market close ======='
)
if not end:
tdy = dt.datetime.today()
end = dt.datetime(tdy.year, tdy.month, tdy.day, 17, 0)
# end
if not start:
tdy = dt.datetime.today()
start = dt.datetime(tdy.year, tdy.month, tdy.day, 6, 0)
start = getLastWorkDay(start)
end = pd.to_datetime(end)
start = pd.to_datetime(start)
# startDay = start.strftime("%Y%m%d")
# Get the maximum data in order to set the 200 MA on a 60 minute chart
fullstart = pd.Timestamp.today()
fullstart = fullstart - pd.Timedelta(days=40)
fullstart = fullstart.strftime("%Y%m%d")
params = setParams(symbol, minutes, fullstart, key=key)
response = requests.get(BASE_URL, params=params)
if showUrl:
logging.info(response.url)
if response.status_code != 200:
raise Exception(
f"{response.status_code}: {response.content.decode('utf-8')}")
meta = {'code': 200}
if (response.text and isinstance(response.text, str)
and response.text.startswith('You have reached')):
d = pd.Timestamp.now()
dd = pd.Timestamp(d.year, d.month, d.day + 1, 3, 0, 0)
setLimitReached('bc', dd)
logging.warning(f'API max queries: {response.text}')
meta['message'] = response.text
return meta, pd.DataFrame(), None
result = response.json()
if not result['results']:
logging.warning(
'''Failed to retrieve any data. Barchart sends the following greeting: {result['status']}'''
)
return result['status'], pd.DataFrame(), None
meta['message'] = result['status']['message']
df = pd.DataFrame(result['results'])
for i, row in df.iterrows():
d = pd.Timestamp(row['timestamp'])
newd = pd.Timestamp(d.year, d.month, d.day, d.hour, d.minute, d.second)
df.at[i, 'timestamp'] = newd
df.set_index(df.timestamp, inplace=True)
df.index.rename('date', inplace=True)
maDict = movingAverage(df.close, df, start)
if start > df.index[0]:
rstart = df.index[0]
rend = df.index[-1]
df = df.loc[df.index >= start]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg = msg + f'\nThe Requested start was({start}).'
msg = msg + f'\nBarchart returned data beginning {rstart} and ending {rend}'
msg += '''If you are seeking a chart from today, its possible Barchart has not made'''
msg += 'the data available yet. (Should be available by 4:45PM but they are occasionally late)'
msg += 'You can copy the image yourself, wait, or try a different API. Open File->StockAPI'
logging.warning(msg)
meta['code2'] = 199
meta['message'] = meta['message'] + msg
return meta, df, maDict
if end < df.index[-1]:
df = df.loc[df.index <= end]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index <= end]
# If we just sliced off all our data. Set warning message
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg = msg + f'\nThe Requested end was({end}).'
meta['code2'] = 199
meta['message'] = meta['message'] + msg
logging.warning(f'{meta}')
return meta, df, maDict
deleteMe = list()
for key in maDict:
if key == 'vwap':
continue
if len(df) != len(maDict[key]):
deleteMe.append(key)
for key in deleteMe:
del maDict[key]
# Note we are dropping columns ['symbol', 'timestamp', 'tradingDay[] in favor of ohlcv
df = df[['open', 'high', 'low', 'close', 'volume']].copy(deep=True)
return meta, df, maDict
def getWTD_intraday(symbol, start=None, end=None, minutes=5, showUrl=False):
'''
Implement the interface to retrieve intraday data. showUrl is not part of this api.
Note that the requested range will always be the maximum to get the maximum size MA.
:symbol: The ticker symbol
:start: Timestamp or time string. The beginning of requested data
:end: Timestamp or time string. The end of requested data
:minutes: int between 1 and 60
:showUrl: not used
:return: meta, df, maDict
:depends: On the settings of 'zero_substance/chart' for moving average settings
'''
# Intraday base differs from others. It has the intrday subdomain instead of api subdomain
if getLimitReached('wtd'):
msg = 'World Trade Data limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= called WorldTradingData. 25 calls per day limit =======')
base = 'https://intraday.worldtradingdata.com/api/v1/intraday?'
original_minutes = minutes
if not isinstance(original_minutes, int):
original_minutes = 5
minutes = ni(minutes)
if not isinstance(minutes, int) or minutes < 0 or minutes > 60:
raise ValueError(
'Only candle intervals between 1 and 60 are supported')
if not start:
tdy = pd.Timestamp.now()
tdy = getLastWorkDay(tdy)
start = pd.Timestamp(tdy.year, tdy.month, tdy.day, 9, 25)
else:
start = pd.Timestamp(start)
if not end:
tdy = pd.Timestamp.now()
tdy = getLastWorkDay(tdy)
end = pd.Timestamp(tdy.year, tdy.month, tdy.day, 16, 5)
else:
end = pd.Timestamp(end)
# Retrieve the maximum data to get the longest possible moving averages
daRange = 30
if minutes == 1:
daRange = 7
params = getParams(symbol, minutes, daRange)
response = requests.get(base, params=params)
meta = {'code': response.status_code}
if response.status_code != 200:
meta = {'code': 666, 'message': response.text}
return meta, pd.DataFrame(), None
result = response.json()
if 'intraday' not in result.keys():
if 'message' in result.keys():
d = pd.Timestamp.now()
dd = pd.Timestamp(d.year, d.month, d.day + 1, 3, 0, 0)
setLimitReached('wtd', dd)
logging.warning(
f"WorldTradeData limit reached: {result['message']}")
meta['message'] = result['message']
else:
meta['message'] = 'Failed to retrieve data from WorldTradingData'
return meta, pd.DataFrame(), None
if result['timezone_name'] != 'America/New_York':
msg = f'''Time zone returned a non EST zone: {result['timezone_name']}'''
raise ValueError(msg)
meta['message'] = result['symbol'] + ': ' + result[
'stock_exchange_short'] + ': ' + result['timezone_name']
df = pd.DataFrame(data=result['intraday'].values(),
index=result['intraday'].keys())
df.open = pd.to_numeric(df.open)
df.high = pd.to_numeric(df.high)
df.low = pd.to_numeric(df.low)
df.close = pd.to_numeric(df.close)
df.volume = pd.to_numeric(df.volume)
df.index = pd.to_datetime(df.index)
# resample for requested interval if necessary
if original_minutes != minutes:
srate = f'{original_minutes}T'
df_ohlc = df[['open']].resample(srate).first()
df_ohlc['high'] = df[['high']].resample(srate).max()
df_ohlc['low'] = df[['low']].resample(srate).min()
df_ohlc['close'] = df[['close']].resample(srate).last()
df_ohlc['volume'] = df[['volume']].resample(srate).sum()
df = df_ohlc.copy()
# Get requested moving averages
maDict = movingAverage(df.close, df, start)
# Trim off times not requested
if start > pd.Timestamp(df.index[0]):
# rstart = df.index[0]
# rend = df.index[-1]
df = df.loc[df.index >= start]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg += f'\nThe Requested start was({start}).'
meta['code2'] = 199
meta['message'] = meta['message'] + msg
return meta, df, maDict
if end < df.index[-1]:
df = df.loc[df.index <= end]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index <= end]
# If we just sliced off all our data. Set warning message
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg = msg + f'\nThe Requested end was({end}).'
meta['code2'] = 199
meta['message'] = meta['message'] + msg
logging.error(f'{meta}')
return meta, df, maDict
# If we don't have a full ma, delete -- Later:, implement a 'delayed start' ma in graphstuff and remove this stuff
keys = list(maDict.keys())
for key in keys:
if len(df) != len(maDict[key]):
del maDict[key]
# Note we are dropping columns ['symbol', 'timestamp', 'tradingDay[] in favor of ohlcv
df = df.copy(deep=True)
return meta, df, maDict