def getFh_intraday(symbol,
start=None,
end=None,
minutes=5,
showUrl=True,
key=None):
'''
Common interface for apiChooser.
:params start: Time string or naive pandas timestamp or naive datetime object.
:params end: Time string or naive pandas timestamp or naive datetime object.
'''
if getLimitReached('fh'):
msg = 'Finnhub limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= Called Finnhub -- no practical limit, 60/minute =======')
base = 'https://finnhub.io/api/v1/stock/candle?'
start = getDaTime(start, isStart=True)
end = getDaTime(end, isStart=False)
if not isinstance(minutes, int):
minutes = 60
resolution = ni(minutes)
rstart, rend = getStartForRequest(start, end, minutes)
meta, j = fh_intraday(base, symbol, rstart, rend, resolution, key=key)
if meta['code'] != 200:
return meta, pd.DataFrame(), None
assert set(['o', 'h', 'l', 'c', 't', 'v', 's']).issubset(set(j.keys()))
if len(j['o']) == 0:
meta['code'] = 666
logging.error('Error-- no data')
return meta, pd.DataFrame(), None
df = getdf(j)
df = resample(df, minutes, resolution)
# remove candles that lack data
df = df[df['open'] > 0]
maDict = movingAverage(df.close, df, start)
meta, df, maDict = trimit(df, maDict, start, end, meta)
return meta, df, maDict
def apiChooserList(self, start, end, api=None):
'''
Given the current list of apis as av, bc, fh and ib, determine if the given api will
likely return data for the given times.
:params start: A datetime object or time stamp indicating the intended start of the chart.
:params end: A datetime object or time stamp indicating the intended end of the chart.
:params api: Param must be one of mav, bc, fh or ib. If given, the return value in
(api, x, x)[0] will reflect the bool result of the api
:return: (bool, rulesviolated, suggestedStocks) The first entry is only valid if api is
an argument.
'''
start = pd.Timestamp(start)
end = pd.Timestamp(end)
# Need a naive time showing NewYorkTime right now
ne = pd.Timestamp.now("US/Eastern")
n = pd.Timestamp(ne.year, ne.month, ne.day, ne.hour, ne.minute,
ne.second)
violatedRules = []
suggestedApis = self.getPreferences()
if len(suggestedApis) == 0:
self.api = None
return (False, ['No stock Api is selected'], [])
# nopen = dt.datetime(n.year, n.month, n.day, 9, 30)
nclose = dt.datetime(n.year, n.month, n.day, 16, 30)
# Rule 1 Barchart will not return todays data till 16:30
# Rule 1a Barchart will not return yesterdays data after 12 till 1630
tradeday = pd.Timestamp(start.year, start.month, start.day)
todayday = pd.Timestamp(n.year, n.month, n.day)
yday = todayday - pd.Timedelta(days=1)
y = pd.Timestamp(yday.year, yday.month, yday.day, 11, 59)
if tradeday == todayday and n < nclose and 'bc' in suggestedApis:
suggestedApis.remove('bc')
violatedRules.append(
'Barchart free data will not return todays data till 16:30')
if tradeday == yday and end > y and n < nclose and 'bc' in suggestedApis:
suggestedApis.remove('bc')
violatedRules.append(
'Barchart free data will not yesterdays data after 12 till today at 16:30'
)
# Rule 2 No support any charts greater than 7 days prior to today for Alphavantage
# Rule 2 No support any charts greated than 7 days prior to tody for World Trade Data
# Rule 2 No support any charts greater than 30 days for Barchart
if n > start:
delt = n - start
if delt.days > 31 and 'bc' in suggestedApis:
suggestedApis.remove('bc')
lastday = n - pd.Timedelta(days=31)
violatedRules.append(
'Barchart data before {} is unavailable.'.format(
lastday.strftime("%b %d")))
if delt.days > 6 and 'av' in suggestedApis:
suggestedApis.remove('av')
lastday = n - pd.Timedelta(days=6)
violatedRules.append(
'AlphaVantage data before {} is unavailable.'.format(
lastday.strftime("%b %d")))
# Rule 3 Don't call ib if the library is not installed
# Rule 4 Don't call ib if its not connected
if self.apiset.value('gotibapi', type=bool):
if 'ib' in suggestedApis and not ib.isConnected():
suggestedApis.remove('ib')
violatedRules.append('IBAPI is not connected.')
elif 'ib' in suggestedApis:
suggestedApis.remove('ib')
violatedRules.append('IBAPI is not installed')
# Rule 5 No data is available for the future
if start > n:
suggestedApis = []
violatedRules.append('No data is available for the future.')
# Rule No 6 Don't call barchart if there is no apikey in settings
# Rule No 6 Don't call WorldTradeDate if there is no apikey in settings
# Rule No 6 Don't call alphavantage if there is no apikey in settings
# Rule No 6 Don't call finnhub if there is no api key in settings
# mk = ManageKeys()
# bc_key = mk.getKey('bc')
# av_key = mk.getKey('av')
# fh_key = mk.getKey('fh')
if not self.keydict['bc'] and 'bc' in suggestedApis:
suggestedApis.remove('bc')
violatedRules.append(
'There is no apikey in the database for barchart')
if not self.keydict['av'] and 'av' in suggestedApis:
suggestedApis.remove('av')
violatedRules.append(
'There is no apikey in the database for alphavantage')
if not self.keydict['fh'] and 'fh' in suggestedApis:
suggestedApis.remove('fh')
violatedRules.append(
'There is no apikey in the database for finnhub')
# Rule No 7 API limit has been reached [bc, av, fh]
deleteme = []
for token in suggestedApis:
if token == 'ib' or token is None:
continue
if getLimitReached(token, self.apiset):
deleteme.append(token)
violatedRules.append(
f'You have reached your quota for {token}')
for token in deleteme:
suggestedApis.remove(token)
api = api in suggestedApis if api else False
self.api = suggestedApis[0] if suggestedApis else None
self.violatedRules = violatedRules
return (api, violatedRules, suggestedApis)
def getmav_intraday(symbol,
start=None,
end=None,
minutes=None,
showUrl=False,
key=None):
'''
Limited to getting minute data intended to chart day trades. Note that start and end are not
sent to the api request.
:params symb: The stock ticker
:params start: A date time string or datetime object to indicate the beginning time.
:params end: A datetime string or datetime object to indicate the end time.
:params minutes: An int for the candle time, 5 minute, 15 minute etc. If minutes is not one of
Alphavantage's accepted times, we will resample.
:returns: (status, df, maDict) The DataFrame has minute data indexed by time with columns open, high, low
low, close, volume and indexed by pd timestamp. If not specified, this
will return a weeks data.
'''
if getLimitReached('av'):
msg = 'AlphaVantage limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= Called alpha 500 calls per day limit, 5/minute =======')
start = pd.to_datetime(start) if start else None
end = pd.to_datetime(end) if end else None
if not minutes:
minutes = 1
original_minutes = minutes
resamp, (minutes, interval, original_minutes) = ni(minutes)
params = {}
params['function'] = FUNCTION['intraday']
if minutes:
params['interval'] = minutes
params['symbol'] = symbol
params['outputsize'] = 'full'
params['datatype'] = DATATYPES[0]
# params['apikey'] = APIKEY
params['apikey'] = key if key else getKey()
request_url = f"{BASE_URL}"
response = requests.get(request_url, params=params)
if showUrl:
logging.info(response.url)
if response.status_code != 200:
raise Exception(
f"{response.status_code}: {response.content.decode('utf-8')}")
result = response.json()
keys = list(result.keys())
msg = f'{keys[0]}: {result[keys[0]]}'
metaj = {'code': 200, 'message': msg}
if len(keys) == 1:
d = pd.Timestamp.now()
dd = pd.Timestamp(d.year, d.month, d.day, d.hour, d.minute + 2,
d.second)
setLimitReached('av', dd)
logging.warning(msg)
return metaj, pd.DataFrame(), None
dataJson = result[keys[1]]
df = pd.DataFrame(dataJson).T
df.index = pd.to_datetime(df.index)
if df.index[0] > df.index[-1]:
df.sort_index(inplace=True)
if end:
if end < df.index[0]:
msg = 'WARNING: You have requested data that is unavailable:'
msg = msg + f'\nYour end date ({end}) is before the earliest first date ({df.index[0]}).'
logging.warning(msg)
metaj['code'] = 666
metaj['message'] = msg
return metaj, pd.DataFrame(), None
df.rename(columns={
'1. open': 'open',
'2. high': 'high',
'3. low': 'low',
'4. close': 'close',
'5. volume': 'volume'
},
inplace=True)
df.open = pd.to_numeric(df.open)
df.high = pd.to_numeric(df.high)
df.low = pd.to_numeric(df.low)
df.close = pd.to_numeric(df.close)
df.volume = pd.to_numeric(df.volume)
# Alphavantage indexes the candle ends as a time index. So the beginninng of the daay is 9:31
# I think that makes them off by one when processing forward. IB, and others, index candle
# beginnings. To make the APIs harmonious, we will transalte the index time down by
# one interval. I think the translation will always be the interval sent to mav. So
# '1min' will translate down 1 minute etc. We saved the translation as a return
# from ni().
# for i, row in df.iterrows():
delt = pd.Timedelta(minutes=interval)
df.index = df.index - delt
# df.index[i] = df.index[i] - delt
if resamp:
srate = f'{original_minutes}T'
df_ohlc = df[['open']].resample(srate).first()
df_ohlc['high'] = df[['high']].resample(srate).max()
df_ohlc['low'] = df[['low']].resample(srate).min()
df_ohlc['close'] = df[['close']].resample(srate).last()
df_ohlc['volume'] = df[['volume']].resample(srate).sum()
df = df_ohlc.copy()
maDict = movingAverage(df.close, df, start)
# Trim the data to the requested time frame. If we slice it all off set status message and return
if start:
# Remove preemarket hours from the start variable
starttime = start.time()
opening = dt.time(9, 30)
if opening > starttime:
start = pd.Timestamp(start.year, start.month, start.day, 9, 30)
if start > df.index[0]:
df = df[df.index >= start]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
if len(df) == 0:
msg = f"\nWARNING: you have sliced off all the data with the end date {start}"
logging.warning(msg)
metaj['code'] = 666
metaj['message'] = msg
return metaj, pd.DataFrame(), maDict
if end:
if end < df.index[-1]:
df = df[df.index <= end]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index <= end]
if len(df) < 1:
msg = f"\nWARNING: you have sliced off all the data with the end date {end}"
logging.warning(msg)
metaj['code'] = 666
metaj['message'] = msg
return metaj, pd.DataFrame(), maDict
# If we don't have a full ma, delete -- Later:, implement a 'delayed start' ma in graphstuff
keys = list(maDict.keys())
for key in keys:
if len(df) != len(maDict[key]):
del maDict[key]
return metaj, df, maDict
def getbc_intraday(symbol,
start=None,
end=None,
minutes=5,
showUrl=False,
key=None):
'''
Note that getHistory will return previous day's prices until 15 minutes after the market
closes. We will generate a warning if our start or end date differ from the date of the
response. Given todays date at 14:00, it will retrive the previous business days stuff.
Given not start parameter, we will return data for the last weekday. Today or earlier.
We will return everything we between start and end. It may be incomplete.
Its now limiting yesterdays data. At 3:00, the latest I get is yesterday
up to 12 noon.
Retrieve candle data measured in minutes as given in the minutes parameter
:params start: A datetime object or time string to indicate the begin time for the data. By
default, start will be set to the most recent weekday at market open.
:params end: A datetime object or time string to indicate the end time for the data
:params minutes: An int for the candle time, 5 minute, 15 minute etc
:return (status, data): A tuple of (status as dictionary, data as a DataFrame ) This status is
seperate from request status_code.
:raise: ValueError if response.status_code is not 200.
'''
if getLimitReached('bc'):
msg = 'BarChart limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= Called Barchart -- 150 call limit, data available after market close ======='
)
if not end:
tdy = dt.datetime.today()
end = dt.datetime(tdy.year, tdy.month, tdy.day, 17, 0)
# end
if not start:
tdy = dt.datetime.today()
start = dt.datetime(tdy.year, tdy.month, tdy.day, 6, 0)
start = getLastWorkDay(start)
end = pd.to_datetime(end)
start = pd.to_datetime(start)
# startDay = start.strftime("%Y%m%d")
# Get the maximum data in order to set the 200 MA on a 60 minute chart
fullstart = pd.Timestamp.today()
fullstart = fullstart - pd.Timedelta(days=40)
fullstart = fullstart.strftime("%Y%m%d")
params = setParams(symbol, minutes, fullstart, key=key)
response = requests.get(BASE_URL, params=params)
if showUrl:
logging.info(response.url)
if response.status_code != 200:
raise Exception(
f"{response.status_code}: {response.content.decode('utf-8')}")
meta = {'code': 200}
if (response.text and isinstance(response.text, str)
and response.text.startswith('You have reached')):
d = pd.Timestamp.now()
dd = pd.Timestamp(d.year, d.month, d.day + 1, 3, 0, 0)
setLimitReached('bc', dd)
logging.warning(f'API max queries: {response.text}')
meta['message'] = response.text
return meta, pd.DataFrame(), None
result = response.json()
if not result['results']:
logging.warning(
'''Failed to retrieve any data. Barchart sends the following greeting: {result['status']}'''
)
return result['status'], pd.DataFrame(), None
meta['message'] = result['status']['message']
df = pd.DataFrame(result['results'])
for i, row in df.iterrows():
d = pd.Timestamp(row['timestamp'])
newd = pd.Timestamp(d.year, d.month, d.day, d.hour, d.minute, d.second)
df.at[i, 'timestamp'] = newd
df.set_index(df.timestamp, inplace=True)
df.index.rename('date', inplace=True)
maDict = movingAverage(df.close, df, start)
if start > df.index[0]:
rstart = df.index[0]
rend = df.index[-1]
df = df.loc[df.index >= start]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg = msg + f'\nThe Requested start was({start}).'
msg = msg + f'\nBarchart returned data beginning {rstart} and ending {rend}'
msg += '''If you are seeking a chart from today, its possible Barchart has not made'''
msg += 'the data available yet. (Should be available by 4:45PM but they are occasionally late)'
msg += 'You can copy the image yourself, wait, or try a different API. Open File->StockAPI'
logging.warning(msg)
meta['code2'] = 199
meta['message'] = meta['message'] + msg
return meta, df, maDict
if end < df.index[-1]:
df = df.loc[df.index <= end]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index <= end]
# If we just sliced off all our data. Set warning message
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg = msg + f'\nThe Requested end was({end}).'
meta['code2'] = 199
meta['message'] = meta['message'] + msg
logging.warning(f'{meta}')
return meta, df, maDict
deleteMe = list()
for key in maDict:
if key == 'vwap':
continue
if len(df) != len(maDict[key]):
deleteMe.append(key)
for key in deleteMe:
del maDict[key]
# Note we are dropping columns ['symbol', 'timestamp', 'tradingDay[] in favor of ohlcv
df = df[['open', 'high', 'low', 'close', 'volume']].copy(deep=True)
return meta, df, maDict
def getWTD_intraday(symbol, start=None, end=None, minutes=5, showUrl=False):
'''
Implement the interface to retrieve intraday data. showUrl is not part of this api.
Note that the requested range will always be the maximum to get the maximum size MA.
:symbol: The ticker symbol
:start: Timestamp or time string. The beginning of requested data
:end: Timestamp or time string. The end of requested data
:minutes: int between 1 and 60
:showUrl: not used
:return: meta, df, maDict
:depends: On the settings of 'zero_substance/chart' for moving average settings
'''
# Intraday base differs from others. It has the intrday subdomain instead of api subdomain
if getLimitReached('wtd'):
msg = 'World Trade Data limit was reached'
logging.info(msg)
return {'code': 666, 'message': msg}, pd.DataFrame(), None
logging.info(
'======= called WorldTradingData. 25 calls per day limit =======')
base = 'https://intraday.worldtradingdata.com/api/v1/intraday?'
original_minutes = minutes
if not isinstance(original_minutes, int):
original_minutes = 5
minutes = ni(minutes)
if not isinstance(minutes, int) or minutes < 0 or minutes > 60:
raise ValueError(
'Only candle intervals between 1 and 60 are supported')
if not start:
tdy = pd.Timestamp.now()
tdy = getLastWorkDay(tdy)
start = pd.Timestamp(tdy.year, tdy.month, tdy.day, 9, 25)
else:
start = pd.Timestamp(start)
if not end:
tdy = pd.Timestamp.now()
tdy = getLastWorkDay(tdy)
end = pd.Timestamp(tdy.year, tdy.month, tdy.day, 16, 5)
else:
end = pd.Timestamp(end)
# Retrieve the maximum data to get the longest possible moving averages
daRange = 30
if minutes == 1:
daRange = 7
params = getParams(symbol, minutes, daRange)
response = requests.get(base, params=params)
meta = {'code': response.status_code}
if response.status_code != 200:
meta = {'code': 666, 'message': response.text}
return meta, pd.DataFrame(), None
result = response.json()
if 'intraday' not in result.keys():
if 'message' in result.keys():
d = pd.Timestamp.now()
dd = pd.Timestamp(d.year, d.month, d.day + 1, 3, 0, 0)
setLimitReached('wtd', dd)
logging.warning(
f"WorldTradeData limit reached: {result['message']}")
meta['message'] = result['message']
else:
meta['message'] = 'Failed to retrieve data from WorldTradingData'
return meta, pd.DataFrame(), None
if result['timezone_name'] != 'America/New_York':
msg = f'''Time zone returned a non EST zone: {result['timezone_name']}'''
raise ValueError(msg)
meta['message'] = result['symbol'] + ': ' + result[
'stock_exchange_short'] + ': ' + result['timezone_name']
df = pd.DataFrame(data=result['intraday'].values(),
index=result['intraday'].keys())
df.open = pd.to_numeric(df.open)
df.high = pd.to_numeric(df.high)
df.low = pd.to_numeric(df.low)
df.close = pd.to_numeric(df.close)
df.volume = pd.to_numeric(df.volume)
df.index = pd.to_datetime(df.index)
# resample for requested interval if necessary
if original_minutes != minutes:
srate = f'{original_minutes}T'
df_ohlc = df[['open']].resample(srate).first()
df_ohlc['high'] = df[['high']].resample(srate).max()
df_ohlc['low'] = df[['low']].resample(srate).min()
df_ohlc['close'] = df[['close']].resample(srate).last()
df_ohlc['volume'] = df[['volume']].resample(srate).sum()
df = df_ohlc.copy()
# Get requested moving averages
maDict = movingAverage(df.close, df, start)
# Trim off times not requested
if start > pd.Timestamp(df.index[0]):
# rstart = df.index[0]
# rend = df.index[-1]
df = df.loc[df.index >= start]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index >= start]
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg += f'\nThe Requested start was({start}).'
meta['code2'] = 199
meta['message'] = meta['message'] + msg
return meta, df, maDict
if end < df.index[-1]:
df = df.loc[df.index <= end]
for ma in maDict:
maDict[ma] = maDict[ma].loc[maDict[ma].index <= end]
# If we just sliced off all our data. Set warning message
lendf = len(df)
if lendf == 0:
msg = '\nWARNING: all data has been removed.'
msg = msg + f'\nThe Requested end was({end}).'
meta['code2'] = 199
meta['message'] = meta['message'] + msg
logging.error(f'{meta}')
return meta, df, maDict
# If we don't have a full ma, delete -- Later:, implement a 'delayed start' ma in graphstuff and remove this stuff
keys = list(maDict.keys())
for key in keys:
if len(df) != len(maDict[key]):
del maDict[key]
# Note we are dropping columns ['symbol', 'timestamp', 'tradingDay[] in favor of ohlcv
df = df.copy(deep=True)
return meta, df, maDict