def minute_data(minutes, ticker):
today = datetime.today()
try:
df = get_historical_intraday(ticker, today,
output_format='pandas')['open']
except KeyError:
yesterday = today - timedelta(1)
df = get_historical_intraday(ticker, yesterday,
output_format='pandas')['open']
return list(df.head(minutes))
def get_last_x_minute_data(self, ticker, minutes=0):
today = datetime.today()
yesterday = today - timedelta(1)
try:
df = get_historical_intraday(ticker, today,
output_format='pandas')['close']
except KeyError:
df = get_historical_intraday(ticker,
yesterday,
output_format='pandas')['close']
if minutes:
return list(df.head(minutes))
else:
return list(df)
def get_intraday_data(listname=SIConfig.symbols_list_name):
intraday_dict = dict()
list = SIStockStorage.get_symbol_list(listname)
for stock in list:
df = pd.DataFrame()
today = datetime.today()
days = SIConfig.intradaily_interval
count = 0
while days > count:
print("- DOWNLOAD INTRADAY [" + stock + "]: D-" + str(count))
reference_day = today - timedelta(days=count)
new_data = get_historical_intraday(stock,
date=reference_day,
output_format='pandas')
df = pd.concat([df, new_data])
count = count + 1
df.index = pd.to_datetime(df.index)
intraday_dict[stock] = df
return intraday_dict
def build_data(self):
keep_cols = ['open', 'high', 'low', 'close', 'volume']
data = {}
for sym in self.symbols:
data[sym] = {}
for dt in self.dt_range():
retry = True
while retry:
try:
ret = get_historical_intraday(sym, dt, output_format='pandas')
retry = False
except Exception as e:
print('data download error for ', sym, dt)
print(e)
retry = bool(input('proceed?'))
if len(ret) >= 65:
for c in ret.columns:
if c not in keep_cols:
ret = ret.drop(c, axis=1)
ret = ret.fillna(method='bfill')
print('received data for ', sym, ' - ', dt)
data[sym][dt] = ret.to_dict(orient='index')
else:
print('missing values for ', sym, ' on ', dt)
print('got back:', ret)
return data
def test_intraday_pandas(self):
data = get_historical_intraday("AAPL", output_format='pandas')
assert isinstance(data, pd.DataFrame)
assert isinstance(data.index, pd.DatetimeIndex)
self.verify_timeframe(data)
def get_intraday(ticker):
data = get_historical_intraday(ticker, token=token)
time = []
value = []
for i in range(len(data)):
time.append(data[i]["label"])
value.append(data[i]["close"])
for i in range(len(time)):
pattern = r'^\d+\s\w\w'
result = re.match(pattern, time[i])
if result:
result = result.group(0)
if len(result) == 4:
hour = result[0]
ampm = result[2:]
else:
hour = result[0:2]
ampm = result[3:]
real=hour+':00 '+ampm
time[i] = real
data_frame = pd.DataFrame({"Time":time, "Value":value})
data_frame["Time"] = pd.to_datetime(data_frame["Time"], format='%I:%M %p').dt.time
data_frame['Time'] = data_frame['Time'].astype(str)
return data_frame
def getHistoricalIntradayByMinute(ticker, day=None):
historicalIntradayData = {}
try:
historicalIntradayData = get_historical_intraday(ticker, day)
except Exception as ex:
Logger.error('Failed querying IEX historical intraday data for {}'.format(ticker))
return historicalIntradayData
def get_IEX_minutely(ticker: str, undo_days: int) -> pd.DataFrame:
tuples: List[Tuple[int]] = []
returns: pd.DataFrame
end: dt.datetime = dt.datetime.today()
date_list: List[dt.datetime] = [
end - dt.timedelta(days=d) for d in range(undo_days)
]
for i in range(len(date_list)):
ret = str(np.array(date_list[::-1])[i])[:10]
ret = tuple(map(int, ret.split('-')))
tuples.append(ret)
dataframes: List[pd.DataFrame] = []
df_: pd.DataFrame
dat: dt.datetime
for date_ in tuples:
dat = dt.datetime(*date_)
df_ = get_historical_intraday(ticker, date=dat, output_format='pandas')
dataframes.append(df_)
returns = pd.concat(dataframes, axis=0)
returns = pd.DataFrame({
'High': returns['high'].values,
'Low': returns['low'].values,
'Open': returns['open'].values,
'Close': returns['close'].values,
'Volume': returns['volume'].values
})
return returns.dropna()
def get_day_quotes(self, ticker, timestamp):
"""Collects all quotes from the day of the market timestamp."""
quotes = []
# The timestamp is expected in market time.
response = get_historical_intraday(ticker, timestamp)
if not response:
self.logs.error("Failed to request historical data for %s on %s." %
(ticker, timestamp))
return None
for data in response:
try:
market_time_str = "%s %s" % (data["date"], data["minute"])
try:
market_time = MARKET_TIMEZONE.localize(
datetime.strptime(market_time_str, "%Y-%m-%d %H:%M"))
except ValueError:
self.logs.warn("Failed to decode market time: %s" %
market_time_str)
continue
price = data["average"]
if not price or price < 0:
self.logs.warn("Invalid price: %s" % price)
continue
quote = {"time": market_time, "price": price}
quotes.append(quote)
except KeyError as e:
self.logs.warn("Failed to find data: %s" % e)
return quotes
def test_intraday_pandas_pass_datetime(self):
u_date = "20190821"
data = get_historical_intraday("AAPL", date=u_date, output_format="pandas")
assert isinstance(data, pd.DataFrame)
assert data.index[0].strftime("%Y%m%d") == u_date
self.verify_timeframe(data)
def GetIntradayPrices(stk, key, day):
df = get_historical_intraday(stk,
day,
token="sk_290f4ad121254d7696a9a00c48748bb4")
data = []
for minAver in df:
data.append(minAver.get(key))
return data
def volume(parameters):
date = datetime(int(parameters["year"]), int(parameters["month"]),
int(parameters["day"]))
vl = get_historical_intraday(
parameters["company"],
date,
output_format='pandas',
token="sk_da231175d2cb425bbf943b1946966cfb")["volume"][0]
return vl
def get_time_series_intra_day(symbol, date=dt.now()):
"""
:param symbol:
:param date: type datetime.datetime
:return: df: pd.DataFrame
"""
df = get_historical_intraday(symbol, date=date)
return df
def area_plot_1_day_candlestick(symbol):
# from twelvedata import TDClient
from iexfinance.stocks import get_historical_intraday
token = 'pk_b4d054d6cd7640a38df80d61172890b3'
if dt.datetime.now().hour == 9:
if dt.datetime.now().minute >30:
start = dt.datetime.now()
else:
start = dt.datetime.now() - dt.timedelta(1)
elif dt.datetime.now().hour > 9:
start = dt.datetime.now()
else:
start = dt.datetime.now() - dt.timedelta(1)
# start=dt.datetime(2020, 12, 12)
start_date = start.replace(hour=9, minute=30, second=0, microsecond=0)
print(start_date)
end_date = start.replace(hour=16, minute=0, second=0, microsecond=0)
# Initialize client - apikey parameter is requiered
# api_key = '701005b908bc42b8800ca2fac03f2736'
df = get_historical_intraday(symbol, ouput_format='pandas', token=token, start=start_date, end=end_date)
print(df)
dt_all = pd.date_range(start=df.index[0], end=df.index[-1])
# print(dt_all)
dt_obs = [d.strftime("%Y-%m-%d") for d in df.index]
dt_breaks = [d for d in dt_all.strftime("%Y-%m-%d").tolist() if not d in dt_obs]
figure = go.Figure(
data=[
go.Candlestick(
x=df.index,
high=df['high'],
low=df['low'],
open=df['open'],
close=df['close'],
)
]
)
figure.update_layout(
xaxis_title="date",
yaxis_title="price ($)",
)
figure.update_xaxes(
rangeslider_visible=False,
rangeselector=dict(
buttons=list([
# dict(count=1, label="day", step="day", stepmode="todate"),
dict(count=30, label="last half hour", step="minute", stepmode="todate"),
dict(count=1, label="last hour", step="hour", stepmode="todate"),
dict(count=4, label="last 4 hours", step="hour", stepmode="todate"),
dict(step="all")
])
),
rangebreaks=[dict(values=dt_breaks)], # hide dates with no values
)
area_div = plot(figure, output_type='div')
return area_div
def extract_historical(self):
"""
Extract function to extract data and create a table to be used later in machine learning
"""
start = datetime(2020, 6, 1)
end = datetime(2020,9,16)
self.df = get_historical_intraday(self.ticker, start=start,token="pk_04da3e6c36334468ac1513b3adfe1531")
return self.df
def getHistoricalData(self, StartDate, EndDate):
'''StartDate and EndDate are date objects'''
#for date in daterange(StartDate, EndDate):
# self.historicalData[date] = get_historical_intraday(self.ticker, date, output_format='pandas', token = IEX_TOKEN)
for date in daterange(StartDate, EndDate):
day_data = get_historical_intraday(self.ticker,
date,
output_format='pandas',
token=IEX_TOKEN)
self.historicalData = self.historicalData.append(day_data)
def extractStockHistoricalData(self, stocks, endDate, daysback):
#extract minute stock data for a period of time. parameter stocks is a list of stock symbols
try:
date = endDate
for stock in stocks:
logger.logInfo('Getting data for: ' + stock)
historicalData = pd.DataFrame()
for i in range(daysback):
if i == 0:
historicalData = get_historical_intraday(stock, date, output_format='pandas')
date = date - timedelta(days=1)
continue
tempData = get_historical_intraday(stock, date, output_format='pandas')
historicalData = historicalData.append(tempData)
date = date - timedelta(days=1)
logger.logInfo('Saving data for: ' + stock)
fileName = '%s\%s_%i_days_ended_on_%s.csv' %(resourcesDirectory, stock, daysback, str(endDate.date()))
historicalData.to_csv(fileName)
# TODO: call the uploadDataToS3() method
logger.logInfo('Data for ' + stock + ' is saved in ' + fileName)
except Exception as ex:
logger.logError('Error extracting stock data. Error: ' + ex)
def grafica(update, context):
try:
fitxer = "%d.png" % random.randint(1000000, 9999999)
dataframe = get_historical_intraday(context.args[0],
output_format='pandas')
dataframe = dataframe.filter(items=['close'])
dataframe.plot()
plt.savefig(fitxer, bbox_inches='tight')
context.bot.send_photo(chat_id=update.effective_chat.id,
photo=open(fitxer, 'rb'))
os.remove(fitxer)
except Exception as e:
print(e)
context.bot.send_message(chat_id=update.effective_chat.id, text='💣')
def get_intraday_data(ticker, date=None):
"""get_intraday_data(ticker, date) -> Pandas Data Frame
Gets intraday data (open, close, high, low, volume) for date as a Panda's Data
Frame.
"""
try:
# TODO: CACHE
return iexstocks.get_historical_intraday(ticker,
date=date,
output_format='pandas',
token=IEX_TOKEN)
except:
assert False
def get_day_market_prices(name: str, date: datetime):
date_list = list()
tmp_date = datetime(date.year, date.month, date.day)
while not date_list:
response = get_historical_intraday(name,
tmp_date,
output_format='pandas',
token=config.config["IEX_API_KEY"])
date_list = list(
map(lambda x: (parser.parse(x[3] + " " + x[5]), x[0]),
response.values))
tmp_date = next_day(tmp_date)
return date_list
def __get_intraday_asset_data(symbol: Union[AnyStr, Iterable[AnyStr]],
start: datetime,
end: datetime) -> pd.DataFrame:
"""
Retrieve intraday historical market date from iexfinance.com
"""
data = pd.DataFrame([])
for i in tqdm(range((end - start).days + 1)):
date = start + datetime.timedelta(days=i)
df = get_historical_intraday(
symbol, date, output_format='pandas').fillna(method='ffill')
data = pd.concat([data, df], 0, sort=True)
data.name = symbol
return data
def queryIEXForStockData(stock, date):
#print (stock, date)
df = get_historical_intraday(stock, date=date, output_format='pandas')
#print (df.head())
df['date'] = df.index
df['stock'] = stock
df = ta.add_all_ta_features(df,
"marketOpen",
"marketHigh",
"marketLow",
"marketClose",
"marketVolume",
fillna=True)
df = df.round(5)
return (df)
def get_latest_price(self):
try:
latest = get_historical_intraday(self.ticker,
output_format='pandas')
latest = latest['open']
for i in latest:
if math.isnan(i):
pass
else:
latest = i
break
except:
print(
"Could not fetch latest price. (Markets are probably closed)")
latest = 0
return latest
def trial1():
aapl = Stock('AAPL', output_format='pandas', token=secret_key)
print(aapl.get_balance_sheet())
print('~~~~~~~~~~~~~~~~~~~~~')
print(aapl.get_quote())
print('~~~~~~~~~~~~~~~~~~~~~')
# print(get_social_sentiment("AAPL", token=secret_key))
tsla = Stock('TSLA', output_format='pandas', token=secret_key)
print(tsla.get_price())
batch = Stock(["TSLA", "AAPL"], token=secret_key)
print(batch.get_price())
print('~~~~~~~~~~~~~~~~~~~~~')
start = datetime(2017, 1, 1)
end = datetime(2018, 1, 1)
df = get_historical_data("TSLA",
start,
end,
output_format='pandas',
token=secret_key)
print(df.head())
fig1, ax1 = plt.subplots()
ax1.set_title("Open")
ax1.plot(df['open'])
fig2, ax2 = plt.subplots()
ax2.set_title("Volume")
ax2.plot(df['volume'])
print('~~~~~~~~~~~~~~~~~~~~~')
date = datetime(2018, 11, 27)
df = get_historical_intraday("AAPL",
output_format='pandas',
token=secret_key)
print(df.head())
print('~~~~~~~~~~~~~~~~~~~~~')
df = aapl.get_cash_flow()
print(df.head())
df = aapl.get_estimates()
print(df.head())
df = aapl.get_price_target()
print(df.head())
# print(get_social_sentiment("AAPL", token=secret_key))
# plt.show()
pass
def download(self, ticker):
log.info(f'Loading intraday data for {ticker}')
path = f'data/intraday/{ticker}_intraday.{self._from.strftime("%Y-%m-%d")}_{self._to.strftime("%Y-%m-%d")}.pkl'
print(f'Target file path {path}')
os.environ['IEX_TOKEN'] = self._iex_token
dfs = []
for dt in pd.date_range(self._from, self._to, freq='B'):
print(f"Loading intraday data for {dt}")
df = get_historical_intraday(ticker, dt, output_format='pandas')
if len(df) > 0:
df = df.loc[:, ['date', 'marketOpen', 'marketClose', 'marketVolume']]
df.rename(columns={'marketOpen': 'open', 'marketClose': 'close', 'marketVolume': 'volume'},
inplace=True)
dfs.append(df)
intraday = pd.concat(dfs)
intraday.index = pd.to_datetime(intraday.index)
intraday.date = pd.to_datetime(intraday.date)
print(f"Saved intraday data to {path}")
intraday.to_pickle(path)
def area_plot_1_day(symbol):
# from twelvedata import TDClient
link = 'https://cloud.iexapis.com/stable/stock/{]/chart/date/20210528?token={}'
from iexfinance.stocks import get_historical_intraday
token = 'pk_b4d054d6cd7640a38df80d61172890b3'
# if dt.datetime.now().hour == 9:
# if dt.datetime.now().minute >30:
# start = dt.datetime.now()
# else:
# start = dt.datetime.now() - dt.timedelta(1)
# elif dt.datetime.now().hour > 9:
# start = dt.datetime.now() - dt.timedelta(4)
# else:
# start = dt.datetime.now() - dt.timedelta(1)
start = dt.datetime.now()
start_date = start.replace(hour=9, minute=30, second=0, microsecond=0)
print(start_date)
end_date = start.replace(hour=16, minute=0, second=0, microsecond=0)
# Initialize client - apikey parameter is requiered
# api_key = '701005b908bc42b8800ca2fac03f2736'
df = get_historical_intraday(symbol, ouput_format='pandas', token=token, start=start_date, end=end_date)
print("df: ", df)
# clean data
dt_all = pd.date_range(start=df.index[0], end=df.index[-1])
dt_obs = [d.strftime("%Y-%m-%d") for d in df.index]
dt_breaks = [d for d in dt_all.strftime("%Y-%m-%d").tolist() if not d in dt_obs]
# figure = px.area(x=df.index, y=df['close'])
figure = choose_color_area_1_day(df)
print(dt_breaks)
figure.update_xaxes(
rangebreaks=[dict(values=dt_breaks)] # hide dates with no values
)
figure.update_layout(
xaxis_title="date",
yaxis_title="price ($)",
)
area_div = plot(figure, output_type='div')
return area_div
def update_data(self, data):
keep_cols = ['open', 'high', 'low', 'close', 'volume']
today = datetime.datetime.now().date()
today = datetime.datetime(today.year, today.month, today.day)
d = datetime.timedelta(days=1)
for sym in list(data):
cur_dt = list(data[sym])[-1] + d
while cur_dt <= today:
if datetime.date.weekday(cur_dt) < 5:
ret = get_historical_intraday(sym, cur_dt, output_format='pandas')
if len(ret) >= 65:
for c in ret.columns:
if c not in keep_cols:
ret = ret.drop(c, axis=1)
ret = ret.fillna(method='bfill')
ret = ret.to_dict(orient='index')
data[sym][cur_dt] = ret
print(sym, ' updated for ', cur_dt)
else:
print('missing values for ', sym, ' on ', cur_dt)
print('got back:', ret)
cur_dt += d
return data
def get_data_iex(ticker):
from iexfinance.account import get_usage
print(get_usage(quota_type='messages', token=globals.IEX_SECRET_KEY))
from iexfinance.altdata import get_social_sentiment
#print(get_social_sentiment("AAPL", token=globals.IEX_SECRET_KEY))
from iexfinance.stocks import Stock
#aapl = Stock("AAPL", token=globals.IEX_SECRET_KEY)
#print(aapl.get_estimates())
#print(aapl.get_price_target())
from datetime import datetime
from iexfinance.stocks import get_historical_intraday
#date = datetime(2018, 11, 27)
#get_historical_intraday("AAPL", date)
df = get_historical_intraday(
"AAPL", output_format='pandas',
token=globals.IEX_SECRET_KEY) # Can go up to 3 months back minutely
print(df)
# REAL TIME
from iexfinance.stocks import Stock
tsla = Stock('TSLA', token=globals.IEX_SECRET_KEY
) # CAUSING ERROR, maybe because the market isn't open now??
tsla.get_price()
batch = Stock(["TSLA", "AAPL"], token=globals.IEX_SECRET_KEY)
batch.get_price()
def show(self):
API_PUBLIC_KEY = 'pk_65b52176e67f4e8d86a61b3273d98107'
API_SECRET_KEY = 'sk_0df76468e959446e88abe99920684244'
date = datetime(2019, 10, 25)
assets = ['NCR', 'BLK', 'EFX', 'COF', 'ACN']
datasets = []
for asset in assets:
datasets.append(
get_historical_intraday(asset,
date,
output_format='pandas',
token=API_SECRET_KEY))
style.use('seaborn-bright')
y = {}
x = {}
count = {}
index = {}
figs = {}
axs = {}
# We just need the idx, idx is the date and time of the api call
for idx, _ in datasets[0].iterrows():
file_count = 0
#file_index = 0
for ind, data in enumerate(datasets):
# Init dictionary entries to empty arrays.
if ind not in y:
y[ind] = []
if ind not in x:
x[ind] = []
if ind not in count:
count[ind] = 0
if ind not in index:
index[ind] = 0
# Iterate through datasets.
row = data.loc[idx]
if not pd.isnull(row['average']):
plt.suptitle(assets[ind] + ' Stock on ' +
date.strftime('%m/%d/%Y'))
plt.xlabel('Minutes from Market Open')
plt.ylabel('Price (Dollars)')
y[ind].append(row['average'])
x[ind].append(count[ind])
fpath = './media/%s.png' % assets[file_count]
print(fpath)
x_mask = x[ind][index[ind] - 5:index[ind] + 5]
y_mask = y[ind][index[ind] - 5:index[ind] + 5]
index[ind] += 1
ax = plt.gca()
# recompute the ax.dataLim
ax.relim()
# update ax.viewLim using the new dataLim
ax.autoscale_view()
plt.scatter(x_mask, y_mask, c='g')
if os.path.isfile(
fpath): #save most recent graph for each company
os.remove(fpath)
plt.savefig(fpath)
plt.pause(0.5)
file_count += 1
count[ind] += 1
def getHistoricalData(self, StartDate, EndDate):
"""StartDate and EndDate are date objects"""
for date in daterange(StartDate, EndDate):
self.historicalData[date] = get_historical_intraday(
self.ticker, date, output_format='pandas', token=IEX_TOKEN)
