`sym` CHAR(20),
PRIMARY KEY(sym,date)
);'''
cursor.execute('CREATE DATABASE IF NOT EXISTS ' +
Database) #creat database if not exists
cursor.execute('USE ' + Database) #select the database
cursor.execute(create_RealtimeData) #create table
cursor.execute(create_HistoricalData) #create table
#end and start dates for the historical quotes
end = date.today()
start = end - timedelta(
days=365) # this calculates exactly a year back from today
# stroing stock packet (json) for each stock
apple = Share('AAPL')
google = Share('GOOGL')
nvidia = Share('NVDA')
yahoo = Share('YHOO')
amazon = Share('AMZN')
print('Reading Data and Putting them in Database.Quit with Ctrl+C')
try:
# using yahoo finance api to save data into a pandas dataframe
df_apple = pdr.get_data_yahoo('AAPL', start, end)
df_google = pdr.get_data_yahoo('GOOGL', start, end)
df_nvidia = pdr.get_data_yahoo('NVDA', start, end)
df_yahoo = pdr.get_data_yahoo('YHOO', start, end)
df_amazon = pdr.get_data_yahoo('AMZN', start, end)
# function to get a row for stock symbols to insert into the table
def current_stock_price(self):
symbol_f = self.symbol
data = Share(symbol_f)
share_value = (data.get_open())
return share_value
panel_data.to_frame().head(9)
import pandas_datareader as pdr
from datetime import datetime
ibm = pdr.get_data_yahoo(symbols='IBM', start=datetime(2000, 1, 1), end=datetime(2012, 1, 1))
print(ibm['Adj Close'])
#%%
#https://pypi.org/project/yahoo-finance/
pip install yahoo-finance
import yahoo-finance
from yahoo_finance import Share
yahoo = Share('YHOO')
yahoo.get_price()
>>> print yahoo.get_open()
yahoo_finance.Share("SBI.NSE")
symbol = yahoo_finance.Share("GOOG")
google_data = symbol.get_historical("2019-06-01", "2019-06-30")
google_df = pd.DataFrame(google_data)
# Output data into CSV
google_df.to_csv("/home/username/google_stock_data.csv")
#
pip install yahoofinancials
from yahoofinancials import YahooFinancials
tech_stocks = ['AAPL', 'MSFT', 'INTC']
def webhook():
"""Webhook."""
if not kik.verify_signature(request.headers.get('X-Kik-Signature'),
request.get_data()):
return Response(status=403)
messages = messages_from_json(request.json['messages'])
chat_record = ChatRecord(json.dumps(request.json['messages']))
db.session.add(chat_record)
db.session.commit()
for message in messages:
if isinstance(message, TextMessage):
logging.info(message)
if '$' in message.body:
send_text(message.from_user, message.chat_id, 'Looking up...')
for symbol in re.findall(r'\$\w(?:\w)*(?:\.\w+)?',
message.body):
symbol = symbol[1:]
yahoo = Share(symbol)
if yahoo.get_price():
text = 'Price of {} is {}'.format(
symbol, yahoo.get_price())
send_text(message.from_user, message.chat_id, text)
send_link(
message.from_user,
message.chat_id,
url='https://finance.yahoo.com/q?s={}'.format(
symbol),
title='Yahoo finace: {}'.format(symbol),
pic_url='https://chart.finance.yahoo.com/z?s={}'.
format(symbol),
)
else:
text = 'We couldn\'t find a ticker with {}.'.format(
symbol)
send_text(message.from_user, message.chat_id, text)
keyboards = [
'$' + t['symbol']
if '^' not in t['symbol'] else t['symbol']
for t in lookup(symbol)
][:4]
if keyboards:
text = 'Are you looking for...'
send_text(message.from_user, message.chat_id, text,
keyboards)
else:
text = 'What are you looking for?'
send_text(message.from_user, message.chat_id, text)
elif '^' in message.body:
send_text(message.from_user, message.chat_id, 'Looking up...')
for symbol in re.findall(r'\^\w(?:\w)*(?:\.\w+)?',
message.body):
send_link(
message.from_user,
message.chat_id,
url='https://finance.yahoo.com/q?s={}'.format(symbol),
title='Yahoo finace: {}'.format(symbol),
pic_url='https://chart.finance.yahoo.com/z?s={}'.
format(symbol),
)
elif 'lookup' in message.body.lower():
lookup_text = re.findall(r'lookup (\w+)', message.body.lower())
if lookup_text:
text = 'Are you looking for...'
keyboards = [
'$' + t['symbol'] for t in lookup(lookup_text[0])
][:8]
send_text(message.from_user, message.chat_id, text,
keyboards)
else:
text = 'What are you looking for?'
send_text(message.from_user, message.chat_id, text)
else:
if 'hi' in message.body.lower(
) or 'hello' in message.body.lower():
text = 'Hi {}!'.format(message.from_user)
else:
text = 'I don\'t understand message'
send_text(message.from_user, message.chat_id, text)
text = 'For live stock quotes type "$" followed by a ticker symbol or "lookup" followed by a company name.' # noqa
send_text(message.from_user, message.chat_id, text)
text = 'For example, if you want to look up Apple, type "$AAPL" or "lookup Apple".'
send_text(message.from_user, message.chat_id, text)
text = 'For index quotes, start with "^". For example, "^DJI" for Dow Jones Industrial Average.'
send_text(message.from_user, message.chat_id, text)
text = 'Try it now:'
send_text(message.from_user, message.chat_id, text,
["Lookup Apple"])
return Response(status=200)
if __name__ == '__main__':
# - setup command line arguments
parser = argparse.ArgumentParser()
parser.add_argument('symbol', help='the symbol of the stock to collect')
parser.add_argument('topic_name', help='the kafka topic push to')
parser.add_argument('kafka_broker',
help='the location of the kafka broker')
# - parse arguments
args = parser.parse_args()
symbol = args.symbol
topic_name = args.topic_name
kafka_broker = args.kafka_broker
# - instantiate a simple kafka producer
producer = KafkaProducer(bootstrap_servers=kafka_broker)
stock = Share(symbol)
# - schedule and run the fetch_price function every second
schedule.every(1).second.do(fetch_price, producer, stock)
# - setup proper shutdown hook
atexit.register(shutdown_hook, producer)
while True:
schedule.run_pending()
time.sleep(1)
def getStocksFromSource(indexes=data, sortBy=SORT_BY_TOP):
''' '''
stocks = []
index = ['AGTC']
# for stock in data["Ticker"][:100]:
# for stock in index:
for stock in data:
try:
print(stock)
# print(type(stock))
yf_data = yqd.load_yahoo_quote(stock, '20170301', '20170830')
# yf_data = yqd.load_yahoo_quote('ABEO', '20170712', '20170725')
# print(yf_data)
share = Share(stock)
# history part
history = []
for i, day in enumerate(yf_data[1:-1]):
daily_data = day.split(',')
history.append([
i,
str(daily_data[0]),
float(daily_data[1]),
float(daily_data[2]),
float(daily_data[3]),
float(daily_data[4]),
float(daily_data[6])
])
# print(history)
# comments part
comments = []
new_StockTwits_comments = []
url = 'https://api.stocktwits.com/api/2/streams/symbol/{0}.json'.format(
stock)
print(url)
try:
r = requests.get(url).json()
print(len(r['messages']))
for message in r['messages']:
try:
new_tweet = {
'id':
message['id'],
'body':
message['body'],
'created_at':
message['created_at'],
'core_body':
nltk_service.clean_tweet(message['body']),
'nltk_sentiment':
nltk_service.get_tweet_sentiment(message['body']),
# 'azure_sentiment': azure_sentiment_service.GetSentiment(message['body'])
}
try:
new_tweet[
'azure_sentiment'] = azure_sentiment_service.GetSentiment(
message['body'])
except Exception as e:
new_tweet['azure_sentiment'] = 0.5
print(e)
# print(new_tweet['azure_sentiment'])
new_StockTwits_comments.append(new_tweet)
except Exception as e:
print(e)
# pass
except Exception as e:
print('stock tweets part problem')
print(e)
# new_StockTwits_comments = [{'id': message['id'], 'body': message['body'], 'created_at': message['created_at']} for message in r['messages']]
print(len(new_StockTwits_comments))
stock = {
'index': stock,
'open': share.get_open(),
'change': share.get_change(),
'percent_change': share.get_percent_change(),
'prev_close': share.get_prev_close(),
'price': share.get_price(),
'volume': share.get_volume(),
'history': history,
'new_StockTwits_comments': new_StockTwits_comments
}
# stock_json = json.dumps(stock)
# print(type(stock_json))
print(len(history))
if len(history) != 0:
# f.write(stock['index']+'/n')
stocks.append(stock)
except Exception as e:
print(e)
pass
print(len(stocks))
return stocks
# f.close()
# get_price()
# get_change()
# get_percent_change()
# get_volume()
# get_prev_close()
# get_open()
# get_avg_daily_volume()
# get_stock_exchange()
# get_market_cap()
# get_book_value()
# get_ebitda()
# get_dividend_share()
# get_dividend_yield()
# get_earnings_share()
# get_days_high()
# get_days_low()
# get_year_high()
# get_year_low()
# get_50day_moving_avg()
# get_200day_moving_avg()
# get_price_earnings_ratio()
# get_price_earnings_growth_ratio()
# get_price_sales()
# get_price_book()
# get_short_ratio()
# get_trade_datetime()
# get_historical(start_date, end_date)
# get_name()
# refresh()
# get_percent_change_from_year_high()
# get_percent_change_from_year_low()
# get_change_from_year_low()
# get_change_from_year_high()
# get_percent_change_from_200_day_moving_average()
# get_change_from_200_day_moving_average()
# get_percent_change_from_50_day_moving_average()
# get_change_from_50_day_moving_average()
# get_EPS_estimate_next_quarter()
# get_EPS_estimate_next_year()
# get_ex_dividend_date()
# get_EPS_estimate_current_year()
# get_price_EPS_estimate_next_year()
# get_price_EPS_estimate_current_year()
# get_one_yr_target_price()
# get_change_percent_change()
# get_dividend_pay_date()
# get_currency()
# get_last_trade_with_time()
# get_days_range()
# get_year_range()
#!/usr/bin/python
# based on the article found at http://bilgin.esme.org/BitsBytes/KalmanFilterforDummies.aspx
from datetime import date
from time import time
from math import sqrt
import pylab
from pylab import array, random
from yahoo_finance import Share
start_date = date.fromtimestamp(time() -
60 * 60 * 24 * 30).strftime('%Y-%m-%d')
end_date = date.today().strftime('%Y-%m-%d')
share = Share('TWTR')
share_history = share.get_historical(start_date, end_date)
readings = []
for day in share_history:
readings.append(float(day['Close']))
# kalman filter implementation
# ============================
process_variance = 1e-5
estimated_measurement_variance = 0.1**2
posteri_estimate = 0.0
posteri_error_estimate = 1.0
estimates = []
estimate_errors = []
for reading in readings:
# predict
s_day = '2016-07-01'
e_day = '2017-02-22'
data = pd.DataFrame(spd.get_historical(s_day,e_day))
"""
# 找到有效天，作为一个index，再找到一个上海的指数，作为每个数据都有的列
"""
spd = Share('000034.sz')
s_day = '2016-07-01'
e_day = '2017-02-21'
temp = pd.DataFrame(spd.get_historical(s_day,e_day))
temp.Volume = pd.to_numeric(temp.Volume)
date_index = temp["Date"][temp.Volume >0] # 155天，但因为下面发现上证指数数据不全，所以以下面为准
"""
s_day = '2016-07-01'
e_day = '2017-02-21'
spd = Share('000001.ss')
sh_index = pd.DataFrame(spd.get_historical(s_day, e_day))
sh_index.Close = pd.to_numeric(sh_index.Close)
sh_index.index = sh_index.Date # 147天
# sh_index.reindex(date_index) 发现很坑的一点是上证指数只有147天的数据，但看了一下幸好都在上面date_index里
# 所以之后都用sh_index吧，会少一些天
# 研究一下如何计算用这个dataframe计算收益率：想到了！用错开的去减，上海return也放进去
close_last = np.append(np.array(sh_index.Close[1:]), np.nan)
sh_index['Close_last'] = close_last
sh_index['return_rate'] = np.log(sh_index.Close / sh_index.Close_last)
# 按100以上区别
def random_select_stock(code_array, num=30):
size = code_array.shape[0]
def checklist(symbol, ibd50_list, ibd_session):
"""
Looks up information on a given stock market symbol.
The returned dictionary contains all information from
Dr. Wish's Stock Checklist for HONR348M.
"""
stock = {}
# Load price data from yahoo.
share = Share(symbol)
ks = yahoo_ks(symbol)
# Basics
basics = stock["basics"] = {}
basics["date"] = datetime.now().strftime("%m/%d/%Y %I:%M:%S%z")
basics["symbol"] = symbol
basics["equity_name"] = share.get_name()
basics["price"] = float(share.get_price())
basics["52w_low"] = float(share.get_year_low())
basics["52w_high"] = float(share.get_year_high())
basics["percent_from_52w_low"] = share.get_percent_change_from_year_low()
basics["percent_from_52w_high"] = share.get_percent_change_from_year_high()
# IBD (Stocks only)
ibd = stock["ibd"] = ibd_stock_checkup(symbol, ibd_session)
# ibd["industry"]
ibd["industry_rank"] = float(ibd["industry_rank"])
# ibd["industry_top5"]
# ibd["3y_eps_growth"]
# ibd["3y_sales_growth"]
# ibd["eps_change"]
ibd["eps_rating"] = float(ibd["eps_rating"])
ibd["rs_rating"] = float(ibd["rs_rating"])
# ibd["acc_distr_rating"]
ibd["ibd_rating"] = float(ibd["ibd_rating"])
ibd["in_ibd50"] = symbol in ibd50_list
# ibd["fundamental_greens"]
# ibd["technical_greens"]
ibd["next_earning"] = datetime.strptime(ibd["next_earning"], '%m/%d/%Y')
# Yahoo Finance (Stocks only)
yahoo = stock["yahoo"] = {}
yahoo["pe"] = float(share.get_price_earnings_ratio())
yahoo["peg"] = float(share.get_price_earnings_growth_ratio())
yahoo["ps"] = float(share.get_price_sales())
yahoo["market_cap"] = share.get_market_cap()
yahoo["float"] = ks["Float"]
yahoo["annual_roe"] = ks["Return on Equity"]
yahoo["percent_inst"] = ks["% Held by Institutions"]
yahoo["percent_float_short"] = ks["Short % of Float"]
yahoo["short_ratio"] = float(share.get_short_ratio())
# Evidence of an uptrend/downtrend
uptrend = stock["uptrend"] = {}
downtrend = stock["downtrend"] = {}
pdstockdata = data.DataReader(symbol, 'yahoo', '1900-01-01')
sd = StockDataFrame.retype(pdstockdata)
sd.BOLL_PERIOD = 15
close1 = sd['close'][-1]
close2 = sd['close'][-2]
low1 = sd['low'][-1]
low2 = sd['low'][-2]
high1 = sd['high'][-1]
high2 = sd['high'][-2]
avg_30d = sd['close_30_sma'][-1]
avg_4w = sd['close_20_sma'][-1]
avg_10w = sd['close_50_sma'][-1]
avg_30w = sd['close_150_sma'][-1]
high_52w = sd['high'].tail(250).max()
lbb1 = sd['boll_lb'][-1]
lbb2 = sd['boll_lb'][-2]
ubb1 = sd['boll_ub'][-1]
ubb2 = sd['boll_ub'][-2]
# Find all GLTs (ATH not broken for at least another 90 days)
last_ath = 0.0
ath = Series()
for day, day_high in sd['high'].iteritems():
last_ath = max(last_ath, day_high)
ath.set_value(day, last_ath)
ath_days = sd[sd['high'] == ath]['high']
glt = Series()
for i, (day, high) in enumerate(ath_days.iteritems()):
next_day = ath_days.keys()[i +
1] if i < len(ath_days) - 1 else Timestamp(
str(date.today()))
if next_day - day >= Timedelta('90 days'):
glt.set_value(day, high)
uptrend["c>30d_avg"] = close1 > avg_30d
uptrend["c>10w_avg"] = close1 > avg_10w
uptrend["c>30w_avg"] = close1 > avg_30w
uptrend["4w>10w>30w"] = avg_4w > avg_10w > avg_30w
# uptrend["w_rwb"] =
uptrend["last_glt_date"] = glt.keys()[-1].to_datetime().date(
) if len(glt) > 0 else None
uptrend["last_glt_high"] = glt[-1] if len(glt) > 0 else None
uptrend["above_last_glt"] = len(glt) > 0 and close1 > glt[-1]
uptrend["macd_hist_rising"] = sd['macdh'][-1] > sd['macdh_4_sma'][-1]
uptrend["stoch_fast>slow"] = sd['rsv_10_4_sma'][-1] > sd[
'rsv_10_4_sma_4_sma'][-1]
# uptrend["bb_up_expansion_l2"]
# uptrend["rs>30d_avg"] = (Need Investors.com data)
# uptrend["rs_rising"] = (Need Investors.com data)
uptrend["52w_high_l2"] = high_52w == high1 or high_52w == high2
uptrend["ath_l2"] = ath[-1] == high1 or ath[-2] == high2
uptrend["1y_doubled"] = close1 >= 2 * sd['close'][-255:-245].mean()
# uptrend["bounce_30d_l2"] =
# uptrend["bounce_10w_l2"] =
# uptrend["bounce_30w_l2"] =
uptrend["stoch<50"] = sd['rsv_10_4_sma'][-1] < 50
uptrend["<bb_lower_l2"] = low1 < lbb1 or low2 < lbb2
uptrend[
"above_avg_volume"] = sd['volume'][-1] > 1.5 * sd['volume_50_sma'][-1]
downtrend["c<30d_avg"] = close1 < avg_30d
downtrend["c<10w_avg"] = close1 < avg_10w
downtrend["c<30w_avg"] = close1 < avg_30w
downtrend["4w<10w<30w"] = avg_4w < avg_10w < avg_30w
# downtrend["w_bwr"] =
downtrend["macd_hist_falling"] = sd['macdh'][-1] < sd['macdh_4_sma'][-1]
downtrend["stoch_fast<slow"] = sd['rsv_10_4_sma'][-1] < sd[
'rsv_10_4_sma_4_sma'][-1]
# downtrend["bb_down_expansion_l2"]
# downtrend["bounce_30d_l2"] =
# downtrend["bounce_10w_l2"] =
# downtrend["bounce_30w_l2"] =
downtrend["stoch>50"] = sd['rsv_10_4_sma'][-1] > 50
downtrend[">bb_upper_l2"] = high1 > ubb1 or high2 > ubb2
return stock
def getPriceNChange(str):
stock = Share(str)
return stock.get_price() + ' ' + stock.get_change(
) + ' (' + stock.get_percent_change() + ')'
def yahoo_example():
ticker = Share('YHOO')
print(ticker.get_price())
def getFullName(str):
return Share(str).get_name() + " (" + str + ")"
print("\n\n")
stockArray = []
csvfile = open('companylist.csv', 'r')
fieldnames = ("Symbol","Name","LastSale", "MarketCap", "IPOyear", "Sector", "industry")
reader = csv.DictReader(csvfile, fieldnames)
reader.next()
for row in reader:
s = Stock(row["Symbol"], row["Name"], row["LastSale"], row["MarketCap"], row["IPOyear"], row["Sector"], row["industry"])
stockArray.append(s)
for s in stockArray:
try:
quoteFromGoogle = getQuotes(s.symbol)
shareFromYahoo = Share(s.symbol)
todaysPrice = quoteFromGoogle[0]["LastTradePrice"]
yearLow = shareFromYahoo.get_year_low()
print("Today's price for " + s.symbol + " is " + todaysPrice)
print("52 Year Low: " + yearLow)
if float(todaysPrice) <= float(yearLow):
print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
body = s.symbol + " (" + s.name + ") " +" is now at 52 weeks low!! \n Today's price is at $" + todaysPrice + "; 52 weeks low was $" + yearLow "
message = client.messages.create(to="+14152793685", from_="+15109015113", body=body)
print(body)
print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
except:
print("#####################################")
from datetime import datetime, timedelta
from yahoo_finance import Share
# extract the symbol code of the sp500
sp500 = pd.read_csv('SP500.csv')
source = sp500["Symbol"]
today = datetime.strptime(time.strftime("%Y/%m/%d"),
"%Y/%m/%d") #string to date
from_date = today - timedelta(days=30) # date - days
today = today.strftime("%Y-%m-%d")
from_date = from_date.strftime("%Y-%m-%d")
print(from_date, today)
ticker_info = Share(source[0]).get_historical(from_date, today)
stock = [(x['Date'], x['Close']) for x in ticker_info]
stocks = pd.DataFrame(stock, columns=['Date', source[0]]).set_index('Date')
for ticker in source[1:]:
try:
ticker_info = Share(ticker).get_historical(from_date, today)
stock = [(x['Date'], x['Close']) for x in ticker_info]
stock = pd.DataFrame(stock, columns=['Date', ticker]).set_index('Date')
stocks = pd.concat([stocks, stock], axis=1)
stocks.to_csv('stock_data.csv')
except:
pass
stocks.to_csv('stock_data.csv')
def share_price(ticker):
from yahoo_finance import Share
company = Share(ticker)
price = float(company.get_price())
return price
import time
from yahoo_finance import Share
import sys
# Collect 30 mins of Finance data on 5 companies, one value per minute
share = Share(sys.argv[1])
for i in range(30):
share.refresh()
print share.get_price()
time.sleep(60)
# RUN SCRIPT IN PYTHON
# python yahoo.py LNKD> linkedin.txt
# python yahoo.py FB> facebook.txt
# python yahoo.py IBM> ibm.txt
# python yahoo.py YHOO> yahoo.txt
# python yahoo.py NYT> nyt.txt
a = [[
1 * (lI[0][0][1] > temp), lS[0], 1 * (sI[0][0][1] > temp),
sS[0]
],
map(lambda x: 1 * (x[1] > temp), tC[0])]
r, t, s = self.env.step(a)
if __name__ == '__main__':
window_size = 24
ticker = sys.argv[1]
data_path = '/tmp/ticker_data_{}.csv'.format(ticker)
results_path = '/tmp/tradelog_{}.csv'.format(ticker)
share = Share(ticker)
try:
data = share.get_historical('2016-04-25', '2017-04-29')
data = [[
d['Date'],
float(d['Open']),
float(d['High']),
float(d['Low']),
float(d['Close']),
int(d['Volume'])
] for d in data]
env = StockTradeSimulator([d[1:] for d in data], 0, window_size)
pd.DataFrame(data).to_csv(data_path)
except:
with open(data_path, 'w') as file:
file.writelines('Bad Ticker')
import os
os.system('cls' if os.name == 'nt' else 'clear')
startDate = date.today() - timedelta(days=30)
#endDate = date.today()-timedelta(days=1)
endDate = date.today()
#print (startDate, endDate)
#dates = pd.date_range(startDate, periods=60)
#print(dates)
def cdf(dev_divided_by_std):
return (1.0 + math.erf(dev_divided_by_std / sqrt(2.0))) / 2.0
uwti = Share('UWTI')
uup = Share('UUP')
uwtiHistory_raw = uwti.get_historical(str(startDate), str(endDate))
uupHistory_raw = uup.get_historical(str(startDate), str(endDate))
datestrs = [startDate] * (len(uwtiHistory_raw))
pricestrs_uwti = [0.0] * (len(uwtiHistory_raw))
for i in range(len(uwtiHistory_raw)):
datestrs[i] = uwtiHistory_raw[i]['Date']
pricestrs_uwti[i] = float(uwtiHistory_raw[i]['Close'])
pricestrs_uup = [0.0] * (len(uwtiHistory_raw))
for i in range(len(uupHistory_raw)):
pricestrs_uup[i] = float(uupHistory_raw[i]['Close'])
def load_yahoo_stock_price(stock, iLocal=True, YAHOO_LINKS_DATA=None):
filename = YAHOO_LINKS_DATA.get(stock)
if (filename is None):
raise Exception("MISSING " + stock)
# print("YAHOO_DATA_LINK" , stock, filename);
tsspec = cTimeSeriesDatasetSpec()
tsspec.mName = "Yahoo_Stock_Price_" + stock
tsspec.mDescription = "Yahoo Stock Price using yahoo-finance package"
df_train = pd.DataFrame()
if (iLocal):
filename = "data/yahoo/" + filename
else:
base_uri = "https://raw.githubusercontent.com/antoinecarme/TimeSeriesData/master/YahooFinance/"
filename = base_uri + filename
print("YAHOO_DATA_LINK_URI", stock, filename)
if (os.path.isfile(filename)):
# print("already downloaded " + stock , "reloading " , filename);
df_train = pd.read_csv(filename)
else:
# return None;
from yahoo_finance import Share
stock_obj = Share(stock)
today = date.today()
today
before = date(today.year - 5, today.month, today.day)
# print(today, before)
lst = stock_obj.get_historical(before.isoformat(), today.isoformat())
# print(stock , len(lst));
if (len(lst) > 0):
for k in lst[0].keys():
for i in range(0, len(lst)):
lst_k = []
for line1 in lst:
lst_k = lst_k + [line1[k]]
df_train[k] = lst_k
# df_train.to_csv(filename);
else:
# df_train.to_csv(filename);
return None
tsspec.mFullDataset = pd.DataFrame()
tsspec.mFullDataset[stock] = df_train['Close'].apply(float)
tsspec.mFullDataset['Date'] = df_train['Date'].apply(
lambda x: datetime.datetime.strptime(x, "%Y-%m-%d"))
# print(tsspec.mFullDataset.head());
tsspec.mFullDataset.sort_values(by='Date', ascending=True, inplace=True)
tsspec.mFullDataset.reset_index(inplace=True, drop=True)
# print(tsspec.mFullDataset.head());
tsspec.mTimeVar = "Date"
tsspec.mSignalVar = stock
lHorizon = 7 # 7 days
if (lHorizon > tsspec.mFullDataset.shape[0]):
# nysecomp/yahoo_VRS.csv is too small
lHorizon = 1
tsspec.mHorizon = {}
tsspec.mHorizon[stock] = lHorizon
tsspec.mPastData = tsspec.mFullDataset[:-lHorizon]
tsspec.mFutureData = tsspec.mFullDataset.tail(lHorizon)
# print(tsspec.mFullDataset.head())
return tsspec
def stockdata(comp,percentage, start, end,marketDict):
stock = Share(comp)
history = stock.get_historical(start, end)
n = len(history)
H,C,L,O = np.zeros(n),np.zeros(n),np.zeros(n),np.zeros(n)
for i in range(n):
H[i] = history[-(i+1)]['High']
C[i] = history[-(i+1)]['Close']
L[i] = history[-(i+1)]['Low']
O[i] = history[-(i+1)]['Open']
H,C,L,O,V = obtainHCLO(comp,start,end)
Y5 = np.sign(C[5:] - C[:-5])
Y10 = np.sign(C[10:] - C[:-10])
X,Y = IO(H,C,L,O)
H,C,L,O = clusterPrice(comp,start,end,marketDict)
Xcluster , Ycluster = IO(H,C,L,O)
X = np.hstack((X,Xcluster))
N = 15
X1 = X[:-1,:]
X5 = X[:-5,:]
X10 = X[:-10,:]
Y1 = Y[N:]
Y5 = Y5[N:]
Y10 = Y10[N:]
sample = int(len(X)*percentage/100.)
Xtrain1,Ytrain1 = X1[:sample,:],Y1[:sample]
Xtrain5,Ytrain5 = X5[:sample,:],Y5[:sample]
Xtrain10,Ytrain10 = X10[:sample,:],Y10[:sample]
Xtest1,Ytest1 = X1[sample:,:],Y1[sample:]
Xtest5,Ytest5 = X5[sample:,:] , Y5[sample:]
Xtest10,Ytest10 = X10[sample:,:] , Y10[sample:]
np.savetxt(comp +'1_xtrain.txt',Xtrain1)
np.savetxt(comp +'1_ytrain.txt',Ytrain1)
np.savetxt(comp +'1_xtest.txt',Xtest1)
np.savetxt(comp +'1_ytest.txt',Ytest1)
np.savetxt(comp +'5_xtrain.txt',Xtrain5)
np.savetxt(comp +'5_ytrain.txt',Ytrain5)
np.savetxt(comp +'5_xtest.txt',Xtest5)
np.savetxt(comp +'5_ytest.txt',Ytest5)
np.savetxt(comp +'10_xtrain.txt',Xtrain10)
np.savetxt(comp +'10_ytrain.txt',Ytrain10)
np.savetxt(comp +'10_xtest.txt',Xtest10)
np.savetxt(comp +'10_ytest.txt',Ytest10)
def build_ETF(ETF, ETF_dic, host='localhost:27017'):
N = len(ETF)
# Initialize the collection ETF
client = MongoClient([host])
# client.drop_database('RoboAdvisor')
db = client.RoboAdvisor
db.ETF.drop()
posts = db.ETF
# Add ticker as _id
for e in range(N):
posts.insert_one({'_id': ETF[e]})
# Add Asset class
for e in range(N):
asset = ETF[e]
assetClass = ETF_dic[asset]
posts.update_one({'_id': asset}, {'$set': {"asset_class": assetClass}})
# Add summary data
add_summary(ETF, posts, N)
# Add historical data
summary_data = get_summary_data_all(host)
for e in range(N):
asset = ETF[e]
today = time.strftime("%Y-%m-%d")
print(asset + " is start!")
ETFs_data = Share(asset).get_historical(
summary_data["inception_date"][asset], today)
ETFs_data = json_normalize(ETFs_data)
ETFs_data = ETFs_data.set_index("Date")
dates = sorted(ETFs_data["Adj_Close"].keys())
n = len(dates)
# date = dates[0].to_datetime().strftime('%Y-%m-%d')
# date = dates[0].to_datetime()
date = dates[0]
close = ETFs_data["Adj_Close"][dates[0]]
volume = ETFs_data["Volume"][dates[0]]
posts.update_one({'_id': asset}, {
'$set': {
'hist_data': [{
'date':
datetime.datetime.strptime(date, '%Y-%m-%d'),
'quotes': {
'close': float(close),
'volume': float(volume)
}
}]
}
})
for x in range(1, n):
# date = dates[x].to_datetime().strftime('%Y-%m-%d')
# date = dates[x].to_datetime()
date = dates[x]
close = ETFs_data["Close"][dates[x]] # "%.2f" %
volume = ETFs_data["Volume"][dates[x]]
histData = {
'$push': {
'hist_data': {
'date': datetime.datetime.strptime(date, '%Y-%m-%d'),
'quotes': {
'close': float(close),
'volume': float(volume)
}
}
}
}
posts.update_one({'_id': asset}, histData)
return
tassets = Tassets(balance)
cassets = Cassets(balance)
tliab = Tliab(balance)
cliab = Cliab(balance)
p_e = P_E(summary)
eps = EPS(summary)
nincome = Nincome(incomestatement)
div = Div(summary)
divpaid = Divpaid(cashflowpage)
trev = Trev(incomestatement)
#yiel = Yield(summary)
equity = Equity(balance)
cashflow = CashFlow(cashflowpage)
#This one uses a different method
yahoo = Share(stock)
stockp = float(yahoo.get_price())
Ip_e = IndusP_E(competitors, stock)
tnx = TNX(tnxpage)
pasttassets = PastTassets(balance)
pasttliab = PastTliab(balance)
pasttrev = PastTrev(incomestatement)
pastnincome = PastNincome(incomestatement)
pastcashflow = PastCashFlow(cashflowpage)
#printing the variables
print '\n', title
print "Total Assets = ", '%24s' % ('$' + str(tassets))
print "Total Current Assets = ", '%16s' % ('$' + str(cassets))
for line_number, line in enumerate(f):
try:
Codes.append(line.split('\n')[0].split('\t')[0])
Names.append(line.split('\n')[0].split('\t')[1])
except:
pass
for line_number, line in enumerate(f1):
try:
Codes.append(line.split('\n')[0].split(',')[0])
Names.append(line.split('\n')[0].split(',')[1])
except:
pass
try:
share = Share(Code)
Stocks = share.get_historical(Start_Date, End_Date)
Opening = []
Closing = []
Low = []
High = []
Volumes = []
f = open('Stock_Details.csv', 'w')
def getStock(_id, start, end): #string
stock = Share(str(_id) + '.TW')
today = datetime.date.today() #todays date
data = stock.get_historical(start, end)
return data
def check_valid_symbol(symbol):
"""We identify invalid shares if there is no official company name
associated with the specified ticker.
"""
s = Share(symbol)
return s.get_name() is not None
def YaOrg(gather=[
'debtToEquity', 'Trailing P/E', 'Price/Sales', 'priceToBook',
'profitMargins', 'operatingMargins', 'returnOnAssets', 'returnOnEquity',
'revenuePerShare', 'Market Cap', 'enterpriseValue', 'forwardPE',
'pegRatio', 'enterpriseToRevenue', 'enterpriseToEbitda', 'totalRevenue',
'grossProfit', 'ebitda', 'netIncomeToCommon', 'trailingEps',
'earningsGrowth', 'revenueGrowth', 'totalCash', 'totalCashPerShare',
'totalDebt', 'currentRatio', 'bookValue', 'operatingCashflow', 'beta',
'heldPercentInsiders', 'heldPercentInstitutions', 'sharesShort',
'shortRatio', 'shortPercentOfFloat', 'sharesShortPriorMonth',
'currentPrice', 'sharesOutstanding'
]):
df = pd.DataFrame(columns=[
'Date', 'Unix', 'Ticker', 'Price', 'stock_p_change', 'SP500',
'sp500_p_change', 'Difference', 'DE Ratio', 'Trailing P/E',
'Price/Sales', 'Price/Book', 'Profit Margin', 'Operating Margin',
'Return on Assets', 'Return on Equity', 'Revenue Per Share',
'Market Cap', 'Enterprise Value', 'Forward P/E', 'PEG Ratio',
'Enterprise Value/Revenue', 'Enterprise Value/EBITDA', 'Revenue',
'Gross Profit', 'EBITDA', 'Net Income Avl to Common ', 'Diluted EPS',
'Earnings Growth', 'Revenue Growth', 'Total Cash',
'Total Cash Per Share', 'Total Debt', 'Current Ratio',
'Book Value Per Share', 'Cash Flow', 'Beta', 'Held by Insiders',
'Held by Institutions', 'Shares Short (as of', 'Short Ratio',
'Short % of Float', 'Shares Short (prior ', 'Status'
])
#time.sleep(2)
file_list = os.listdir("YaParse")
counter = 0
passfiles = 0
for each_file in file_list:
ticker = each_file.split(".json")[0]
full_file_path = "YaParse/" + each_file
source = open(full_file_path, "r").read()
data = Share(ticker.upper())
counter += 1
print("File No: " + str(counter))
try:
value_list = []
for each_data in gather:
try:
#value = float(source.split(gather + ':</td><td class="yfnc_tabledata1">')[1].split('</td>')[0])
regex = re.escape(
each_data) + r'.*?(\d{1,8}\.\d{1,8}M?B?K?|N/A)%?'
value = re.search(regex, source)
value = (value.group(1))
if "B" in value:
value = float(value.replace("B", '')) * 1000000000
elif "M" in value:
value = float(value.replace("M", '')) * 1000000
value_list.append(value)
except Exception as e:
if not (each_data == 'Trailing P/E' or each_data
== 'Price/Sales' or each_data == 'Market Cap'):
print("Exception1: " + str(e))
print("Data: " + str(each_data) + " File: " +
str(each_file))
#time.sleep(5)
value = "N/A"
if each_data == 'Price/Sales':
value = data.get_price_sales()
if each_data == 'Market Cap':
value = data.get_market_cap()
if each_data == 'Trailing P/E':
value = data.get_price_earnings_ratio()
try:
if "B" in value:
value = float(value.replace("B", '')) * 1000000000
elif "M" in value:
value = float(value.replace("M", '')) * 1000000
except Exception as e:
value = "N/A"
value_list.append(value)
#print(value_list)
#time.sleep(5)
if value_list.count("N/A") > 0:
print("Passing on this file: " + str(each_file) +
" ::::Count of N/A: " + str(value_list.count("N/A")))
passfiles += 1
pass
else:
try:
df = df.append(
{
'Date': "N/A",
'Unix': "N/A",
'Ticker': ticker,
'Price': "N/A",
'stock_p_change': "N/A",
'SP500': "N/A",
'sp500_p_change': "N/A",
'Difference': "N/A",
'DE Ratio': value_list[0],
'Trailing P/E': value_list[1],
'Price/Sales': value_list[2],
'Price/Book': value_list[3],
'Profit Margin': value_list[4],
'Operating Margin': value_list[5],
'Return on Assets': value_list[6],
'Return on Equity': value_list[7],
'Revenue Per Share': value_list[8],
'Market Cap': value_list[9],
'Enterprise Value': value_list[10],
'Forward P/E': value_list[11],
'PEG Ratio': value_list[12],
'Enterprise Value/Revenue': value_list[13],
'Enterprise Value/EBITDA': value_list[14],
'Revenue': value_list[15],
'Gross Profit': value_list[16],
'EBITDA': value_list[17],
'Net Income Avl to Common ': value_list[18],
'Diluted EPS': value_list[19],
'Earnings Growth': value_list[20],
'Revenue Growth': value_list[21],
'Total Cash': value_list[22],
'Total Cash Per Share': value_list[23],
'Total Debt': value_list[24],
'Current Ratio': value_list[25],
'Book Value Per Share': value_list[26],
'Cash Flow': value_list[27],
'Beta': value_list[28],
'Held by Insiders': value_list[29],
'Held by Institutions': value_list[30],
'Shares Short (as of': value_list[31],
'Short Ratio': value_list[32],
'Short % of Float': value_list[33],
'Shares Short (prior ': value_list[34],
'Current Price': value_list[35],
'Shares Outstanding': value_list[36],
'Status': "N/A"
},
ignore_index=True)
except Exception as e:
print("Problem in DF Append: " + str(e))
print("Data: " + str(each_data) + "File: " +
str(each_file))
print(
'Error on line {}'.format(
sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
time.sleep(5)
#print("DataFrame: ", df)
except Exception as e:
print("Problem as a whole: " + str(e))
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
pass
df.to_csv('YaParse_sample_NO_NA.csv')
print("Done making csv")
print("No. of files passed: " + str(passfiles))
def current_stock_value(self):
symbol_f = self.symbol
data = Share(symbol_f)
share_value = (data.get_open())
return float(share_value) * float(self.shares)
def share_buy(user, shareID, nbuy):
#check shareID exists
try:
try:
from yahoo_finance import Share
company = Share(shareID)
price = float(company.get_price())
except:
price = 0
#if share price exists and non-zero then proceed
if price > 0:
import MySQLdb
db = MySQLdb.connect("localhost", u, p, "TAKESTOCKDB")
#get wallet data, add shareID if required
try:
cursor = db.cursor()
sql = """SELECT %s FROM WALLET WHERE USER = '******'""" % (
share_defeature(shareID), user)
cursor.execute(sql)
except:
db.commit()
db.close()
share_add(shareID)
share_buy(user, shareID, nbuy)
return
shares1 = cursor.fetchone()
shares1 = shares1[0]
sql = """SELECT CREDITS FROM WALLET WHERE USER = '******'""" % (user)
cursor.execute(sql)
credits1 = cursor.fetchone()
credits1 = credits1[0]
#get share price
price = share_price(shareID)
#valid trade request
if nbuy * price <= credits1:
status = 'success'
credits2 = credits1 - (nbuy * price)
shares2 = shares1 + nbuy
sql = """UPDATE WALLET SET CREDITS=%f, %s=%d WHERE USER = '******'""" % (
credits2, share_defeature(shareID), shares2, user)
cursor.execute(sql)
db.commit()
sql = """INSERT INTO LOG(USER, COMMAND, COMPANY, NSHARES, PRICE, STATUS)
VALUES ('%s', 'buy', '%s', %d, %f, '%s')""" % (
user, shareID, nbuy, price, status)
cursor.execute(sql)
db.commit()
db.close()
subj = 'Purchase success: ' + str(
nbuy) + ' ' + shareID + ' shares'
body = query(user)
send_email(user, subj, body)
else:
status = 'fail'
sql = """INSERT INTO LOG(USER, COMMAND, COMPANY, NSHARES, PRICE, STATUS)
VALUES ('%s', 'buy', '%s', %d, %f, '%s')""" % (
user, shareID, nbuy, price, status)
cursor.execute(sql)
db.commit()
db.close()
subj = 'Purchase fail: ' + str(
nbuy) + ' ' + shareID + ' shares'
body = query(user)
send_email(user, subj, body)
else:
print 'shareID error'
send_email(suemail1, 'sql problem', 'from a buy command')
except:
user_add(user)
share_buy(user, shareID, nbuy)
return
from yahoo_finance import Share
#yahoo = Share('YHOO')
#yahoo = Share('SPXC')
yahoo = Share('TFM')
#yahoo = Share('INDU')
#INDEXSP
#yahoo = Share('NDX')
print yahoo
print yahoo.get_open()
#'36.60'
print yahoo.get_price()
print yahoo.get_price_earnings_ratio()
print 'get_dividend_share: ',yahoo.get_dividend_share()
print 'get_dividend_yield: ',yahoo.get_dividend_yield()
print 'get_earnings_share: ',yahoo.get_earnings_share()
print 'get_price_earnings_ratio: ',yahoo.get_price_earnings_ratio()
print 'get_price_earnings_growth_ratio: ',yahoo.get_price_earnings_growth_ratio()
print 'get_year_high: ',yahoo.get_year_high()
print 'get_year_low: ',yahoo.get_year_low()
print 'get_days_high: ',yahoo.get_days_high()
print 'get_days_low: ',yahoo.get_days_low()
print 'get_ebitda: ',yahoo.get_ebitda()
print 'get_book_value: ',yahoo.get_book_value()
#'36.84'
#print yahoo.get_trade_datetime()
#'2014-02-05 20:50:00 UTC+0000'
#get_avg_daily_volume()
'BDCS',
'BDD',
'BFOR',
]
etf_data = []
for index, ETF in enumerate(ETF_master_list):
etf_dict = {
'model': 'portfolio.ETF',
'pk': index + 1,
'fields': {},
}
fund = Share(ETF)
fields = {
'name': fund.get_name(),
'symbol': ETF,
'last_trade': fund.get_price(),
'dividend_yield': fund.get_dividend_yield(),
'absolute_change': fund.get_change(),
'percentage_change': fund.get_percent_change(),
'year high': fund.get_year_high(),
'year low': fund.get_year_low(),
'50 day moving average': fund.get_50day_moving_avg(),
'200 day moving average': fund.get_200day_moving_avg(),
'average_daily_volume': fund.get_avg_daily_volume()
}
