def readcsv(self, csvFileName):
stockData = StockData()
try:
data = pd.read_csv("Stock Histories/" + str(csvFileName) + ".csv", index_col='date', parse_dates=['date'])
except OSError as e:
stockData.findStockData(str(csvFileName))
data = pd.read_csv("Stock Histories/" + str(csvFileName) + ".csv", index_col='date', parse_dates=['date'])
return data
def check_month_change(self,udate):
data = StockData('INFY',self)
mydate = date.today()
#html = data.get_historical_data(mydate.strftime('%d-02-%Y'),mydate.strftime('%d-%m-%Y'),'m')
html = data.get_today_data()
line =html.split('\n')
line = line[1].split(',')
data_date = int(line[0].split('-')[1])
val = False
if data_date - udate > 0:
val= True
return val
def update_monthly(self):
count = 0
with open(self.index) as myfile:
for line in myfile:
if count is not 0:
line = line.split(',')
try:
stock = StockData(line[2],self)
stock.save_historical_data('1-1-1976',self.date.strftime('%d-%m-%Y'),'m')
except:
continue
#self.update_hlc(line[2])
else :
count = 1
def createSimpleChartDataSetWith2TimePoints(self):
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_long_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.set_finished_storing()
return test_stock_data1
def write_stock_data_to_file(stock,
stock_data_tuple_list,
period=Constants.MONTH):
if stock is None:
return
if Utility.is_empty(stock_data_tuple_list):
return
file_name = get_stock_data_file_name(stock, period)
field_name_tuple = tuple(
("date", "open", "high", "low", "close", "volume", "roi"))
with open(file_name, 'w', newline='') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=field_name_tuple)
writer.writeheader()
for stock_data_tuple in stock_data_tuple_list:
stock_data = StockData(stock_data_tuple)
if stock_data is None:
continue
stock_data_dict = {
"date": stock_data.date,
"open": stock_data.open,
"high": stock_data.high,
"low": stock_data.low,
"close": stock_data.close,
"volume": stock_data.volume,
"roi": stock_data.roi
}
writer.writerow(stock_data_dict)
def pull_stock_data(self, ticker):
# initialize the stock object
stock = StockData(ticker)
# scrape yahoo finance
result = self.__yfScraper.get_data(stock)
# scraper market watch (only if yahoo finance successful
result = result and self.__mwScraper.get_data(stock)
# return stock only if both YF and MW successful
return stock if result else None
def createSimpleChartDataSetWith2TimePoints(self):
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_long_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.set_finished_storing()
return test_stock_data1
def __init__(self):
self.scheduler = BlockingScheduler()
self.pool = None
if (platform.system() == 'Windows'):
self.stockDeal = Trader_gxzq(no=TDX_USER,pwd=TDX_PWD,dimpwd=TDX_DIMPWD)
self.isopen = False
self.localData = LocalData.getInstance()
self.stockData = StockData.getInstance()
self.tdxData = TdxData.getInstance()
self.mailUtil = MailUtil.getInstance()
def testAdd1HistoricalData(self):
test_stock_data = StockData(self.test_contract)
test_stock_data.set_is_storing_long_term()
test_stock_data.add_historical_data_point(self.price_1, self.dt_1)
test_chart_data_set = test_stock_data.get_historical_chart_data_set()
result_prices = test_chart_data_set.get_prices()
expected_prices = [self.price_1]
self.assertEqual(result_prices, expected_prices)
def testGoogleData():
framerate = 1
s = StockData("GOOG", [2010, 1, 1], [2012, 1, 1])
stockData = s.getStockData("Adj Close")
fig = pyplot.figure()
ax = fig.add_subplot(2,1,1)
pyplot.title("Google Adjusted Close Data Over 2 Year")
bx = fig.add_subplot(2,1,2)
pyplot.title("Google Adjusted Close Spectrum 2 Year")
# bx.set_xscale('log')
bx.set_yscale('log')
wave = synthesize(stockData, framerate)
spectrum = wave.make_spectrum()
# print np.polyfit(range(len(spectrum.hs)), abs(spectrum.hs), 1)
ax.plot(wave.ys)
bx.plot(abs(spectrum.hs))
pyplot.show()
def testAdd2HistoricalData(self):
test_stock_data = StockData(self.test_contract)
test_stock_data.set_is_storing_long_term()
test_stock_data.add_historical_data_point(self.price_1, self.dt_1)
test_stock_data.add_historical_data_point(self.price_2, self.dt_2)
test_chart_data_set = test_stock_data.get_historical_chart_data_set()
result_prices = test_chart_data_set.get_prices()
expected_prices = np.array([self.price_1, self.price_2])
self.check_array(expected_prices, result_prices)
def setup_stocks_data(stocks):
global ticks_data, stocks_data, errs_data
for stock in stocks:
stock_contract = create_stock_contract(stock)
stock_data = StockData(stock_contract)
stocks_data.append(stock_data)
ticks_data.append(None)
errs_data.append(None)
ticks_data = np.array(ticks_data)
errs_data = np.array(errs_data)
def testAdd1HistoricalData(self):
test_stock_data = StockData(self.test_contract)
test_stock_data.set_is_storing_long_term()
test_stock_data.add_historical_data_point(self.price_1, self.dt_1)
test_chart_data_set = test_stock_data.get_historical_chart_data_set()
result_prices = test_chart_data_set.get_prices()
expected_prices = [self.price_1]
self.assertEqual(result_prices, expected_prices)
def testAdd2HistoricalData(self):
test_stock_data = StockData(self.test_contract)
test_stock_data.set_is_storing_long_term()
test_stock_data.add_historical_data_point(self.price_1, self.dt_1)
test_stock_data.add_historical_data_point(self.price_2, self.dt_2)
test_chart_data_set = test_stock_data.get_historical_chart_data_set()
result_prices = test_chart_data_set.get_prices()
expected_prices = np.array([self.price_1, self.price_2])
self.check_array(expected_prices, result_prices)
def calPred(stkname):
df = stkdt.findStockData(stkname)
num_data = len(df)
df = df.reset_index()
copy_df = pd.DataFrame(df)
dt = df['Date']
df = df[['Close', 'Volume']]
pred_len = int(120)
df['Prediction'] = df[['Close']].shift(-pred_len)
x = np.array(df.drop(['Prediction'], 1))
x = preprocessing.scale(x)
x_pred = x[-pred_len:]
x = x[:-pred_len]
y = np.array(df['Prediction'])
y = y[:-pred_len]
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
clf = LinearRegression(learning_rate=1)
clf.train(x_train, y_train)
price_prediction = clf.predict(x_pred)
df.dropna(inplace=True)
df['Prediction'] = np.nan
last_date = pd.to_datetime(dt.iloc[-1])
last_sec = last_date.timestamp()
one_day_sec = 86400
next_sec = last_sec + one_day_sec
for i in price_prediction:
next_date = datetime.datetime.fromtimestamp(next_sec)
datetime.datetime.fromtimestamp(next_sec)
next_sec += 86400
df.loc[next_date] = [np.nan for _ in range(len(df.columns) - 1)] + [i]
df = pd.DataFrame(df)
pred_df = df[-120:]
copy_df = copy_df.set_index('Date')
pred_df = pd.DataFrame(pred_df['Prediction'])
return pred_df, copy_df
def main(argv=None):
global options
options = parseCommandLine()
startTime = time()
results = []
printResult = True # print only results of first iteration
fileMode = "w"
for i in range(2000):
inv = globals()[options.strategy]()
start = 85
end = -1
with open(options.dataFile) as dataFile:
with open(options.outputFile, fileMode) as outputFile:
data = dataFile.readlines()
name, date, openPrice, highPrice, lowPrice, firstClosePrice, vol = data[
start].split("\t")
name, date, openPrice, highPrice, lowPrice, lastClosePrice, vol = data[
end].split("\t")
firstClosePrice, lastClosePrice = float(
firstClosePrice), float(lastClosePrice)
for line in data[start:end]:
if line[0] == "<":
continue
name, date, openPrice, highPrice, lowPrice, closePrice, vol = line.split(
"\t")
data = StockData(name, date, openPrice, highPrice,
lowPrice, closePrice, vol)
#print date, closePrice
if printResult:
outputFile.write(repr(inv.next(data)))
else:
inv.next(data)
results.append(inv.getBalance())
printResult = False
fileMode = "r"
print "Financial result", sum(results) / float(len(results))
print "Price increase", 1000 * (lastClosePrice / firstClosePrice)
if options.verbose:
print "Execution time", time() - startTime
def setup_roi(stock_data_tuple_list, financial_data_tuple_list):
if Utility.is_empty(stock_data_tuple_list):
return
if Utility.is_empty(financial_data_tuple_list):
return
j = 0
for i in range(len(stock_data_tuple_list)):
stock_data = StockData(stock_data_tuple_list[i])
price = stock_data.get_close()
if price == 0:
continue
while j < len(financial_data_tuple_list):
financial_data = FinancialData(financial_data_tuple_list[j])
if datetime.strptime(stock_data.get_date(),
Constants.DATE_FORMAT) >= datetime.strptime(
financial_data.get_date(),
Constants.DATE_FORMAT):
pe = round(
100.0 * financial_data.get_net_profit_per_share_in_year() /
price, Constants.DOUBLE_FIXED_DECIMAL)
pb = 0
if financial_data.get_book_value_per_share() != 0:
pb = round(
price / financial_data.get_book_value_per_share(),
Constants.DOUBLE_FIXED_DECIMAL)
# roi = round(financial_data.rate * financial_data.roe * pe * Constants.ROI_COEFFICIENT,
# Constants.DOUBLE_FIXED_DECIMAL)
roi = round(
financial_data.roe * pe * Constants.ROI_COEFFICIENT,
Constants.DOUBLE_FIXED_DECIMAL)
if roi < 0:
roi = 0
stock_data.set_pe(pe)
stock_data.set_pb(pb)
stock_data.set_roi(roi)
stock_data_tuple_list[i] = stock_data.to_tuple(include_id=True)
break
else:
j += 1
def testCalculateParamsSTAndLTBetasOn2Stocks(self):
stocks_data = []
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_long_term()
test_stock_data1.add_historical_data_point(50, self.dt_1)
test_stock_data1.add_historical_data_point(60, self.dt_2)
test_stock_data1.set_finished_storing()
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(20, self.dt_1)
test_stock_data1.add_historical_data_point(30, self.dt_2)
stocks_data.append(test_stock_data1)
test_stock_data2 = StockData(self.test_contract)
test_stock_data2.set_is_storing_long_term()
test_stock_data2.add_historical_data_point(10, self.dt_1)
test_stock_data2.add_historical_data_point(12, self.dt_2)
test_stock_data2.set_finished_storing()
test_stock_data2.set_is_storing_short_term()
test_stock_data2.add_historical_data_point(40, self.dt_1)
test_stock_data2.add_historical_data_point(60, self.dt_2)
stocks_data.append(test_stock_data2)
RunHFTModel.calculate_params(stocks_data)
def testTruncateTickSeries(self):
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.add_historical_data_point(70, "20140511 19:15:26")
test_stock_data1.add_historical_data_point(80, "20140511 19:15:27")
test_stock_data1.add_historical_data_point(90, "20140511 19:15:28")
test_stock_data1.add_historical_data_point(100, "20140511 19:15:29")
test_stock_data1.set_finished_storing()
length = 5
chart_ds = test_stock_data1.get_historical_short_term_chart_data_set()
tick_series = chart_ds.get_prices()
new_tick_series = RunHFTModel.truncate_tick_series(tick_series, length)
expected = [60, 70, 80, 90, 100]
self.check_array(expected, new_tick_series)
import pyopencl as cl
from pyopencl import array
import numpy
import pandas as pd
import os
from StockData import StockData
if __name__ == "__main__":
#a array of values we'll pass to the kernel to perform computations to
os.environ['PYOPENCL_COMPILER_OUTPUT'] = '1'
analyst = StockData('goog.csv')
closing_value = numpy.array(analyst.getClosingValue(),dtype=numpy.float32)
rsi_14_day = numpy.array(analyst.getRSIArray(),dtype=numpy.float32)
sam_50_day = numpy.array(analyst.getSMA_50_day(), dtype=numpy.float32)
sam_25_day = numpy.array(analyst.getSMA_25_day(), dtype=numpy.float32)
a = range(35*36)
## Step #1. Obtain an OpenCL platform.
platform = cl.get_platforms()[0]
## It would be necessary to add some code to check the check the support for
## the necessary platform extensions with platform.extensions
## Step #2. Obtain a device id for at least one device (accelerator).
device = platform.get_devices()[0]
## It would be necessary to add some code to check the check the support for
## the necessary device extensions with device.extensions
# Established by Jianmin MAO
# Phone/WeChat: 18194038783
# QQ: 877245759
"""
This file contains all the testing for the functionality of StockData class
"""
from StockData import StockData
# Please go through all the sub-questions one by one and check the results
# E1.1 StockData.__init__(path)
data_path = r'.\training\data'
myDataManager = StockData(data_path)
# E1.2 StockData.read(symbols)
myDataManager.read(['000001', '000002'])
# E1.3 StockData.get_data_by_symbol(symbol, start_date, end_date)
myDataManager.get_data_by_symbol('000001', 19990301, 19990602).head(10)
# E1.4 StockData.get_data_by_date(adate, symbols):
myDataManager.get_data_by_date(19990323, ['000001', '000002'])
# E1.5 StockData.get_data_by_filed(field, symbols):
myDataManager.get_data_by_field('S_DQ_OPEN', ['000001', '000002'])
# E2.1 StockData.format_date(symbol)
myDataManager.format_date('000001')
myDataManager.raw_data['000001']['TRADE_DT']
# E2.2 StockData.plot(symbol, field)
clf = LinearRegression(learning_rate=1)
clf.train(x_train, y_train)
price_prediction = clf.predict(x_pred)
df.dropna(inplace=True)
df['Prediction'] = np.nan
last_date = pd.to_datetime(dt.iloc[-1])
last_sec = last_date.timestamp()
one_day_sec = 86400
next_sec = last_sec + one_day_sec
for i in price_prediction:
next_date = datetime.datetime.fromtimestamp(next_sec)
datetime.datetime.fromtimestamp(next_sec)
next_sec += 86400
df.loc[next_date] = [np.nan for _ in range(len(df.columns) - 1)] + [i]
df = pd.DataFrame(df)
pred_df = df[-120:]
copy_df = copy_df.set_index('Date')
pred_df = pd.DataFrame(pred_df['Prediction'])
return pred_df, copy_df
pred_df, df = calPred('ADANIPORTS')
stkdt.storeData(pred_df)
plt.plot(df, pred_df, "Stock Price Prediction of RELIANCE", 'Date', 'Price',
'blue')
from StockData import StockData
import os
path = os.getcwd()
data_path = os.path.join(path + '\\data\\')
if __name__ == '__main__':
st = StockData(data_path)
stock_pool = ['000021', '000022', '000003', '000002', '300393', '600626']
st.read(stock_pool)
df = st.get_data_by_symbol('000021', '20100101', '20100901')
print(df.head())
df = st.get_adate_symbol('20100104', ['000021', '000003', '000002'])
print(df.head())
df = st.get_data_by_field('open', ['000021', '000003', '000002', '300393'])
print(df.head())
# It's an adjustment price function and you only need call it once
# before computing all indicators of certain stock you need.
# Because if store = True, then it would be stored in self.
st.adjust_data('000021', True, 'forwards')
st.plot('000021', 'close')
df = st.resample('000021', 5)
df = st.resample('300393', 20)
print(df.head())
for window in [5, 20, 60]:
st.moving_average(
'000021',
'close',
True,
window=window,
)
st.atr('000003', window=window)
def testCalculateParamsSTAndLTBetasOn2Stocks(self):
stocks_data = []
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_long_term()
test_stock_data1.add_historical_data_point(50, self.dt_1)
test_stock_data1.add_historical_data_point(60, self.dt_2)
test_stock_data1.set_finished_storing()
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(20, self.dt_1)
test_stock_data1.add_historical_data_point(30, self.dt_2)
stocks_data.append(test_stock_data1)
test_stock_data2 = StockData(self.test_contract)
test_stock_data2.set_is_storing_long_term()
test_stock_data2.add_historical_data_point(10, self.dt_1)
test_stock_data2.add_historical_data_point(12, self.dt_2)
test_stock_data2.set_finished_storing()
test_stock_data2.set_is_storing_short_term()
test_stock_data2.add_historical_data_point(40, self.dt_1)
test_stock_data2.add_historical_data_point(60, self.dt_2)
stocks_data.append(test_stock_data2)
RunHFTModel.calculate_params(stocks_data)
def testBridgeBootstrap(self):
# Store historical ticks
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.set_finished_storing()
stocks_data = [test_stock_data1]
# Store incoming ticks
p1 = 20
p2 = 21
p3 = 22
p4 = 23
p5 = 24
dt1 = datetime.strptime("2014-05-11 19:15:23",
DataType.DATE_TIME_FORMAT_LONG)
dt2 = datetime.strptime("2014-05-11 19:15:24",
DataType.DATE_TIME_FORMAT_LONG)
dt3 = datetime.strptime("2014-05-11 19:15:25",
DataType.DATE_TIME_FORMAT_LONG)
dt4 = datetime.strptime("2014-05-11 19:15:26",
DataType.DATE_TIME_FORMAT_LONG)
dt5 = datetime.strptime("2014-05-11 19:15:27",
DataType.DATE_TIME_FORMAT_LONG)
tick_1 = np.array([dates.date2num(dt1), p1])
tick_2 = np.array([dates.date2num(dt2), p2])
tick_3 = np.array([dates.date2num(dt3), p3])
tick_4 = np.array([dates.date2num(dt4), p4])
tick_5 = np.array([dates.date2num(dt5), p5])
# Create test ticks data.
tick_series = np.array(tick_1)
tick_series = np.vstack([tick_series, tick_2])
tick_series = np.vstack([tick_series, tick_3])
tick_series = np.vstack([tick_series, tick_4])
tick_series = np.vstack([tick_series, tick_5])
ticks_data = [tick_series]
# Bridge ticks and retrieve results
end_time = datetime.strptime("2014-05-11 19:15:35",
DataType.DATE_TIME_FORMAT_LONG)
RunHFTModel.bridge_historical_and_present_ticks(
stocks_data, ticks_data, end_time)
chart_ds = stocks_data[0].get_historical_short_term_chart_data_set()
# Test for prices
result_prices = chart_ds.get_prices()
expected_prices = [50, 60, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24]
self.check_array(expected_prices, result_prices)
# Test date labels
expected_datelabels = [
'20140511 19:15:24', '20140511 19:15:25', '20140511 19:15:26',
'20140511 19:15:27', '20140511 19:15:28', '20140511 19:15:29',
'20140511 19:15:30', '20140511 19:15:31', '20140511 19:15:32',
'20140511 19:15:33', '20140511 19:15:34', '20140511 19:15:35'
]
result_datelabels = chart_ds.get_dates_labels()
self.check_array(expected_datelabels, result_datelabels)
def testStandardDeviationOnHistoricalShortTermData(self):
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.add_historical_data_point(70, "20140511 19:15:26")
test_stock_data1.add_historical_data_point(80, "20140511 19:15:27")
test_stock_data1.add_historical_data_point(90, "20140511 19:15:28")
test_stock_data1.add_historical_data_point(110, "20140511 19:15:29")
test_stock_data1.set_finished_storing()
stdev = test_stock_data1.get_short_term_std()
expected = 0.039971566 * 100
self.assertAlmostEqual(expected, stdev, 5)
def testTruncateTickSeries(self):
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.add_historical_data_point(70, "20140511 19:15:26")
test_stock_data1.add_historical_data_point(80, "20140511 19:15:27")
test_stock_data1.add_historical_data_point(90, "20140511 19:15:28")
test_stock_data1.add_historical_data_point(100, "20140511 19:15:29")
test_stock_data1.set_finished_storing()
length = 5
chart_ds = test_stock_data1.get_historical_short_term_chart_data_set()
tick_series = chart_ds.get_prices()
new_tick_series = RunHFTModel.truncate_tick_series(tick_series, length)
expected = [60, 70, 80, 90, 100]
self.check_array(expected, new_tick_series)
import pyopencl as cl
from pyopencl import array
import numpy
import pandas as pd
import os
from StockData import StockData
if __name__ == "__main__":
#a array of values we'll pass to the kernel to perform computations to
os.environ['PYOPENCL_COMPILER_OUTPUT'] = '1'
analyst = StockData('goog.csv')
closing_value = numpy.array(analyst.getClosingValue(), dtype=numpy.float32)
rsi_14_day = numpy.array(analyst.getRSIArray(), dtype=numpy.float32)
sam_50_day = numpy.array(analyst.getSMA_50_day(), dtype=numpy.float32)
sam_25_day = numpy.array(analyst.getSMA_25_day(), dtype=numpy.float32)
a = range(35 * 36)
## Step #1. Obtain an OpenCL platform.
platform = cl.get_platforms()[0]
## It would be necessary to add some code to check the check the support for
## the necessary platform extensions with platform.extensions
## Step #2. Obtain a device id for at least one device (accelerator).
device = platform.get_devices()[0]
## It would be necessary to add some code to check the check the support for
## the necessary device extensions with device.extensions
def testStandardDeviationOnHistoricalShortTermData(self):
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.add_historical_data_point(70, "20140511 19:15:26")
test_stock_data1.add_historical_data_point(80, "20140511 19:15:27")
test_stock_data1.add_historical_data_point(90, "20140511 19:15:28")
test_stock_data1.add_historical_data_point(110, "20140511 19:15:29")
test_stock_data1.set_finished_storing()
stdev = test_stock_data1.get_short_term_std()
expected = 0.039971566 * 100
self.assertAlmostEqual(expected, stdev, 5)
class Get_Tweets():
def __init__(self):
#self.import_tweets()
self.raw_tweets()
self.clean_tweets()
self.search_tweets()
def connect(self):
#enter your twitter Api consumer key
consumer_key = ''
#enter the consumer secret key
consumer_secret = ''
#enter the access token
access_token = ''
#enter the access toke secret
access_token_secret = ''
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth, wait_on_rate_limit=True)
return api
def import_tweets(self):
# Enter the use account you want the tweets imported from
user = '******'
api = self.connect()
tweets = tweepy.Cursor(api.user_timeline, screen_name=user).items(5)
return tweets
def makingdf(self, firstColum):
try:
tweets = self.import_tweets()
StockTweets = pd.DataFrame(data=[tweet.text for tweet in tweets],
columns=[firstColum])
except tweepy.TweepError:
pass
else:
return StockTweets
def raw_tweets(self):
try:
raw_tweets = self.makingdf('Raw_Tweets')
raw_tweets['TxtBlob'] = raw_tweets['Raw_Tweets'].apply(
self.txtblob_polarity)
raw_tweets['Nltk'] = raw_tweets['Raw_Tweets'].apply(
self.nltk_polarity)
except TypeError:
print("The twitter user account entered not found")
else:
print(raw_tweets.to_string())
def clean_data(self, clean_tweets):
try:
clean_tweets['Clean_Tweets'] = clean_tweets['Clean_Tweets'].apply(
self.cleaning_tweets).apply(
lambda x: self.tokenize_tweets(x.lower()))
clean_tweets['Clean_Tweets'] = clean_tweets['Clean_Tweets'].apply(
self.remove_stopwords).apply(self.remove_special_char)
clean_tweets['TxtBlob'] = clean_tweets['Clean_Tweets'].apply(
self.txtblob_polarity)
clean_tweets['Nltk'] = clean_tweets['Clean_Tweets'].apply(
self.nltk_polarity)
except TypeError:
pass
else:
return clean_tweets
def clean_tweets(self):
try:
clean_tweets = self.makingdf('Clean_Tweets')
clean_tweets = self.clean_data(clean_tweets)
print(clean_tweets.to_string())
except AttributeError:
pass
def search_tweets(self):
try:
StockTweets = self.makingdf('Clean_Tweets')
Stock = 'HNST'
StockTweets['Clean_Tweets'] = StockTweets[
StockTweets['Clean_Tweets'].str.contains(pat=Stock,
case=False)]
StockTweets = StockTweets[StockTweets['Clean_Tweets'].notna()]
StockTweets = self.clean_data(StockTweets)
print(StockTweets.to_string())
stock_Ticker = input(
"Please enter the Stock Ticker Symbol for the data do be fetched(i.e NFLX): "
)
StockData(stock_Ticker)
def write_stock_data_to_database(stock_code,
stock_data_list,
period=Constants.MONTH):
connect = None
record_list = []
delete_sql = StockData.get_delete_sql()
insert_sql = StockData.get_insert_sql()
if Utility.is_empty(stock_data_list):
print("stock_data_list is empty, return")
return
try:
connect = sqlite3.connect(Constants.DATA_DATABASE_ORION_DB)
cursor = connect.cursor()
cursor.execute(delete_sql, (period, stock_code))
for stock_data in stock_data_list:
now = datetime.now().strftime(Constants.DATE_TIME_FORMAT)
if isinstance(stock_data, dict):
stock_data_obj = StockData()
stock_data_obj.set_stock_code(stock_code)
stock_data_obj.set_date(stock_data['date'])
stock_data_obj.set_time("00:00")
stock_data_obj.set_period(period)
stock_data_obj.set_open(stock_data['open'])
stock_data_obj.set_high(stock_data['high'])
stock_data_obj.set_low(stock_data['low'])
stock_data_obj.set_close(stock_data['close'])
stock_data_obj.set_volume(stock_data['volume'])
stock_data_obj.set_created(now)
stock_data_obj.set_modified(now)
elif isinstance(stock_data, tuple):
stock_data_obj = StockData(stock_data)
stock_data_obj.set_modified(now)
record = stock_data_obj.to_tuple(include_id=False)
record_list.append(record)
cursor.executemany(insert_sql, record_list)
connect.commit()
except sqlite3.Error as e:
print('e:', e)
finally:
if connect is not None:
connect.close()
global profit, total_bought, total_sold, stocks_left
if (rsi_14_day[i] > upper_bound_rsi and sma_25_day[i] < sma_50_day[i]):
profit = profit - closing_value[i]
total_bought = total_bought + 1
stocks_left = stocks_left + 1
# print "Buying a volume_of_shares"
elif (rsi_14_day[i] < lower_bound_rsi and sma_25_day[i] > sma_50_day[i]):
if stocks_left > 0:
profit = profit + closing_value[i]
total_sold = total_sold + 1
stocks_left -= 1
# print "Selling a share"
return
analyst = StockData('table.csv')
closing_value = analyst.getClosingValue()
rsi_14_day = analyst.getRSIArray()
sma_50_day = analyst.getSMA_50_day()
sma_25_day = analyst.getSMA_25_day()
max_rsi = analyst.getMaxRSI()
min_rsi = analyst.getMinRSI()
avg_rsi = analyst.getAverageRSI()
no_of_entries = analyst.getNumberOfEntries()
print max_rsi, min_rsi, avg_rsi
for j in range(min_rsi, avg_rsi):
# for j in range(33, 35):
# make sequential index across all data sets
index = [x + index_len for x in macd_data_signal.index]
self.macd_ChartSubplot.plot(index, macd_data_signal, color='blue')
index = [x + index_len for x in macd_data_signal.index]
self.macd_ChartSubplot.plot(index, macd_data, color='red')
index_len = index_len + len(index)
if __name__ == "__main__":
#get filenames for all.csv files in the directory of interest
stock_data_files = get_stock_data_files("minute") # minute or daily data
# put data in list in pandas frame
list_all_stock_data_in_df = []
for filename in stock_data_files:
df_all_data = pd.read_csv(filename) # one day of data
list_all_stock_data_in_df.append(df_all_data)
# for the list of all stock data for each file, calculate all indicators and run the backtest strategies
# in preparation for display of the data
# AllStockData has the list of all raw and and calculated stock market data and indicators
AllStockData = StockData(list_all_stock_data_in_df)
# main window
win = tk.Tk()
win.title("Stock Data BackTest Analysis")
win.resizable(False, False)
app = BaseWindow(win, AllStockData)
win.mainloop()
from StockData import StockData
import matplotlib.pyplot as plt
import numpy as np
import sys
title = ''
nDays = int(sys.argv[1])
nDaysBeforeEnd = int(sys.argv[2])
for arg in sys.argv[3:]:
dataHandle = StockData(arg)
dates, changes, closes = dataHandle.getStockData()
start = len(closes) - nDays
if start < 0:
start = 0
end = len(closes) - nDaysBeforeEnd
data = closes[start:end]
normData = []
normData = np.divide(data, data[0])
normData = normData - normData[0]
plt.plot(dates[start:end], normData, label=arg)
title += arg + ' vs '
title = title[:-4] + ' max normalized closes over time'
plt.hlines(0, 0, len(normData) - 1, 'k')
plt.xticks(rotation='vertical')
plt.title(title)
plt.xlabel('Date')
def testBridgeBootstrap(self):
# Store historical ticks
test_stock_data1 = StockData(create_stock_contract(self.stock))
test_stock_data1.set_is_storing_short_term()
test_stock_data1.add_historical_data_point(50, "20140511 19:15:24")
test_stock_data1.add_historical_data_point(60, "20140511 19:15:25")
test_stock_data1.set_finished_storing()
stocks_data = [test_stock_data1]
# Store incoming ticks
p1 = 20
p2 = 21
p3 = 22
p4 = 23
p5 = 24
dt1 = datetime.strptime("2014-05-11 19:15:23", DataType.DATE_TIME_FORMAT_LONG)
dt2 = datetime.strptime("2014-05-11 19:15:24", DataType.DATE_TIME_FORMAT_LONG)
dt3 = datetime.strptime("2014-05-11 19:15:25", DataType.DATE_TIME_FORMAT_LONG)
dt4 = datetime.strptime("2014-05-11 19:15:26", DataType.DATE_TIME_FORMAT_LONG)
dt5 = datetime.strptime("2014-05-11 19:15:27", DataType.DATE_TIME_FORMAT_LONG)
tick_1 = np.array([dates.date2num(dt1), p1])
tick_2 = np.array([dates.date2num(dt2), p2])
tick_3 = np.array([dates.date2num(dt3), p3])
tick_4 = np.array([dates.date2num(dt4), p4])
tick_5 = np.array([dates.date2num(dt5), p5])
# Create test ticks data.
tick_series = np.array(tick_1)
tick_series = np.vstack([tick_series, tick_2])
tick_series = np.vstack([tick_series, tick_3])
tick_series = np.vstack([tick_series, tick_4])
tick_series = np.vstack([tick_series, tick_5])
ticks_data = [tick_series]
# Bridge ticks and retrieve results
end_time = datetime.strptime("2014-05-11 19:15:35", DataType.DATE_TIME_FORMAT_LONG)
RunHFTModel.bridge_historical_and_present_ticks(stocks_data, ticks_data, end_time)
chart_ds = stocks_data[0].get_historical_short_term_chart_data_set()
# Test for prices
result_prices = chart_ds.get_prices()
expected_prices = [50, 60, 23, 24,
24, 24, 24, 24,
24, 24, 24, 24]
self.check_array(expected_prices, result_prices)
# Test date labels
expected_datelabels = ['20140511 19:15:24'
, '20140511 19:15:25'
, '20140511 19:15:26'
, '20140511 19:15:27'
, '20140511 19:15:28'
, '20140511 19:15:29'
, '20140511 19:15:30'
, '20140511 19:15:31'
, '20140511 19:15:32'
, '20140511 19:15:33'
, '20140511 19:15:34'
, '20140511 19:15:35'
]
result_datelabels = chart_ds.get_dates_labels()
self.check_array(expected_datelabels, result_datelabels)
