def run(initial_m=0, initial_b = 1):
points = dp.get_data() #Retrieves data using the data_parser code.
learning_rate = 0.00005
#initial_b = 1 # initial y-intercept guess
#initial_m = 0 # initial slope guess
num_iterations = 1000
print("Starting gradient descent at b = {0}, m = {1}, error = {2}".format(initial_b, initial_m, compute_total_error(initial_b, initial_m, points)))
print("Running...")
[m, b, error] = gradient_descent_driver(points, initial_b, initial_m, learning_rate, num_iterations, early_stop_number = 5, modify_lr = True)
print("After {0} iterations b = {1}, m = {2}, error = {3}".format(num_iterations, b, m, compute_total_error(b, m, points)))
#plot_line_data(points, m, b)
#plot_error_data(error)
return m,b
def preprocess_data(self, M, mode, data_dir):
if mode == 'europe':
stan_data, regions, start_date, geocode = data_parser.get_data_europe(data_dir, show=False)
weighted_fatalities = np.loadtxt(join(data_dir, 'europe_data', 'weighted_fatality.csv'),
skiprows=1, delimiter=',', dtype=str)
elif mode == 'US_county':
stan_data, regions, start_date, geocode = data_parser.get_data(
M, data_dir, processing=self.processing, state=False, fips_list=self.fips_list,
validation=self.validation_withholding, supercounties=self.supercounties, clustering=self.clustering)
weighted_fatalities = self.get_weighted_fatalities(regions)
# # wf_file = join(self.data_dir, 'us_data', 'weighted_fatality.csv')
# wf_file = join(self.data_dir, 'us_data', 'weighted_fatality_new.csv')
# weighted_fatalities = np.loadtxt(wf_file, skiprows=1, delimiter=',', dtype=str)
# if self.supercounties:
# supercounty_weighted_fatalities = self.load_supercounties_fatalities()
# weighted_fatalities = np.concatenate([weighted_fatalities, supercounty_weighted_fatalities], axis=0)
# # grab just the rows that are in regions, in that order
# weighted_fatalities = weighted_fatalities
# new_weighted_fatalities = []
# for fips in regions:
# new_weighted_fatalities.append(weighted_fatalities[weighted_fatalities[:, 0] == str(fips).zfill(5)][-1])
# weighted_fatalities = np.stack(new_weighted_fatalities)
elif mode == 'US_state':
stan_data, regions, start_date, geocode = data_parser.get_data(
M, data_dir, processing=self.processing, state=True, fips_list=self.fips_list,
validation=self.validation_withholding)
wf_file = join(data_dir, 'us_data', 'state_weighted_fatality.csv')
weighted_fatalities = np.loadtxt(wf_file, skiprows=1, delimiter=',', dtype=str)
self.N2 = stan_data['N2']
return stan_data, regions, start_date, geocode, weighted_fatalities
def main(): #main driver function
points_t = dp.get_data() #Retrieves data using the data_parser code.
points = points_t[:70]
test = points_t[70:]
learning_rate = 0.00005
initial_b = 0 # initial y-intercept guess
initial_m = 0 # initial slope guess
num_iterations = 1000
tuning_rate = 5
for type_i in [None, 'l1', 'l2']:
print("Starting gradient descent at b = {0}, m = {1}, error = {2}".
format(
initial_b, initial_m,
compute_total_error(initial_b,
initial_m,
points,
reg_type=type_i,
tuning_rate=tuning_rate)))
print("Running...")
[m, b, error] = gradient_descent_driver(points,
initial_m,
initial_b,
learning_rate,
num_iterations,
early_stop_number=5,
modify_lr=True,
reg_type=type_i,
tuning_rate=tuning_rate)
print("After {0} iterations b = {1}, m = {2}, error = {3}".format(
num_iterations, b, m,
compute_total_error(b,
m,
points,
reg_type=type_i,
tuning_rate=tuning_rate)))
plot_line_data(points, m, b)
plot_error_data(error)
print(
"Mean Squared error on test after calculating {0} based linear regression using b = {1}, m = {2}, error = {3}"
.format(
type_i, b, m,
compute_total_error(b,
m,
test,
reg_type=None,
tuning_rate=None)))
return
Location of file(s):
1. required to run the program
2. required to save the models & states to
############################################################'''
PREPROCESSED_TRAIN_SET = "preprocessed_training_set.json"
SAVE_MODEL_TO = "models/"
SAVE_STATES_TO = "states/attention/states.hdf5"
SAVE_SCORES_TO = "scores/attention/attention_scores.pkl"
SAVE_LOGS_TO = "TBlogs/"
TRAINING_LOG = "training_performance_log.txt"
'''############################################################
Get data (premise, hypothesis, labels) for training
############################################################'''
premise_hypo_pair, correct_labels = dp.get_data(PREPROCESSED_TRAIN_SET)
X_train = premise_hypo_pair
y_train = to_categorical(correct_labels)
# Shuffle the dataset with different random_state to perform stratified split of training and validation set
X_train, X_val, y_train, y_val = train_test_split(X_train,
y_train,
test_size=0.2,
random_state=1,
stratify=y_train)
'''############################################################
Define & initialize constants for lstm architecture
> constants for network architecture
> for network optimization
############################################################'''
if __name__ == '__main__':
data_dir = sys.argv[1]
N2 = 120
alpha_mu = 0.5
alpha_var = 1
num_alphas = 8
# interventions = get_npis(data_dir)
# regions = get_counties_isolated_NPIs(interventions, 'public schools').values.tolist()
# regions.sort()
# for i in range(len(regions)):
# regions[i] = str(regions[i])
stan_data, regions, start_date, geocode = get_data(
100, 'data', processing=Processing.REMOVE_NEGATIVE_VALUES, state=False)
r0_file_path = join('results', 'real_county', 'summary.csv')
r0_file = pd.read_csv(r0_file_path)
means = r0_file['mean'].values
M = len(geocode)
means = means[0:M]
all_r0 = {}
for r in range(M):
all_r0[str(geocode[r]).zfill(5)] = means[r]
serial_interval = np.loadtxt(join('data', 'us_data',
'serial_interval.csv'),
skiprows=1,
print('\nPreprocessed test set loaded.')
else:
PREPROCESSED_TEST_SET = pre.get_data(
RAW_TEST_DATA, "TEST") # Run preprocessing of test_set
with open('../data/preprocessed_data/preprocessed_test_set.json',
'w') as fp:
json.dump(PREPROCESSED_TEST_SET, fp) # Dump the preprocessed json file
print('\nPreprocessing of Test Set Complete!')
print('File {} saved to {}'.format('preprocessed_test_set.json',
"../data/preprocessed_data/"))
'''################################################
Get data (premise, hypothesis, labels) for training
################################################'''
X_test = dp.get_data(PREPROCESSED_TEST_SET,
"TEST") # Parse and get X_test data
print('\nTest set parsing complete.')
y_test = [] # Read labels from text file
with open(LABELS_FILE, "r", errors='ignore') as test_labels:
for line in test_labels:
y_test.append(line.split(' ')[1])
y_test = to_categorical(y_test) # Categorize labels into binary format
'''################################################
Define & initialize constants for lstm architecture
> constants for network architecture
> for network optimization
################################################'''
#learning_rate = 0.000001
import numpy as np
import matplotlib.pyplot as plt
import data_parser as dp
import gradient_descent as gd
import time
points = dp.get_data()
def compute_total_error(m,b): #Computes total mean squared error
totalError = 0
for i in range(len(points)):
x = points[i,0]
y = points[i,1]
totalError += (y - (m * x + b)) ** 2 #Error is calculated as y' = mx + b(Assuming linear regression) so E = (y-y')^2, summed over all points
return totalError/float(len(points)) #Returning the mean squared error.
def total_error(point_pair): #driver function for compute_total_error
return compute_total_error(point_pair[0], point_pair[1])
def compute_jacobian(point_pair, h = 1e-5): #computes the jacobian of the function total_error
n = len(point_pair)
jacobian = np.zeros(n) #initialize the jacobian matrix
for i in range(n):
x_i = np.zeros(n)
x_i[i] += h #add the limit value, any small value > 0 should do
jacobian[i] = (total_error(point_pair+x_i) - total_error(point_pair))/h #calculate derivative using first principle method f'(x) = lt(h->0) (f(x+h) - f(x))/h
return jacobian #return the jacobian for the pair of points
def compute_hessian(point_pair, h = 1e-5): #computes the hessian of the function total_error, it is found as the derivative of the jacobian
n = len(point_pair)
输入BGR图片，处理为模型输入的灰度图，shape (1, None, None, 1)
"""
dump_process_img('before-meaned', img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img.astype(np.float32)
# img[:, :, 0] -= C.img_channel_mean[0]
# img[:, :, 1] -= C.img_channel_mean[1]
# img[:, :, 2] -= C.img_channel_mean[2]
dump_process_img('meaned', img)
img /= C.img_scaling_factor
img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
dump_process_img('gray', img)
img = np.expand_dims(img, axis=2)
img = np.expand_dims(img, axis=0)
return img
if __name__ == "__main__":
from data_parser import get_data
import config
C = config.Config() #相关配置信息
all_imgs, classes_count, class_mapping = get_data(C)
C.class_mapping = class_mapping
data_gen_train = get_anchor_data_gt(all_imgs,
classes_count,
C,
mode='train')
# for _ in range(20):
next(data_gen_train)
def cum_pl_short():
"""
Chart for back test reports of Cumulative profit and loss
:return: None
"""
data_prop, data = data_parser.get_data(start_date="2000-01-18")
high = data_parser.get_high(data)
low = data_parser.get_low(data)
close = data_parser.get_close(data)
sma = indicators.sma(close)
ema = indicators.ema(close)
macd = indicators.macd(close)
rsi = indicators.rsi(close)
stoch = indicators.stoch(high, low, close)
bbands = indicators.bollinger_bands(close)
pivot = indicators.pivot(data)
chart_1 = ChartElement(data=sma,
label="sma",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.YELLOW)
chart_2 = ChartElement(data=ema,
label="ema",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.PINK)
chart_3 = ChartElement(data=stoch,
label="stoch",
chart_type=ChartType.LINE,
plot=ChartAxis.DIFFERENT_AXIS,
color=ChartColor.PURPLE)
chart_4 = ChartElement(data=bbands,
label="bbands",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.BLUE)
chart_5 = ChartElement(data=pivot,
label="pivot",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.GREEN)
chart_6 = ChartElement(data=rsi,
label="rsi",
chart_type=ChartType.LINE,
plot=1,
color=ChartColor.RED)
chart_7 = ChartElement(data=macd,
label="macd",
chart_type=ChartType.LINE,
plot=2,
color="magenta")
charts = [chart_1, chart_2, chart_3, chart_6]
buy = Condition(data1=sma, data2=ema, operation=Operation.CROSSOVER)
sell = Condition(data1=rsi, data2=70, operation=Operation.GREATER_THAN)
result = strategy.strategy_builder(data_properties=data_prop,
data_list=data,
charts=charts,
buy=buy,
sell=sell,
target=1.0,
sl=0.5,
strategy=strategy.BUY)
cum_short = result['short']['DATE_CUM_PL']
# print(cum_short)
return render_template("cum_pl_short.html", shortData=cum_short)
def chart():
"""
For plotting normal candle chart or along with indicators
:return: None
"""
# prop, data = data_parser.get_data(start_date="2017-08-18")
# result = Strategies.rsi(data, data_properties=prop)
# data_properties = result['data_properties']
# main_chart = []
# for key, values in data_properties.items():
# main_chart.append([key, values])
# params = result['params']
# data = result['data']
# print(params,data_with_indicators)
# final_data = data_with_indicators[1:]
# print(final_data)
data_prop, data = data_parser.get_data(start_date="2007-01-18")
high = data_parser.get_high(data)
low = data_parser.get_low(data)
close = data_parser.get_close(data)
sma = indicators.sma(close)
ema = indicators.ema(close)
macd = indicators.macd(close)
rsi = indicators.rsi(close)
stoch = indicators.stoch(high, low, close)
bbands = indicators.bollinger_bands(close)
pivot = indicators.pivot(data)
chart_1 = ChartElement(data=sma,
label="sma",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.YELLOW)
chart_2 = ChartElement(data=ema,
label="ema",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.PINK)
chart_3 = ChartElement(data=stoch,
label="stoch",
chart_type=ChartType.LINE,
plot=ChartAxis.DIFFERENT_AXIS,
color=ChartColor.PURPLE)
chart_4 = ChartElement(data=bbands,
label="bbands",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.BLUE)
chart_5 = ChartElement(data=pivot,
label="pivot",
chart_type=ChartType.LINE,
plot=ChartAxis.SAME_AXIS,
color=ChartColor.GREEN)
chart_6 = ChartElement(data=rsi,
label="rsi",
chart_type=ChartType.LINE,
plot=1,
color=ChartColor.RED)
chart_7 = ChartElement(data=macd,
label="macd",
chart_type=ChartType.LINE,
plot=2,
color="magenta")
charts = [chart_4, chart_6]
buy = Condition(data1=sma, data2=ema, operation=Operation.CROSSOVER)
sell = Condition(data1=rsi, data2=70, operation=Operation.GREATER_THAN)
result = strategy.strategy_builder(data_properties=data_prop,
data_list=data,
charts=charts,
buy=buy,
sell=sell,
target=1.0,
sl=0.5,
strategy=strategy.BUY)
# strategy.show_back_testing_reports(result, auto_open=True)
data_properties = result['data_properties']
main_chart = []
for key, values in data_properties.items():
main_chart.append([key, values])
params = result['params']
# print(params)
data_list = result['data']
return render_template("chart.html",
chartData=data_list,
chart_params=params,
main_chart_properties=main_chart)