def test_ridge_regression():
stock_d = testdata()
ti = TechnicalIndicators(stock_d)
filename = "test_N225_ridge.pickle"
clffile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "clf", filename)
if os.path.exists(clffile):
os.remove(clffile)
clf = Regression(filename)
ti.calc_ret_index()
ret = ti.stock["ret_index"]
base = ti.stock_raw["Adj Close"][0]
train_X, train_y = clf.train(ret, regression_type="Ridge")
test_y = clf.predict(ret, base)
expected = 19177.97
r = round(test_y[0], 2)
eq_(r, expected)
if os.path.exists(clffile):
os.remove(clffile)
def regression_chicago_crime_weather(p, date_start, date_end, crime_types = None, wards = None):
crime_data = get_chicago_crime_data(p, date_start, date_end, crime_types, wards)
weather_data = get_chicago_weather_data(p, date_start, date_end)
crimes = [x[1] for x in crime_data]
temps = [x[1] for x in weather_data]
fill_missing_values(temps)
r = Regression(temps, u"Temperature (°F)", crimes, get_crimes_label(crime_types))
r.compute()
r.plot()
def run(self):
io = FileIO()
will_update = self.update
if self.csvfile:
stock_tse = io.read_from_csv(self.code, self.csvfile)
msg = "".join(["Read data from csv: ", self.code, " Records: ", str(len(stock_tse))])
print(msg)
if self.update and len(stock_tse) > 0:
index = pd.date_range(start=stock_tse.index[-1], periods=2, freq="B")
ts = pd.Series(None, index=index)
next_day = ts.index[1]
t = next_day.strftime("%Y-%m-%d")
newdata = io.read_data(self.code, start=t, end=self.end)
msg = "".join(["Read data from web: ", self.code, " New records: ", str(len(newdata))])
print(msg)
if len(newdata) < 1:
will_update = False
else:
print(newdata.ix[-1, :])
stock_tse = stock_tse.combine_first(newdata)
io.save_data(stock_tse, self.code, "stock_")
else:
stock_tse = io.read_data(self.code, start=self.start, end=self.end)
msg = "".join(["Read data from web: ", self.code, " Records: ", str(len(stock_tse))])
print(msg)
if stock_tse.empty:
msg = "".join(["Data empty: ", self.code])
print(msg)
return None
if not self.csvfile:
io.save_data(stock_tse, self.code, "stock_")
try:
stock_d = stock_tse.asfreq("B").dropna()[self.days :]
ti = TechnicalIndicators(stock_d)
ti.calc_sma()
ti.calc_sma(timeperiod=5)
ti.calc_sma(timeperiod=25)
ti.calc_sma(timeperiod=50)
ti.calc_sma(timeperiod=75)
ewma = ti.calc_ewma(span=5)
ewma = ti.calc_ewma(span=25)
ewma = ti.calc_ewma(span=50)
ewma = ti.calc_ewma(span=75)
bbands = ti.calc_bbands()
sar = ti.calc_sar()
draw = Draw(self.code, self.name)
ret = ti.calc_ret_index()
ti.calc_vol(ret["ret_index"])
rsi = ti.calc_rsi(timeperiod=9)
rsi = ti.calc_rsi(timeperiod=14)
mfi = ti.calc_mfi()
roc = ti.calc_roc(timeperiod=10)
roc = ti.calc_roc(timeperiod=25)
roc = ti.calc_roc(timeperiod=50)
roc = ti.calc_roc(timeperiod=75)
roc = ti.calc_roc(timeperiod=150)
ti.calc_cci()
ultosc = ti.calc_ultosc()
stoch = ti.calc_stoch()
ti.calc_stochf()
ti.calc_macd()
willr = ti.calc_willr()
ti.calc_momentum(timeperiod=10)
ti.calc_momentum(timeperiod=25)
tr = ti.calc_tr()
ti.calc_atr()
ti.calc_natr()
vr = ti.calc_volume_rate()
ret_index = ti.stock["ret_index"]
clf = Classifier(self.clffile)
train_X, train_y = clf.train(ret_index, will_update)
msg = "".join(["Train Records: ", str(len(train_y))])
print(msg)
clf_result = clf.classify(ret_index)[0]
msg = "".join(["Classified: ", str(clf_result)])
print(msg)
ti.stock.ix[-1, "classified"] = clf_result
reg = Regression(self.regfile, alpha=1, regression_type="Ridge")
train_X, train_y = reg.train(ret_index, will_update)
msg = "".join(["Train Records: ", str(len(train_y))])
base = ti.stock_raw["Adj Close"][0]
reg_result = int(reg.predict(ret_index, base)[0])
msg = "".join(["Predicted: ", str(reg_result)])
print(msg)
ti.stock.ix[-1, "predicted"] = reg_result
if len(self.reference) > 0:
ti.calc_rolling_corr(self.reference)
ref = ti.stock["rolling_corr"]
else:
ref = []
io.save_data(io.merge_df(stock_d, ti.stock), self.code, "ti_")
draw.plot(
stock_d,
ewma,
bbands,
sar,
rsi,
roc,
mfi,
ultosc,
willr,
stoch,
tr,
vr,
clf_result,
reg_result,
ref,
axis=self.axis,
complexity=self.complexity,
)
return ti
except (ValueError, KeyError):
msg = "".join(["Error occured in ", self.code])
print(msg)
return None
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = "../src/pal"
regression = Regression(loader, "Memory")
regression.add_check(name="Memory Allocation", check=lambda res: "Memory Allocation OK" in res[0].log)
regression.add_check(
name="Memory Allocation with Address", check=lambda res: "Memory Allocation with Address OK" in res[0].log
)
regression.add_check(
name="Memory Protection",
check=lambda res: "Memory Allocation Protection (RW) OK" in res[0].log and "Memory Protection (R) OK" in res[0].log,
)
regression.add_check(name="Memory Deallocation", check=lambda res: "Memory Deallocation OK" in res[0].log)
def check_quota(res):
for line in res[0].log:
if line.startswith("Total Memory:"):
return line != "Total Memory: 0"
return False
regression.add_check(name="Get Memory Total Quota", check=check_quota)
from preProcessor import PreProcessor
import argparse
from regression import Regression
from picDrawer import PicDrawer
if __name__ == '__main__':
'''
python run -f [filepath] -s [filepath] -c [stock code]
output:
stock error : implement by regression.score()
picture : implement by drawer
'''
# create pre processor
data_cleaner = PreProcessor()
train_feature, train_label, test_feature, test_label = data_cleaner.run()
reg = Regression()
reg.fit(train_feature, train_label)
pred_result = reg.predict(test_feature)
score = reg.score(test_label, pred_result)
drawer = PicDrawer()
drawer.run()
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = './pal'
# Running Bootstrap
regression = Regression(loader, "bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].out)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].out and \
"argv[0] = file:bootstrap" in res[0].out)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].out and \
"argv[0] = file:bootstrap" in res[0].out and \
"argv[1] = a" in res[0].out and "argv[2] = b" in res[0].out and \
"argv[3] = c" in res[0].out and "argv[4] = d" in res[0].out)
regression.run_checks()
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Symbols")
all_symbols = [
'DkVirtualMemoryAlloc',
'DkVirtualMemoryFree',
'DkVirtualMemoryProtect',
'DkProcessCreate',
'DkProcessExit',
'DkProcessSandboxCreate',
'DkStreamOpen',
'DkStreamWaitForClient',
'DkStreamRead',
'DkStreamWrite',
'DkStreamDelete',
'DkStreamMap',
'DkStreamUnmap',
'DkStreamSetLength',
'DkStreamFlush',
'DkSendHandle',
'DkReceiveHandle',
'DkStreamAttributesQuery',
'DkStreamAttributesQueryByHandle',
'DkStreamAttributesSetByHandle',
'DkStreamGetName',
'DkStreamChangeName',
'DkThreadCreate',
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = '../src/pal'
# Running Bootstrap
regression = Regression(loader, "Bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].log)
regression.add_check(name="Control Block: Executable Name",
check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)
regression.add_check(name="Control Block: Default Manifest",
check=lambda res: "Loaded Manifest: file:manifest" in res[0].log)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].log and \
"argv[0] = file:Bootstrap" in res[0].log)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].log and \
"argv[0] = file:Bootstrap" in res[0].log and \
"argv[1] = a" in res[0].log and "argv[2] = b" in res[0].log and \
"argv[3] = c" in res[0].log and "argv[4] = d" in res[0].log)
regression.add_check(name="Control Block: Debug Stream (Inline)",
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
regression = Regression(loader, "epoll_wait_timeout", None, 50000)
regression.add_check(name="epoll_wait timeout",
args = ['8000'],
check=lambda res: "epoll_wait test passed" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Socket")
regression.add_check(name="TCP Socket Creation",
check=lambda res: "TCP Creation 1 OK" in res[0].log)
regression.add_check(name="TCP Socket Connection",
check=lambda res: "TCP Connection 1 OK" in res[0].log)
regression.add_check(name="TCP Socket Transmission",
check=lambda res: "TCP Write 1 OK" in res[0].log and
"TCP Read 1: Hello World 1" in res[0].log and
"TCP Write 2 OK" in res[0].log and
"TCP Read 2: Hello World 2" in res[0].log)
regression.add_check(name="UDP Socket Creation",
check=lambda res: "UDP Creation 1 OK" in res[0].log)
regression.add_check(name="UDP Socket Connection",
check=lambda res: "UDP Connection 1 OK" in res[0].log)
regression.add_check(name="UDP Socket Transmission",
check=lambda res: "UDP Write 1 OK" in res[0].log and
"UDP Read 1: Hello World 1" in res[0].log and
"UDP Write 2 OK" in res[0].log and
def predict(xlabel,ylabel,x_val,x,y):
meanx=np.mean(x)
stddevx=np.std(x)
meany=np.mean(y)
stddevy=np.std(y)
x=featureNormalize(x)
y=featureNormalize(y)
reg = Regression()
reg.set_learning_rate(0.01)
reg.set_max_iterations(10000)
reg.set_l1_penalty(0.1)
reg.set_l2_penalty(0.1)
reg.set_tolerance(1e-5)
theta, cost, it = reg.polynomial_regression(x, y, 5)
z = np.linspace(-1.9, 2.1, 4/0.01)
prediction = reg.predict(z)
x=np.array(x)*stddevx+meanx
z=np.array(z)*stddevx+meanx
y=np.array(y)*stddevy+meany
prediction=np.array(prediction)*stddevy+meany
x_val=(x_val-meanx)/stddevx
y_val=reg.predict([x_val])
x_val=[x_val]
x_val=np.array(x_val)*stddevx+meanx
y_val=np.array(y_val)*stddevy+meany
fig = plt.figure(figsize=(4,4))
plt.plot(x,y,'.', label='Input data')
plt.plot(z,prediction,'r-', label='Best fit curve')
plt.plot(x_val,y_val,'gx',label='Predicted Data')
plt.legend(loc=4)
title=xlabel + " vs " + ylabel
plt.title(title,size=10)
plt.xticks(size=8)
plt.yticks(size=8)
plt.close('all')
return([y_val,fig])
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running Bootstrap
regression = Regression(loader, "large-mmap", None, 60000)
regression.add_check(name="Ftruncate",
check=lambda res: "large-mmap: ftruncate OK" in res[0].out)
regression.add_check(name="Large mmap",
check=lambda res: "large-mmap: mmap 1 completed OK" in res[0].out and \
"large-mmap: mmap 2 completed OK" in res[0].out)
regression.run_checks()
import matplotlib.animation as animation
from regression import Regression, panda_to_numpy
data_path = "./hollywood.xls"
data = pd.read_excel(data_path)
x_train = data['X2']
y_train = data['X1']
z_train = data['X3']
# y must be an nX1 array
# x myst be an nXm array where m is the number of different variables for the reggression
# regression() returns an 1Xm+1 array wich are the weights +1 is for the constant
x = panda_to_numpy(data, 'X2', 'X3')
y = data['X1']
model = Regression(y, x)
weights = model.train(epochs=200, a=0.0001, print_loss=True)
prediction = model.predict(8, 15)
print(prediction)
# plot the results
ones = np.ones(len(x_train))
x_normalized = np.linspace(x_train.min(), x_train.max(), len(x_train))
z_normalized = np.linspace(z_train.min(), z_train.max(), len(z_train))
x_pred = np.column_stack((x_normalized, z_normalized, ones))
y_pred = weights @ x_pred.T
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_title("Prediction of hollywood movie revenue")
ax.set_zlabel("Revenue")
import pandas as pd
from regression import Regression
x = Regression()
df_predict = pd.read_csv('x_test_v2.csv').fillna(0).astype(int)
# READ DATASET
df = pd.read_csv("trips_summary_covid_pub.csv")
df['date'] = pd.to_datetime(df['date'])
# FILTER PRE AND POST PANDEMIC
df['pre_covid'] = (df.date < "12/19/2020").astype("int")
df_pre_covid = df.loc[df.pre_covid == 1]
df = df.loc[df.pre_covid == 0]
df = df.loc[df['total_revenue'] > 0]
df_pre_covid = df_pre_covid.loc[df_pre_covid['total_revenue'] > 0]
# TOTAL REVENUE MODEL
x.select_cols = ['pickup_community_area', 'hour', 'total_revenue']
x.dummy_cols = ['pickup_community_area', 'hour']
x.y_col = 'total_revenue'
res_revenue = x.poisson_regression(df, 0.7)
# TOTAL REVENUE PRE COVID
x.select_cols = ['pickup_community_area', 'hour', 'total_revenue']
x.y_col = 'total_revenue'
res_revenue_pre = x.poisson_regression(df_pre_covid, 0.7)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Symbols")
all_symbols = [
'DkVirtualMemoryAlloc', 'DkVirtualMemoryFree', 'DkVirtualMemoryProtect',
'DkProcessCreate', 'DkProcessExit', 'DkProcessSandboxCreate',
'DkStreamOpen', 'DkStreamWaitForClient', 'DkStreamRead', 'DkStreamWrite',
'DkStreamDelete', 'DkStreamMap', 'DkStreamUnmap', 'DkStreamSetLength',
'DkStreamFlush', 'DkSendHandle', 'DkReceiveHandle',
'DkStreamAttributesQuery', 'DkStreamAttributesQuerybyHandle',
'DkStreamAttributesSetbyHandle', 'DkStreamGetName', 'DkStreamChangeName',
'DkThreadCreate', 'DkThreadDelayExecution', 'DkThreadYieldExecution',
'DkThreadExit', 'DkThreadResume', 'DkSetExceptionHandler',
'DkExceptionReturn', 'DkSemaphoreCreate', 'DkSemaphoreRelease',
'DkNotificationEventCreate', 'DkSynchronizationEventCreate', 'DkEventSet',
'DkEventClear', 'DkObjectsWaitAny', 'DkObjectReference', 'DkObjectClose',
'DkSystemTimeQuery', 'DkRandomBitsRead', 'DkInstructionCacheFlush',
'DkSegmentRegister', 'DkMemoryAvailableQuota',
'DkCreatePhysicalMemoryChannel', 'DkPhysicalMemoryCommit',
'DkPhysicalMemoryMap'
]
def check_symbols(res):
for sym in all_symbols:
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = '../src/pal'
def manifest_file(file):
if 'SGX_RUN' in os.environ and os.environ['SGX_RUN'] == '1':
return file + '.manifest.sgx'
else:
return file + '.manifest'
# Running Bootstrap
regression = Regression(loader, "Bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].log)
regression.add_check(name="Control Block: Executable Name",
check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].log and \
"argv[0] = file:Bootstrap" in res[0].log)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].log and \
"argv[1] = a" in res[0].log and "argv[2] = b" in res[0].log and \
"argv[3] = c" in res[0].log and "argv[4] = d" in res[0].log)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = '../src/pal'
regression = Regression(loader, "Misc", timeout=5000)
regression.add_check(name="Query System Time",
check=lambda res: "Query System Time OK" in res[0].log)
regression.add_check(name="Delay Execution for 10000 Microseconds",
check=lambda res: "Delay Execution for 10000 Microseconds OK" in res[0].log)
regression.add_check(name="Delay Execution for 3 Seconds",
check=lambda res: "Delay Execution for 3 Seconds OK" in res[0].log)
regression.add_check(name="Generate Random Bits",
check=lambda res: "Generate Random Bits OK" in res[0].log)
regression.run_checks()
def predict2(x_val,x,y):
meanx=np.mean(x)
stddevx=np.std(x)
meany=np.mean(y)
stddevy=np.std(y)
x=featureNormalize(x)
y=featureNormalize(y)
reg = Regression()
reg.set_learning_rate(0.001)
reg.set_max_iterations(10000)
reg.set_l1_penalty(0.1)
reg.set_l2_penalty(0.1)
reg.set_tolerance(1e-5)
theta, cost, it = reg.polynomial_regression(x, y, 5)
x_val=(x_val-meanx)/stddevx
y_val=reg.predict([x_val])
y_val=np.array(y_val)*stddevy+meany
return(y_val)
## This test is specifically for the reference monitor code, not process creation in general.
## It is not well-tested right now, but keep the tests around for future work.
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_SEC']
if not os.path.exists(loader):
print("Reference monitor mode is not available on this platform")
exit(0)
regression = Regression(loader, "Process")
def check_times(target, lines, times):
count = 0
for line in lines:
if target == line:
count += 1
return count == times
regression.add_check(name="Process Creation",
check=lambda res: check_times("Child Process Created", res[0].log, 3))
regression.add_check(name="Process Creation Arguments",
check=lambda res: check_times("argv[0] = Process", res[0].log, 3) and
check_times("argv[1] = Child", res[0].log, 3))
regression.add_check(name="Process Channel Transmission",
check=lambda res: check_times("Process Write 1 OK", res[0].log, 3) and
check_times("Process Read 1: Hello World 1", res[0].log, 3) and
if os.path.exists("dir_delete.tmp"):
shutil.rmtree("dir_delete.tmp")
global dir_files
os.mkdir("dir_exist.tmp")
dir_files = []
for i in range(5):
file = ''.join([random.choice(string.ascii_letters) for i in range(8)])
f = open("dir_exist.tmp/" + file, "w")
f.close()
dir_files.append(file)
os.mkdir("dir_delete.tmp")
regression = Regression(loader, "Directory", prepare_dirs)
regression.add_check(name="Basic Directory Opening",
check=lambda res: "Directory Open Test 1 OK" in res[
0].log and "Directory Open Test 2 OK" in res[0].log
and "Directory Open Test 3 OK" in res[0].log)
regression.add_check(name="Basic Directory Creation",
check=lambda res: "Directory Creation Test 1 OK" in res[
0].log and "Directory Creation Test 2 OK" in res[0].
log and "Directory Creation Test 3 OK" in res[0].log)
def check_read(res):
global dir_files
for file in dir_files:
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running getsockopt
regression = Regression(loader, "getsockopt", None)
regression.add_check(name="getsockopt",
check=lambda res: "getsockopt: Got socket type OK" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
Month=request.form['Month'])
yearList, avgTempList = dataFrame.SplitYearAvgTemp(p)
avgTempList = np.array(avgTempList).astype(np.float)
yearList = np.array(yearList).astype(np.float)
training_x, training_y, test_x, test_y, predictLine_x, predictLine_y, cofficient, intercept = regression.fnLinearRegression1(
yearList, avgTempList)
return dumps({
"training_x": training_x,
"training_y": training_y,
"test_x": test_x,
"test_y": test_y,
"predictLine_x": predictLine_x,
"predictLine_y": predictLine_y,
"cofficient": cofficient,
"intercept": intercept
})
except:
return "error"
if __name__ == '__main__':
global dataFrame
global regression
global classification
dataFrame = DataParser()
regression = Regression()
classification = Classification()
visualization = visualizations.Visualization()
app.run()
def main():
#read input data
my_data = np.genfromtxt('regression_data.txt', dtype=float, delimiter=',')
# create input and output numpy arrays
x = my_data[:,0]
y = my_data[:,1]
# create regression class object
reg = Regression()
# set learning rate
reg.set_learning_rate(0.001)
# set maximum iterations
reg.set_max_iterations(20000)
# set l1 and l2 penalty
reg.set_l1_penalty(0.1)
reg.set_l2_penalty(0.1)
# set tolerance
reg.set_tolerance(1e-5)
# fit a polynomial regression model
theta, cost, it = reg.polynomial_regression(x, y, 6)
print "Regression coefficients :" + str(theta)
print "Minimum cost function: " + str(cost)
print "Iterations taken: " + str(it)
# predict values for new input
z = np.linspace(-2, 2, 4/0.01)
prediction = reg.predict(z)
# plot
fig = plt.figure()
plt.plot(x,y,'.', label='Input data')
plt.plot(z,prediction,'r-', label='Prediction')
plt.legend(loc=4)
fig.suptitle('Polynomial Regression Fit')
plt.xlabel('x (input)')
plt.ylabel('y (predicted)')
plt.savefig('fit_values.eps')
plt.show()
#!/usr/bin/env python2
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Misc", timeout=5000)
regression.add_check(name="Query System Time",
check=lambda res: "Query System Time OK" in res[0].log)
regression.add_check(name="Delay Execution for 10000 Microseconds",
check=lambda res:
"Delay Execution for 10000 Microseconds OK" in res[0].log)
regression.add_check(
name="Delay Execution for 3 Seconds",
check=lambda res: "Delay Execution for 3 Seconds OK" in res[0].log)
regression.add_check(name="Generate Random Bits",
check=lambda res: "Generate Random Bits OK" in res[0].log)
rv = regression.run_checks()
if rv: sys.exit(rv)
regression = Regression(loader, "Hex")
regression.add_check(name="Hex 2 String Helper Function",
check=lambda res: "Hex test 1 is deadbeef" in res[0].log and \
"Hex test 2 is cdcdcdcdcdcdcdcd" in res[0].log)
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation
from regression import Regression, panda_to_numpy
data_path = "./biocarbonate.xls"
data = pd.read_excel(data_path)
x_train = data['X']
y_train = data['Y']
# y must be an nX1 array
# x myst be an nXm array where m is the number of different variables for the reggression
# regression() returns an 1Xm+1 array wich are the weights +1 is for the constant
x = panda_to_numpy(data, 'X')
y = data['Y']
model = Regression(y, x)
weights = model.train(epochs=200000, a=0.00005, print_loss=False)
prediction = model.predict(8)
print(prediction)
# plot the results
ones = np.ones(len(x_train))
x_pred = np.column_stack((np.linspace(x_train.min(), x_train.max(),
len(x_train)), ones))
y_pred = weights @ x_pred.T
plt.title("Prediction of bicarbonate")
plt.xlabel("ph")
plt.ylabel("biocarbonates ppm")
plt.plot(x_train, y_train, "ro", x_pred, y_pred, "g--")
plt.axis([x_train.min(), x_train.max(), y_train.min(), y_train.max()])
global file_exist
file_exist = ''.join(
[random.choice(string.ascii_letters) for i in range(mmap.PAGESIZE)])
with open("file_exist.tmp", "w") as f:
f.write(file_exist)
if os.path.exists("file_nonexist.tmp"):
os.remove("file_nonexist.tmp")
with open("file_delete.tmp", "w") as f:
f.write(file_exist)
# Running File
regression = Regression(loader, "File", prepare_files)
regression.add_check(name="Basic File Opening",
check=lambda res: "File Open Test 1 OK" in res[
0].log and "File Open Test 2 OK" in res[0].log and
"File Open Test 3 OK" in res[0].log)
regression.add_check(name="Basic File Creation",
check=lambda res: "File Creation Test 1 OK" in res[
0].log and "File Creation Test 2 OK" in res[0].log and
"File Creation Test 3 OK" in res[0].log)
regression.add_check(
name="File Reading",
check=lambda res:
("Read Test 1 (0th - 40th): " + file_exist[0:40]) in res[0].log and
def prepare_files(args):
global file_exist
file_exist = ''.join([random.choice(string.ascii_letters) for i in range(mmap.PAGESIZE)])
with open("file_exist.tmp", "w") as f:
f.write(file_exist)
if os.path.exists("file_nonexist.tmp"):
os.remove("file_nonexist.tmp")
with open("file_delete.tmp", "w") as f:
f.write(file_exist)
# Running File
regression = Regression(loader, "File", prepare_files)
regression.add_check(name="Basic File Opening",
check=lambda res: "File Open Test 1 OK" in res[0].log and
"File Open Test 2 OK" in res[0].log and
"File Open Test 3 OK" in res[0].log)
regression.add_check(name="Basic File Creation",
check=lambda res: "File Creation Test 1 OK" in res[0].log and
"File Creation Test 2 OK" in res[0].log and
"File Creation Test 3 OK" in res[0].log)
regression.add_check(name="File Reading",
check=lambda res: ("Read Test 1 (0th - 40th): " + file_exist[0:40]) in res[0].log and
("Read Test 2 (0th - 40th): " + file_exist[0:40]) in res[0].log and
("Read Test 3 (200th - 240th): " + file_exist[200:240]) in res[0].log)
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running stat
regression = Regression(loader, "getcwd")
regression.add_check(name="Getcwd syscall",
check=lambda res: "[bss_cwd_buf] getcwd succeeded: /" in res[0].out and \
"[mmapped_cwd_buf] getcwd succeeded: /" in res[0].out)
rv = regression.run_checks()
if rv:
sys.exit(rv)
import sys
from regression import Regression
loader = sys.argv[1]
# Running sigaltstack
regression = Regression(loader, "sigaltstack")
messages = (
"OK on sigaltstack in main thread before alarm",
"&act == 0x",
"sig 14 count 1 goes off with sp=0x",
"OK on signal stack",
"OK on sigaltstack in handler",
"sig 14 count 2 goes off with sp=0x",
"OK on signal stack",
"OK on sigaltstack in handler",
"sig 14 count 3 goes off with sp=0x",
"OK on signal stack",
"OK on sigaltstack in handler",
"OK on sigaltstack in main thread",
"done exiting",
)
regression.add_check(name="Sigaltstack Test",
check=lambda res: all([x in res[0].out for x in messages]))
rv = regression.run_checks()
if rv: sys.exit(rv)
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running Bootstrap
regression = Regression(loader, "bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].out)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].out and \
"argv[0] = file:bootstrap" in res[0].out)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].out and \
"argv[0] = file:bootstrap" in res[0].out and \
"argv[1] = a" in res[0].out and "argv[2] = b" in res[0].out and \
"argv[3] = c" in res[0].out and "argv[4] = d" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
# Running Exec
regression = Regression(loader, "exec")
regression.add_check(name="2 page child binary",
check=lambda res: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 " in res[0].out)
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
sgx = os.environ.get('SGX_RUN') == '1'
# Running Bootstrap
regression = Regression(loader, "mmap-file", None, 60000)
regression.add_check(name="Private mmap beyond file range",
check=lambda res: "mmap test 6 passed" in res[0].out and \
"mmap test 7 passed" in res[0].out)
regression.add_check(name="Private mmap beyond file range (after fork)",
check=lambda res: "mmap test 1 passed" in res[0].out and \
"mmap test 2 passed" in res[0].out and \
"mmap test 3 passed" in res[0].out and \
"mmap test 4 passed" in res[0].out)
# On SGX, SIGBUS isn't always implemented correctly, for lack
# of memory protection. For now, some of these cases won't work.
if not sgx:
regression.add_check(name="SIGBUS test",
check=lambda res: "mmap test 5 passed" in res[0].out and \
"mmap test 8 passed" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
try:
sgx = os.environ['SGX_RUN']
except KeyError:
sgx = 0
regression = Regression(loader, "Memory")
regression.add_check(name="Memory Allocation",
check=lambda res: "Memory Allocation OK" in res[0].log)
regression.add_check(
name="Memory Allocation with Address",
check=lambda res: "Memory Allocation with Address OK" in res[0].log)
regression.add_check(
name="Memory Protection",
flaky=sgx,
check=lambda res: "Memory Allocation Protection (RW) OK" in res[
0].log and "Memory Protection (R) OK" in res[0].log)
regression.add_check(name="Memory Deallocation",
flaky=sgx,
check=lambda res: "Memory Deallocation OK" in res[0].log)
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running Bootstrap
regression = Regression(loader, "proc")
regression.add_check(name="Base /proc files present",
check=lambda res: "/proc/1/.." in res[0].out and \
"/proc/1/cwd" in res[0].out and \
"/proc/1/exe" in res[0].out and \
"/proc/1/root" in res[0].out and \
"/proc/1/fd" in res[0].out and \
"/proc/1/maps" in res[0].out and \
"/proc/." in res[0].out and \
"/proc/1" in res[0].out and \
"/proc/self" in res[0].out and \
"/proc/meminfo" in res[0].out and \
"/proc/cpuinfo" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
#!/usr/bin/env python2
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_SEC']
if not os.path.exists(loader):
print "Reference monitor mode is not available on this platform"
exit(0)
regression = Regression(loader, "Process")
def check_times(target, lines, times):
count = 0
for line in lines:
if target == line:
count += 1
return count == times
regression.add_check(
name="Process Creation",
check=lambda res: check_times("Child Process Created", res[0].log, 3))
regression.add_check(
name="Process Creation Arguments",
check=lambda res: check_times("argv[0] = Process", res[
0].log, 3) and check_times("argv[1] = Child", res[0].log, 3))
import os, sys
from regression import Regression
loader = sys.argv[1]
# Running Bootstrap
regression = Regression(loader, "proc-path")
regression.add_check(name="Base /proc path present",
check=lambda res: "proc path test success" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running stat
regression = Regression(loader, "stat_invalid_args")
regression.add_check(name="Stat with invalid arguments",
check=lambda res: "stat(invalid-path-ptr) correctly returned error" in res[0].out and \
"stat(invalid-buf-ptr) correctly returned error" in res[0].out and \
"lstat(invalid-path-ptr) correctly returned error" in res[0].out and \
"lstat(invalid-buf-ptr) correctly returned error" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
# Running fstat
regression = Regression(loader, "fstat_cwd")
regression.add_check(
name="Fstat on a directory",
check=lambda res: "fstat returned the fd type as S_IFDIR" in res[0].out)
rv = regression.run_checks()
if rv:
sys.exit(rv)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = '../src/pal'
regression = Regression(loader, "Thread")
regression.add_check(name="Thread Creation",
check=lambda res: "Child Thread Created" in res[0].log and
"Run in Child Thread: Hello World" in res[0].log)
regression.add_check(name="Multiple Threads Run in Parallel",
check=lambda res: "Threads Run in Parallel OK" in res[0].log)
regression.add_check(name="Set Thread Private Segment Register",
check=lambda res: "Private Message (FS Segment) 1: Hello World 1" in res[0].log and
"Private Message (FS Segment) 2: Hello World 2" in res[0].log)
regression.add_check(name="Thread Exit",
check=lambda res: "Child Thread Exited" in res[0].log)
regression.run_checks()
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Exception")
regression.add_check(name="Exception Handling (Div-by-Zero)",
check=lambda res: any([
line.startswith("Div-by-Zero Exception Handler")
for line in res[0].log
]))
regression.add_check(name="Exception Handling (Memory Fault)",
check=lambda res: any([
line.startswith("Memory Fault Exception Handler")
for line in res[0].log
]))
regression.add_check(name="Exception Handler Swap",
check=lambda res: any([
line.startswith("Div-by-Zero Exception Handler 1")
for line in res[0].log
]) and any([
line.startswith("Div-by-Zero Exception Handler 2")
for line in res[0].log
]))
regression.add_check(name="Exception Handling (Set Context)",
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running Bootstrap
regression = Regression(loader, "bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].out)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].out and \
"argv[0] = file:bootstrap" in res[0].out)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].out and \
"argv[0] = file:bootstrap" in res[0].out and \
"argv[1] = a" in res[0].out and "argv[2] = b" in res[0].out and \
"argv[3] = c" in res[0].out and "argv[4] = d" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
# Running Exec
regression = Regression(loader, "exec")
regression.add_check(name="2 page child binary",
check=lambda res: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 " in res[0].out)
print("Bulk IPC not supported on SGX")
exit(0)
## XXX Should really be running these tests as part of CI
if not os.path.exists('/dev/gipc'):
print("GIPC not loaded; skipping these tests\n")
exit(0)
def prepare_files(args):
with open("ipc_mapping.tmp", "w") as f:
f.write("Hello World")
os.ftruncate(f.fileno(), mmap.PAGESIZE)
regression = Regression(loader, "Ipc", prepare_files)
def check_times(target, lines, times):
count = 0
for line in lines:
if target == line:
count += 1
return count == times
regression.add_check(
name="Create and Join Physical Memory Bulk Copy Store",
check=lambda res: check_times(
"Create Physical Memory Store OK", res[0].log, 5) and check_times(
"Join Physical Memory Store OK", res[0].log, 5))
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Socket")
regression.add_check(name="TCP Socket Creation",
check=lambda res: "TCP Creation 1 OK" in res[0].log)
regression.add_check(name="TCP Socket Connection",
check=lambda res: "TCP Connection 1 OK" in res[0].log)
regression.add_check(name="TCP Socket Transmission",
check=lambda res: "TCP Write 1 OK" in res[0].log and
"TCP Read 1: Hello World 1" in res[0].log and
"TCP Write 2 OK" in res[0].log and
"TCP Read 2: Hello World 2" in res[0].log)
regression.add_check(name="UDP Socket Creation",
check=lambda res: "UDP Creation 1 OK" in res[0].log)
regression.add_check(name="UDP Socket Connection",
check=lambda res: "UDP Connection 1 OK" in res[0].log)
regression.add_check(name="UDP Socket Transmission",
check=lambda res: "UDP Write 1 OK" in res[0].log and
"UDP Read 1: Hello World 1" in res[0].log and
"UDP Write 2 OK" in res[0].log and
"UDP Read 2: Hello World 2" in res[0].log)
import pandas as pd
from regression import Regression
x = Regression()
df = pd.read_csv("trips_summary_covid_pub.csv")
df['date'] = pd.to_datetime(df['date'])
df['pre_covid'] = (df.date < "12/19/2020").astype("int")
df = df.loc[df.pre_covid == 0]
# TOTAL REVENUE MODEL
x.select_cols = [
'pickup_community_area', 'hour', 'week_day', 'cases', 'total_revenue'
]
x.dummy_cols = ['pickup_community_area', 'hour', 'week_day']
x.y_col = 'total_revenue'
res_revenue = x.time_split(df)
# TOTAL COUNT MODEL
x.select_cols = ['pickup_community_area', 'hour', 'week_day', 'cases', 'count']
x.y_col = 'count'
res_count = x.time_split(df)
# TOTAL SECONDS MODEL
x.select_cols = [
'pickup_community_area', 'hour', 'week_day', 'cases', 'trip_seconds_tot'
]
x.y_col = 'trip_seconds_tot'
import sys
from regression import Regression
loader = sys.argv[1]
regression = Regression(loader, "proc_cpuinfo", None, 50000)
regression.add_check(name="proc/cpuinfo Linux-based formatting",
check=lambda res: "cpuinfo test passed" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "SendHandle")
def check_times(target, lines, times):
count = 0
for line in lines:
if target == line:
count += 1
return count == times
regression.add_check(name="Send and Receive Handles across Processes",
check=lambda res: check_times("Send Handle OK", res[0].log, 3) and
check_times("Receive Handle OK", res[0].log, 3))
regression.add_check(name="Send Pipe Handle",
check=lambda res: check_times("Receive Pipe Handle: Hello World", res[0].log, 1))
regression.add_check(name="Send Socket Handle",
check=lambda res: check_times("Receive Socket Handle: Hello World", res[0].log, 1))
regression.add_check(name="Send File Handle",
check=lambda res: check_times("Receive File Handle: Hello World", res[0].log, 1))
rv = regression.run_checks()
if rv: sys.exit(rv)
if os.path.exists("dir_delete.tmp"):
shutil.rmtree("dir_delete.tmp")
global dir_files
os.mkdir("dir_exist.tmp")
dir_files = []
for i in range(5):
file = ''.join([random.choice(string.ascii_letters) for i in range(8)])
f = open("dir_exist.tmp/" + file, "w")
f.close()
dir_files.append(file)
os.mkdir("dir_delete.tmp")
regression = Regression(loader, "Directory", prepare_dirs)
regression.add_check(name="Basic Directory Opening",
check=lambda res: "Directory Open Test 1 OK" in res[0].log and
"Directory Open Test 2 OK" in res[0].log and
"Directory Open Test 3 OK" in res[0].log)
regression.add_check(name="Basic Directory Creation",
check=lambda res: "Directory Creation Test 1 OK" in res[0].log and
"Directory Creation Test 2 OK" in res[0].log and
"Directory Creation Test 3 OK" in res[0].log)
def check_read(res):
global dir_files
for file in dir_files:
if ("Read Directory: " + file) not in res[0].log:
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Pipe")
regression.add_check(name="Pipe Creation",
check=lambda res: "Pipe Creation 1 OK" in res[0].log)
regression.add_check(name="Pipe Attributes",
check=lambda res: "Pipe Attribute Query 1 on pipesrv returned OK" in res[0].log)
regression.add_check(name="Pipe Connection",
check=lambda res: "Pipe Connection 1 OK" in res[0].log)
regression.add_check(name="Pipe Transmission",
check=lambda res: "Pipe Write 1 OK" in res[0].log and
"Pipe Read 1: Hello World 1" in res[0].log and
"Pipe Write 2 OK" in res[0].log and
"Pipe Read 2: Hello World 2" in res[0].log)
rv = regression.run_checks()
if rv: sys.exit(rv)
import sys
from regression import Regression
loader = sys.argv[1]
# Running udp
regression = Regression(loader, "udp", None)
regression.add_check(name="udp",
check=lambda res:
"Data: This is packet 0" in res[0].out and
"Data: This is packet 1" in res[0].out and
"Data: This is packet 2" in res[0].out and
"Data: This is packet 3" in res[0].out and
"Data: This is packet 4" in res[0].out and
"Data: This is packet 5" in res[0].out and
"Data: This is packet 6" in res[0].out and
"Data: This is packet 7" in res[0].out and
"Data: This is packet 8" in res[0].out and
"Data: This is packet 9" in res[0].out)
rv = regression.run_checks()
if rv:
sys.exit(rv)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Misc", timeout=5000)
regression.add_check(name="Query System Time",
check=lambda res: "Query System Time OK" in res[0].log)
regression.add_check(name="Delay Execution for 10000 Microseconds",
check=lambda res:
"Delay Execution for 10000 Microseconds OK" in res[0].log)
regression.add_check(
name="Delay Execution for 3 Seconds",
check=lambda res: "Delay Execution for 3 Seconds OK" in res[0].log)
regression.add_check(name="Generate Random Bits",
check=lambda res: "Generate Random Bits OK" in res[0].log)
rv = regression.run_checks()
if rv:
sys.exit(rv)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = '../src/pal'
regression = Regression(loader, "Exception")
def check_exception1(res):
for line in res[0].log:
if not line:
continue
if line.startswith('Div-by-Zero Exception Handler'):
return True
return False
regression.add_check(name="Exception Handling (Div-by-Zero)", check=check_exception1)
def check_exception2(res):
for line in res[0].log:
if not line:
continue
if line.startswith('Memory Fault Exception Handler'):
return True
return False
regression.add_check(name="Exception Handling (Memory Fault)", check=check_exception2)
def check_exception3(res):
found1 = False
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "Process", timeout=8000)
def check_times(target, lines, times):
count = 0
for line in lines:
if target == line:
count += 1
return count == times
regression.add_check(
name="Process Creation",
check=lambda res: check_times("Child Process Created", res[0].log, 3))
regression.add_check(
name="Process Creation Arguments",
check=lambda res: check_times("argv[0] = Process", res[
0].log, 3) and check_times("argv[1] = Child", res[0].log, 3))
regression.add_check(
name="Process Channel Transmission",
check=lambda res: check_times("Process Write 1 OK", res[
0].log, 3) and check_times("Process Read 1: Hello World 1", res[
0].log, 3) and check_times("Process Write 2 OK", res[0].log, 3) and
check_times("Process Read 2: Hello World 2", res[0].log, 3))
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
sgx = os.environ.get('SGX_RUN') == '1'
def manifest_file(file):
if sgx:
return file + '.manifest.sgx'
else:
return file + '.manifest'
# Running Bootstrap
regression = Regression(loader, "Bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].log)
regression.add_check(name="Control Block: Executable Name",
check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].log and \
"argv[0] = file:Bootstrap" in res[0].log)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].log and \
"argv[1] = a" in res[0].log and "argv[2] = b" in res[0].log and \
"argv[3] = c" in res[0].log and "argv[4] = d" in res[0].log)
def run(self):
io = FileIO()
will_update = self.update
if self.csvfile:
stock_tse = io.read_from_csv(self.code, self.csvfile)
msg = "".join([
"Read data from csv: ", self.code, " Records: ",
str(len(stock_tse))
])
print(msg)
if self.update and len(stock_tse) > 0:
index = pd.date_range(start=stock_tse.index[-1],
periods=2,
freq='B')
ts = pd.Series(None, index=index)
next_day = ts.index[1]
t = next_day.strftime('%Y-%m-%d')
newdata = io.read_data(self.code, start=t, end=self.end)
msg = "".join([
"Read data from web: ", self.code, " New records: ",
str(len(newdata))
])
print(msg)
if len(newdata) < 1:
will_update = False
else:
print(newdata.ix[-1, :])
stock_tse = stock_tse.combine_first(newdata)
io.save_data(stock_tse, self.code, 'stock_')
else:
stock_tse = io.read_data(self.code, start=self.start, end=self.end)
msg = "".join([
"Read data from web: ", self.code, " Records: ",
str(len(stock_tse))
])
print(msg)
if stock_tse.empty:
msg = "".join(["Data empty: ", self.code])
print(msg)
return None
if not self.csvfile:
io.save_data(stock_tse, self.code, 'stock_')
try:
stock_d = stock_tse.asfreq('B').dropna()[self.days:]
ti = TechnicalIndicators(stock_d)
ti.calc_sma()
ti.calc_sma(timeperiod=5)
ti.calc_sma(timeperiod=25)
ti.calc_sma(timeperiod=50)
ti.calc_sma(timeperiod=75)
ewma = ti.calc_ewma(span=5)
ewma = ti.calc_ewma(span=25)
ewma = ti.calc_ewma(span=50)
ewma = ti.calc_ewma(span=75)
bbands = ti.calc_bbands()
sar = ti.calc_sar()
draw = Draw(self.code, self.fullname)
ret = ti.calc_ret_index()
ti.calc_vol(ret['ret_index'])
rsi = ti.calc_rsi(timeperiod=9)
rsi = ti.calc_rsi(timeperiod=14)
mfi = ti.calc_mfi()
roc = ti.calc_roc(timeperiod=10)
roc = ti.calc_roc(timeperiod=25)
roc = ti.calc_roc(timeperiod=50)
roc = ti.calc_roc(timeperiod=75)
roc = ti.calc_roc(timeperiod=150)
ti.calc_cci()
ultosc = ti.calc_ultosc()
stoch = ti.calc_stoch()
ti.calc_stochf()
ti.calc_macd()
willr = ti.calc_willr()
ti.calc_momentum(timeperiod=10)
ti.calc_momentum(timeperiod=25)
tr = ti.calc_tr()
ti.calc_atr()
ti.calc_natr()
vr = ti.calc_volume_rate()
ret_index = ti.stock['ret_index']
clf = Classifier(self.clffile)
train_X, train_y = clf.train(ret_index, will_update)
msg = "".join(["Train Records: ", str(len(train_y))])
print(msg)
clf_result = clf.classify(ret_index)[0]
msg = "".join(["Classified: ", str(clf_result)])
print(msg)
ti.stock.ix[-1, 'classified'] = clf_result
reg = Regression(self.regfile, alpha=1, regression_type="Ridge")
train_X, train_y = reg.train(ret_index, will_update)
msg = "".join(["Train Records: ", str(len(train_y))])
base = ti.stock_raw['Adj Close'][0]
reg_result = int(reg.predict(ret_index, base)[0])
msg = "".join(["Predicted: ", str(reg_result)])
print(msg)
ti.stock.ix[-1, 'predicted'] = reg_result
if len(self.reference) > 0:
ti.calc_rolling_corr(self.reference)
ref = ti.stock['rolling_corr']
else:
ref = []
io.save_data(io.merge_df(stock_d, ti.stock), self.code, 'ti_')
draw.plot(stock_d,
ewma,
bbands,
sar,
rsi,
roc,
mfi,
ultosc,
willr,
stoch,
tr,
vr,
clf_result,
reg_result,
ref,
axis=self.axis,
complexity=self.complexity)
return ti
except (ValueError, KeyError) as e:
print("Error occured in", self.code, "at analysis.py")
print('ErrorType:', str(type(e)))
print('ErrorMessage:', str(e))
return None
import sys
from regression import Regression
loader = sys.argv[1]
# Running udp
regression = Regression(loader, "udp", None, 50000)
regression.add_check(
name="udp",
check=lambda res: "Data: This is packet 0" in res[0].out and
"Data: This is packet 1" in res[0].out and "Data: This is packet 2" in res[
0].out and "Data: This is packet 3" in res[0].out and
"Data: This is packet 4" in res[0].out and "Data: This is packet 5" in res[
0].out and "Data: This is packet 6" in res[0].out and
"Data: This is packet 7" in res[0].out and "Data: This is packet 8" in res[
0].out and "Data: This is packet 9" in res[0].out)
rv = regression.run_checks()
if rv:
sys.exit(rv)
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_SEC']
if not os.path.exists(loader):
print "Reference monitor mode is not available on this platform"
exit(0)
# Running Bootstrap
regression = Regression(loader, "Bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].log)
regression.add_check(
name="Control Block: Executable Name",
check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)
regression.add_check(
name="Control Block: Default Manifest",
check=lambda res: "Loaded Manifest: file:manifest" in res[0].log)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].log and \
"argv[0] = file:Bootstrap" in res[0].log)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
#!/usr/bin/env python2
import os, sys, mmap
from regression import Regression
loader = sys.argv[1]
# Running Bootstrap
regression = Regression(loader, "getdents", None, 10000)
# This doesn't catch extraneous entries, but should be fine
# until the LTP test can be run (need symlink support)
regression.add_check(name="Directory listing (32-bit)",
check=lambda res: "getdents: setup ok" in res[0].out and \
"getdents32: . [0x4]" in res[0].out and \
"getdents32: .. [0x4]" in res[0].out and \
"getdents32: file1 [0x8]" in res[0].out and \
"getdents32: file2 [0x8]" in res[0].out and \
"getdents32: dir3 [0x4]" in res[0].out)
regression.add_check(name="Directory listing (64-bit)",
check=lambda res: "getdents: setup ok" in res[0].out and \
"getdents64: . [0x4]" in res[0].out and \
"getdents64: .. [0x4]" in res[0].out and \
"getdents64: file1 [0x8]" in res[0].out and \
"getdents64: file2 [0x8]" in res[0].out and \
"getdents64: dir3 [0x4]" in res[0].out)
rv = regression.run_checks()
if rv: sys.exit(rv)
loader = os.environ['PAL_LOADER']
try:
sgx = os.environ['SGX_RUN']
except KeyError:
sgx = False
def manifest_file(file):
if 'SGX_RUN' in os.environ and os.environ['SGX_RUN'] == '1':
return file + '.manifest.sgx'
else:
return file + '.manifest'
# Running Bootstrap
regression = Regression(loader, "Bootstrap")
regression.add_check(name="Basic Bootstrapping",
check=lambda res: "User Program Started" in res[0].log)
regression.add_check(
name="Control Block: Executable Name",
check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)
regression.add_check(name="One Argument Given",
check=lambda res: "# of Arguments: 1" in res[0].log and \
"argv[0] = file:Bootstrap" in res[0].log)
regression.add_check(name="Five Arguments Given",
args = ['a', 'b', 'c', 'd'],
check=lambda res: "# of Arguments: 5" in res[0].log and \
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = "../src/pal"
regression = Regression(loader, "Pipe")
regression.add_check(name="Pipe Creation", check=lambda res: "Pipe Creation 1 OK" in res[0].log)
regression.add_check(name="Pipe Connection", check=lambda res: "Pipe Connection 1 OK" in res[0].log)
regression.add_check(
name="Pipe Transmission",
check=lambda res: "Pipe Write 1 OK" in res[0].log
and "Pipe Read 1: Hello World 1" in res[0].log
and "Pipe Write 2 OK" in res[0].log
and "Pipe Read 2: Hello World 2" in res[0].log,
)
regression.run_checks()
#!/usr/bin/env python2
import os, sys, mmap
from regression import Regression
loader = os.environ['PAL_LOADER']
regression = Regression(loader, "SendHandle")
def check_times(target, lines, times):
count = 0
for line in lines:
if target == line:
count += 1
return count == times
regression.add_check(
name="Send and Receive Handles across Processes",
check=lambda res: check_times("Send Handle OK", res[
0].log, 3) and check_times("Receive Handle OK", res[0].log, 3))
regression.add_check(name="Send Pipe Handle",
check=lambda res: check_times(
"Receive Pipe Handle: Hello World", res[0].log, 1))
regression.add_check(name="Send Socket Handle",
check=lambda res: check_times(
"Receive Socket Handle: Hello World", res[0].log, 1))
#!/usr/bin/python
import os, sys, mmap
from regression import Regression
loader = '../src/pal'
regression = Regression(loader, "Exception")
regression.add_check(name="Exception Handling (Div-by-Zero)",
check=lambda res: any([line.startswith("Div-by-Zero Exception Handler") for line in res[0].log]))
regression.add_check(name="Exception Handling (Memory Fault)",
check=lambda res: any([line.startswith("Memory Fault Exception Handler") for line in res[0].log]))
regression.add_check(name="Exception Handler Swap",
check=lambda res: any([line.startswith("Div-by-Zero Exception Handler 1") for line in res[0].log]) and
any([line.startswith("Div-by-Zero Exception Handler 2") for line in res[0].log]))
regression.add_check(name="Exception Handling (Set Context)",
check=lambda res: any([line.startswith("Div-by-Zero Exception Handler 1") for line in res[0].log]))
regression.run_checks()
plt.ylabel("")
data.plot.scatter(x=x, y=y, ax=ax, s=10, label=y_label2, legend=False)
# call teh legend, place atop the image on the left
# bbox_to_anchor used to specify exact placement of label
plt.legend(loc="upper left",
labels=[y_label1, y_label2],
bbox_to_anchor=(0, 1.17))
# remove lines marking units on the axis
ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('none')
plt.show()
plt.close()
data = pd.read_csv("cleanedEconFreedomData.csv")
reg = Regression()
y_var = ["GDP per Capita (PPP)"]
x_vars = [
"Judical Effectiveness", "Property Rights", "Fiscal Health",
"Investment Freedom ", "Financial Freedom", "Inflation (%)",
"Public Debt (% of GDP)"
]
reg.OLS("GDP Per Capita Restricted", data, y_var, x_vars)
plot_scatter_with_estimator(reg.data, x_vars, y_var)
x_vars_unrestricted = x_vars
x_vars_restricted = ["Judical Effectiveness", "Property Rights"]
reg.OLS("GDP Per Capita Restricted", data, y_var, x_vars_restricted)
reg.OLS("GDP Per Capita Unrestricted", data, y_var, x_vars_unrestricted)
