示例#1
0
def test_ridge_regression():
    stock_d = testdata()
    ti = TechnicalIndicators(stock_d)

    filename = "test_N225_ridge.pickle"
    clffile = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "clf", filename)

    if os.path.exists(clffile):
        os.remove(clffile)

    clf = Regression(filename)
    ti.calc_ret_index()
    ret = ti.stock["ret_index"]
    base = ti.stock_raw["Adj Close"][0]

    train_X, train_y = clf.train(ret, regression_type="Ridge")

    test_y = clf.predict(ret, base)

    expected = 19177.97
    r = round(test_y[0], 2)
    eq_(r, expected)

    if os.path.exists(clffile):
        os.remove(clffile)
def regression_chicago_crime_weather(p, date_start, date_end, crime_types = None, wards = None):
    crime_data = get_chicago_crime_data(p, date_start, date_end, crime_types, wards)
    weather_data = get_chicago_weather_data(p, date_start, date_end)

    crimes = [x[1] for x in crime_data]
    temps = [x[1] for x in weather_data]
    fill_missing_values(temps)
    
    r = Regression(temps, u"Temperature (°F)", crimes, get_crimes_label(crime_types))    
    r.compute()
    r.plot()
示例#3
0
    def run(self):
        io = FileIO()
        will_update = self.update

        if self.csvfile:
            stock_tse = io.read_from_csv(self.code, self.csvfile)

            msg = "".join(["Read data from csv: ", self.code, " Records: ", str(len(stock_tse))])
            print(msg)

            if self.update and len(stock_tse) > 0:
                index = pd.date_range(start=stock_tse.index[-1], periods=2, freq="B")
                ts = pd.Series(None, index=index)
                next_day = ts.index[1]
                t = next_day.strftime("%Y-%m-%d")
                newdata = io.read_data(self.code, start=t, end=self.end)

                msg = "".join(["Read data from web: ", self.code, " New records: ", str(len(newdata))])
                print(msg)
                if len(newdata) < 1:
                    will_update = False
                else:
                    print(newdata.ix[-1, :])

                stock_tse = stock_tse.combine_first(newdata)
                io.save_data(stock_tse, self.code, "stock_")
        else:
            stock_tse = io.read_data(self.code, start=self.start, end=self.end)

            msg = "".join(["Read data from web: ", self.code, " Records: ", str(len(stock_tse))])
            print(msg)

        if stock_tse.empty:
            msg = "".join(["Data empty: ", self.code])
            print(msg)
            return None

        if not self.csvfile:
            io.save_data(stock_tse, self.code, "stock_")

        try:
            stock_d = stock_tse.asfreq("B").dropna()[self.days :]

            ti = TechnicalIndicators(stock_d)

            ti.calc_sma()
            ti.calc_sma(timeperiod=5)
            ti.calc_sma(timeperiod=25)
            ti.calc_sma(timeperiod=50)
            ti.calc_sma(timeperiod=75)
            ewma = ti.calc_ewma(span=5)
            ewma = ti.calc_ewma(span=25)
            ewma = ti.calc_ewma(span=50)
            ewma = ti.calc_ewma(span=75)
            bbands = ti.calc_bbands()
            sar = ti.calc_sar()
            draw = Draw(self.code, self.name)

            ret = ti.calc_ret_index()
            ti.calc_vol(ret["ret_index"])
            rsi = ti.calc_rsi(timeperiod=9)
            rsi = ti.calc_rsi(timeperiod=14)
            mfi = ti.calc_mfi()
            roc = ti.calc_roc(timeperiod=10)
            roc = ti.calc_roc(timeperiod=25)
            roc = ti.calc_roc(timeperiod=50)
            roc = ti.calc_roc(timeperiod=75)
            roc = ti.calc_roc(timeperiod=150)
            ti.calc_cci()
            ultosc = ti.calc_ultosc()
            stoch = ti.calc_stoch()
            ti.calc_stochf()
            ti.calc_macd()
            willr = ti.calc_willr()
            ti.calc_momentum(timeperiod=10)
            ti.calc_momentum(timeperiod=25)
            tr = ti.calc_tr()
            ti.calc_atr()
            ti.calc_natr()
            vr = ti.calc_volume_rate()

            ret_index = ti.stock["ret_index"]
            clf = Classifier(self.clffile)
            train_X, train_y = clf.train(ret_index, will_update)
            msg = "".join(["Train Records: ", str(len(train_y))])
            print(msg)
            clf_result = clf.classify(ret_index)[0]
            msg = "".join(["Classified: ", str(clf_result)])
            print(msg)
            ti.stock.ix[-1, "classified"] = clf_result

            reg = Regression(self.regfile, alpha=1, regression_type="Ridge")
            train_X, train_y = reg.train(ret_index, will_update)
            msg = "".join(["Train Records: ", str(len(train_y))])
            base = ti.stock_raw["Adj Close"][0]
            reg_result = int(reg.predict(ret_index, base)[0])
            msg = "".join(["Predicted: ", str(reg_result)])
            print(msg)
            ti.stock.ix[-1, "predicted"] = reg_result

            if len(self.reference) > 0:
                ti.calc_rolling_corr(self.reference)
                ref = ti.stock["rolling_corr"]
            else:
                ref = []

            io.save_data(io.merge_df(stock_d, ti.stock), self.code, "ti_")

            draw.plot(
                stock_d,
                ewma,
                bbands,
                sar,
                rsi,
                roc,
                mfi,
                ultosc,
                willr,
                stoch,
                tr,
                vr,
                clf_result,
                reg_result,
                ref,
                axis=self.axis,
                complexity=self.complexity,
            )

            return ti

        except (ValueError, KeyError):
            msg = "".join(["Error occured in ", self.code])
            print(msg)
            return None
示例#4
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = "../src/pal"

regression = Regression(loader, "Memory")

regression.add_check(name="Memory Allocation", check=lambda res: "Memory Allocation OK" in res[0].log)

regression.add_check(
    name="Memory Allocation with Address", check=lambda res: "Memory Allocation with Address OK" in res[0].log
)

regression.add_check(
    name="Memory Protection",
    check=lambda res: "Memory Allocation Protection (RW) OK" in res[0].log and "Memory Protection (R) OK" in res[0].log,
)

regression.add_check(name="Memory Deallocation", check=lambda res: "Memory Deallocation OK" in res[0].log)


def check_quota(res):
    for line in res[0].log:
        if line.startswith("Total Memory:"):
            return line != "Total Memory: 0"
    return False


regression.add_check(name="Get Memory Total Quota", check=check_quota)
示例#5
0
from preProcessor import PreProcessor
import argparse
from regression import Regression
from picDrawer import PicDrawer

if __name__ == '__main__':
    '''
    
        python run -f [filepath] -s [filepath] -c [stock code]
    
    output:
        stock error : implement by regression.score() 
        picture : implement by drawer
    
    '''

    # create pre processor

    data_cleaner = PreProcessor()
    train_feature, train_label, test_feature, test_label = data_cleaner.run()

    reg = Regression()
    reg.fit(train_feature, train_label)
    pred_result = reg.predict(test_feature)

    score = reg.score(test_label, pred_result)

    drawer = PicDrawer()
    drawer.run()
示例#6
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = './pal'

# Running Bootstrap
regression = Regression(loader, "bootstrap")

regression.add_check(name="Basic Bootstrapping",
    check=lambda res: "User Program Started" in res[0].out)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].out and \
            "argv[0] = file:bootstrap" in res[0].out)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].out and \
           "argv[0] = file:bootstrap" in res[0].out and \
           "argv[1] = a" in res[0].out and "argv[2] = b" in res[0].out and \
           "argv[3] = c" in res[0].out and "argv[4] = d" in res[0].out)

regression.run_checks()
示例#7
0
import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Symbols")

all_symbols = [
    'DkVirtualMemoryAlloc',
    'DkVirtualMemoryFree',
    'DkVirtualMemoryProtect',
    'DkProcessCreate',
    'DkProcessExit',
    'DkProcessSandboxCreate',
    'DkStreamOpen',
    'DkStreamWaitForClient',
    'DkStreamRead',
    'DkStreamWrite',
    'DkStreamDelete',
    'DkStreamMap',
    'DkStreamUnmap',
    'DkStreamSetLength',
    'DkStreamFlush',
    'DkSendHandle',
    'DkReceiveHandle',
    'DkStreamAttributesQuery',
    'DkStreamAttributesQueryByHandle',
    'DkStreamAttributesSetByHandle',
    'DkStreamGetName',
    'DkStreamChangeName',
    'DkThreadCreate',
示例#8
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = '../src/pal'

# Running Bootstrap
regression = Regression(loader, "Bootstrap")

regression.add_check(name="Basic Bootstrapping",
    check=lambda res: "User Program Started" in res[0].log)

regression.add_check(name="Control Block: Executable Name",
    check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)

regression.add_check(name="Control Block: Default Manifest",
    check=lambda res: "Loaded Manifest: file:manifest" in res[0].log)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].log and \
            "argv[0] = file:Bootstrap" in res[0].log)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].log and \
           "argv[0] = file:Bootstrap" in res[0].log and \
           "argv[1] = a" in res[0].log and "argv[2] = b" in res[0].log and \
           "argv[3] = c" in res[0].log and "argv[4] = d" in res[0].log)

regression.add_check(name="Control Block: Debug Stream (Inline)",
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

regression = Regression(loader, "epoll_wait_timeout", None, 50000)

regression.add_check(name="epoll_wait timeout",
    args = ['8000'],
    check=lambda res: "epoll_wait test passed" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#10
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Socket")

regression.add_check(name="TCP Socket Creation",
    check=lambda res: "TCP Creation 1 OK" in res[0].log)

regression.add_check(name="TCP Socket Connection",
    check=lambda res: "TCP Connection 1 OK" in res[0].log)

regression.add_check(name="TCP Socket Transmission",
    check=lambda res: "TCP Write 1 OK" in res[0].log and
                      "TCP Read 1: Hello World 1" in res[0].log and
                      "TCP Write 2 OK" in res[0].log and
                      "TCP Read 2: Hello World 2" in res[0].log)

regression.add_check(name="UDP Socket Creation",
    check=lambda res: "UDP Creation 1 OK" in res[0].log)

regression.add_check(name="UDP Socket Connection",
    check=lambda res: "UDP Connection 1 OK" in res[0].log)

regression.add_check(name="UDP Socket Transmission",
    check=lambda res: "UDP Write 1 OK" in res[0].log and
                      "UDP Read 1: Hello World 1" in res[0].log and
                      "UDP Write 2 OK" in res[0].log and
示例#11
0
def predict(xlabel,ylabel,x_val,x,y):
    meanx=np.mean(x)
    stddevx=np.std(x)
    meany=np.mean(y)
    stddevy=np.std(y)
    x=featureNormalize(x)
    y=featureNormalize(y)

    reg = Regression()
    reg.set_learning_rate(0.01)
    reg.set_max_iterations(10000)
    reg.set_l1_penalty(0.1)
    reg.set_l2_penalty(0.1)
    reg.set_tolerance(1e-5)
    theta, cost, it = reg.polynomial_regression(x, y, 5)

    z = np.linspace(-1.9, 2.1, 4/0.01)
    prediction = reg.predict(z)

    x=np.array(x)*stddevx+meanx
    z=np.array(z)*stddevx+meanx
    y=np.array(y)*stddevy+meany
    prediction=np.array(prediction)*stddevy+meany

    x_val=(x_val-meanx)/stddevx
    y_val=reg.predict([x_val])
    x_val=[x_val]
    x_val=np.array(x_val)*stddevx+meanx
    y_val=np.array(y_val)*stddevy+meany

    fig = plt.figure(figsize=(4,4))
    plt.plot(x,y,'.', label='Input data')
    plt.plot(z,prediction,'r-', label='Best fit curve')
    plt.plot(x_val,y_val,'gx',label='Predicted Data')
    plt.legend(loc=4)
    title=xlabel + " vs " + ylabel
    plt.title(title,size=10)
    plt.xticks(size=8)
    plt.yticks(size=8)
    plt.close('all')

    return([y_val,fig])
示例#12
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running Bootstrap
regression = Regression(loader, "large-mmap", None, 60000)

regression.add_check(name="Ftruncate",
    check=lambda res: "large-mmap: ftruncate OK" in res[0].out)

regression.add_check(name="Large mmap",
    check=lambda res: "large-mmap: mmap 1 completed OK" in res[0].out and \
                     "large-mmap: mmap 2 completed OK" in res[0].out)

regression.run_checks()
示例#13
0
import matplotlib.animation as animation
from regression import Regression, panda_to_numpy

data_path = "./hollywood.xls"
data = pd.read_excel(data_path)

x_train = data['X2']
y_train = data['X1']
z_train = data['X3']

# y must be an nX1 array
# x myst be an nXm array where m is the number of different variables for the reggression
# regression() returns an 1Xm+1 array wich are the weights +1 is for the constant
x = panda_to_numpy(data, 'X2', 'X3')
y = data['X1']
model = Regression(y, x)
weights = model.train(epochs=200, a=0.0001, print_loss=True)
prediction = model.predict(8, 15)
print(prediction)

# plot the results
ones = np.ones(len(x_train))
x_normalized = np.linspace(x_train.min(), x_train.max(), len(x_train))
z_normalized = np.linspace(z_train.min(), z_train.max(), len(z_train))
x_pred = np.column_stack((x_normalized, z_normalized, ones))
y_pred = weights @ x_pred.T

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_title("Prediction of hollywood movie revenue")
ax.set_zlabel("Revenue")
import pandas as pd
from regression import Regression

x = Regression()
df_predict = pd.read_csv('x_test_v2.csv').fillna(0).astype(int)

# READ DATASET
df = pd.read_csv("trips_summary_covid_pub.csv")
df['date'] = pd.to_datetime(df['date'])

# FILTER PRE AND POST PANDEMIC
df['pre_covid'] = (df.date < "12/19/2020").astype("int")
df_pre_covid = df.loc[df.pre_covid == 1]
df = df.loc[df.pre_covid == 0]
df = df.loc[df['total_revenue'] > 0]
df_pre_covid = df_pre_covid.loc[df_pre_covid['total_revenue'] > 0]


# TOTAL REVENUE MODEL
x.select_cols = ['pickup_community_area', 'hour', 'total_revenue']
x.dummy_cols = ['pickup_community_area', 'hour']
x.y_col = 'total_revenue'

res_revenue = x.poisson_regression(df, 0.7)

# TOTAL REVENUE PRE COVID
x.select_cols = ['pickup_community_area', 'hour', 'total_revenue']
x.y_col = 'total_revenue'

res_revenue_pre = x.poisson_regression(df_pre_covid, 0.7)
示例#15
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Symbols")

all_symbols = [
    'DkVirtualMemoryAlloc', 'DkVirtualMemoryFree', 'DkVirtualMemoryProtect',
    'DkProcessCreate', 'DkProcessExit', 'DkProcessSandboxCreate',
    'DkStreamOpen', 'DkStreamWaitForClient', 'DkStreamRead', 'DkStreamWrite',
    'DkStreamDelete', 'DkStreamMap', 'DkStreamUnmap', 'DkStreamSetLength',
    'DkStreamFlush', 'DkSendHandle', 'DkReceiveHandle',
    'DkStreamAttributesQuery', 'DkStreamAttributesQuerybyHandle',
    'DkStreamAttributesSetbyHandle', 'DkStreamGetName', 'DkStreamChangeName',
    'DkThreadCreate', 'DkThreadDelayExecution', 'DkThreadYieldExecution',
    'DkThreadExit', 'DkThreadResume', 'DkSetExceptionHandler',
    'DkExceptionReturn', 'DkSemaphoreCreate', 'DkSemaphoreRelease',
    'DkNotificationEventCreate', 'DkSynchronizationEventCreate', 'DkEventSet',
    'DkEventClear', 'DkObjectsWaitAny', 'DkObjectReference', 'DkObjectClose',
    'DkSystemTimeQuery', 'DkRandomBitsRead', 'DkInstructionCacheFlush',
    'DkSegmentRegister', 'DkMemoryAvailableQuota',
    'DkCreatePhysicalMemoryChannel', 'DkPhysicalMemoryCommit',
    'DkPhysicalMemoryMap'
]


def check_symbols(res):
    for sym in all_symbols:
示例#16
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = '../src/pal'

def manifest_file(file):
    if 'SGX_RUN' in os.environ and os.environ['SGX_RUN'] == '1':
        return file + '.manifest.sgx'
    else:
        return file + '.manifest'

# Running Bootstrap
regression = Regression(loader, "Bootstrap")

regression.add_check(name="Basic Bootstrapping",
    check=lambda res: "User Program Started" in res[0].log)

regression.add_check(name="Control Block: Executable Name",
    check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].log and \
            "argv[0] = file:Bootstrap" in res[0].log)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].log and \
           "argv[1] = a" in res[0].log and "argv[2] = b" in res[0].log and \
           "argv[3] = c" in res[0].log and "argv[4] = d" in res[0].log)
示例#17
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = '../src/pal'

regression = Regression(loader, "Misc", timeout=5000)

regression.add_check(name="Query System Time",
    check=lambda res: "Query System Time OK" in res[0].log)

regression.add_check(name="Delay Execution for 10000 Microseconds",
    check=lambda res: "Delay Execution for 10000 Microseconds OK" in res[0].log)

regression.add_check(name="Delay Execution for 3 Seconds",
    check=lambda res: "Delay Execution for 3 Seconds OK" in res[0].log)

regression.add_check(name="Generate Random Bits",
    check=lambda res: "Generate Random Bits OK" in res[0].log)

regression.run_checks()
示例#18
0
def predict2(x_val,x,y):
    meanx=np.mean(x)
    stddevx=np.std(x)
    meany=np.mean(y)
    stddevy=np.std(y)
    x=featureNormalize(x)
    y=featureNormalize(y)

    reg = Regression()
    reg.set_learning_rate(0.001)
    reg.set_max_iterations(10000)
    reg.set_l1_penalty(0.1)
    reg.set_l2_penalty(0.1)
    reg.set_tolerance(1e-5)
    theta, cost, it = reg.polynomial_regression(x, y, 5)

    x_val=(x_val-meanx)/stddevx
    y_val=reg.predict([x_val])
    y_val=np.array(y_val)*stddevy+meany

    return(y_val)
示例#19
0
## This test is specifically for the reference monitor code, not process creation in general.
## It is not well-tested right now, but keep the tests around for future work.

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_SEC']

if not os.path.exists(loader):
    print("Reference monitor mode is not available on this platform")
    exit(0)

regression = Regression(loader, "Process")

def check_times(target, lines, times):
    count = 0
    for line in lines:
        if target == line:
            count += 1
    return count == times

regression.add_check(name="Process Creation",
    check=lambda res: check_times("Child Process Created", res[0].log, 3))

regression.add_check(name="Process Creation Arguments",
    check=lambda res: check_times("argv[0] = Process", res[0].log, 3) and
                      check_times("argv[1] = Child",   res[0].log, 3))

regression.add_check(name="Process Channel Transmission",
    check=lambda res: check_times("Process Write 1 OK",            res[0].log, 3) and
                      check_times("Process Read 1: Hello World 1", res[0].log, 3) and
示例#20
0
    if os.path.exists("dir_delete.tmp"):
        shutil.rmtree("dir_delete.tmp")

    global dir_files
    os.mkdir("dir_exist.tmp")
    dir_files = []
    for i in range(5):
        file = ''.join([random.choice(string.ascii_letters) for i in range(8)])
        f = open("dir_exist.tmp/" + file, "w")
        f.close()
        dir_files.append(file)

    os.mkdir("dir_delete.tmp")


regression = Regression(loader, "Directory", prepare_dirs)

regression.add_check(name="Basic Directory Opening",
                     check=lambda res: "Directory Open Test 1 OK" in res[
                         0].log and "Directory Open Test 2 OK" in res[0].log
                     and "Directory Open Test 3 OK" in res[0].log)

regression.add_check(name="Basic Directory Creation",
                     check=lambda res: "Directory Creation Test 1 OK" in res[
                         0].log and "Directory Creation Test 2 OK" in res[0].
                     log and "Directory Creation Test 3 OK" in res[0].log)


def check_read(res):
    global dir_files
    for file in dir_files:
示例#21
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running getsockopt
regression = Regression(loader, "getsockopt", None)

regression.add_check(name="getsockopt",
    check=lambda res: "getsockopt: Got socket type OK" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#22
0
            Month=request.form['Month'])
        yearList, avgTempList = dataFrame.SplitYearAvgTemp(p)
        avgTempList = np.array(avgTempList).astype(np.float)
        yearList = np.array(yearList).astype(np.float)
        training_x, training_y, test_x, test_y, predictLine_x, predictLine_y, cofficient, intercept = regression.fnLinearRegression1(
            yearList, avgTempList)
        return dumps({
            "training_x": training_x,
            "training_y": training_y,
            "test_x": test_x,
            "test_y": test_y,
            "predictLine_x": predictLine_x,
            "predictLine_y": predictLine_y,
            "cofficient": cofficient,
            "intercept": intercept
        })
    except:
        return "error"


if __name__ == '__main__':
    global dataFrame
    global regression
    global classification
    dataFrame = DataParser()
    regression = Regression()
    classification = Classification()
    visualization = visualizations.Visualization()

    app.run()
示例#23
0
def main():
    #read input data
    my_data = np.genfromtxt('regression_data.txt', dtype=float, delimiter=',')
     
    # create input and output numpy arrays     
    x = my_data[:,0]
    y = my_data[:,1]
     
    # create regression class object
    reg = Regression()
     
    # set learning rate
    reg.set_learning_rate(0.001)
     
    # set maximum iterations
    reg.set_max_iterations(20000)
     
    # set l1 and l2 penalty
    reg.set_l1_penalty(0.1)
    reg.set_l2_penalty(0.1)
     
    # set tolerance
    reg.set_tolerance(1e-5)
     
    # fit a polynomial regression model 
    theta, cost, it = reg.polynomial_regression(x, y, 6)
     
    print "Regression coefficients :" + str(theta)
    print "Minimum cost function: " + str(cost)
    print "Iterations taken: " + str(it)
     
    # predict values for new input
    z = np.linspace(-2, 2, 4/0.01)
    prediction = reg.predict(z)
     
    # plot
    fig = plt.figure()
    plt.plot(x,y,'.', label='Input data')
    plt.plot(z,prediction,'r-', label='Prediction')
    plt.legend(loc=4)
    fig.suptitle('Polynomial Regression Fit')
    plt.xlabel('x (input)')
    plt.ylabel('y (predicted)')
    plt.savefig('fit_values.eps')
    plt.show()
示例#24
0
#!/usr/bin/env python2

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Misc", timeout=5000)

regression.add_check(name="Query System Time",
                     check=lambda res: "Query System Time OK" in res[0].log)

regression.add_check(name="Delay Execution for 10000 Microseconds",
                     check=lambda res:
                     "Delay Execution for 10000 Microseconds OK" in res[0].log)

regression.add_check(
    name="Delay Execution for 3 Seconds",
    check=lambda res: "Delay Execution for 3 Seconds OK" in res[0].log)

regression.add_check(name="Generate Random Bits",
                     check=lambda res: "Generate Random Bits OK" in res[0].log)

rv = regression.run_checks()
if rv: sys.exit(rv)

regression = Regression(loader, "Hex")

regression.add_check(name="Hex 2 String Helper Function",
                     check=lambda res: "Hex test 1 is deadbeef" in res[0].log and \
                     "Hex test 2 is cdcdcdcdcdcdcdcd" in res[0].log)
示例#25
0
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation
from regression import Regression, panda_to_numpy

data_path = "./biocarbonate.xls"
data = pd.read_excel(data_path)

x_train = data['X']
y_train = data['Y']

# y must be an nX1 array
# x myst be an nXm array where m is the number of different variables for the reggression
# regression() returns an 1Xm+1 array wich are the weights +1 is for the constant
x = panda_to_numpy(data, 'X')
y = data['Y']
model = Regression(y, x)
weights = model.train(epochs=200000, a=0.00005, print_loss=False)
prediction = model.predict(8)
print(prediction)

# plot the results
ones = np.ones(len(x_train))
x_pred = np.column_stack((np.linspace(x_train.min(), x_train.max(),
                                      len(x_train)), ones))
y_pred = weights @ x_pred.T

plt.title("Prediction of bicarbonate")
plt.xlabel("ph")
plt.ylabel("biocarbonates ppm")
plt.plot(x_train, y_train, "ro", x_pred, y_pred, "g--")
plt.axis([x_train.min(), x_train.max(), y_train.min(), y_train.max()])
示例#26
0
    global file_exist
    file_exist = ''.join(
        [random.choice(string.ascii_letters) for i in range(mmap.PAGESIZE)])

    with open("file_exist.tmp", "w") as f:
        f.write(file_exist)

    if os.path.exists("file_nonexist.tmp"):
        os.remove("file_nonexist.tmp")

    with open("file_delete.tmp", "w") as f:
        f.write(file_exist)


# Running File
regression = Regression(loader, "File", prepare_files)

regression.add_check(name="Basic File Opening",
                     check=lambda res: "File Open Test 1 OK" in res[
                         0].log and "File Open Test 2 OK" in res[0].log and
                     "File Open Test 3 OK" in res[0].log)

regression.add_check(name="Basic File Creation",
                     check=lambda res: "File Creation Test 1 OK" in res[
                         0].log and "File Creation Test 2 OK" in res[0].log and
                     "File Creation Test 3 OK" in res[0].log)

regression.add_check(
    name="File Reading",
    check=lambda res:
    ("Read Test 1 (0th - 40th): " + file_exist[0:40]) in res[0].log and
示例#27
0
def prepare_files(args):
    global file_exist
    file_exist = ''.join([random.choice(string.ascii_letters) for i in range(mmap.PAGESIZE)])

    with open("file_exist.tmp", "w") as f:
        f.write(file_exist)

    if os.path.exists("file_nonexist.tmp"):
        os.remove("file_nonexist.tmp")

    with open("file_delete.tmp", "w") as f:
        f.write(file_exist)

# Running File
regression = Regression(loader, "File", prepare_files)

regression.add_check(name="Basic File Opening",
    check=lambda res: "File Open Test 1 OK" in res[0].log and
                      "File Open Test 2 OK" in res[0].log and
                      "File Open Test 3 OK" in res[0].log)

regression.add_check(name="Basic File Creation",
    check=lambda res: "File Creation Test 1 OK" in res[0].log and
                      "File Creation Test 2 OK" in res[0].log and
                      "File Creation Test 3 OK" in res[0].log)

regression.add_check(name="File Reading",
    check=lambda res: ("Read Test 1 (0th - 40th): " + file_exist[0:40]) in res[0].log and
                      ("Read Test 2 (0th - 40th): " + file_exist[0:40]) in res[0].log and
                      ("Read Test 3 (200th - 240th): " + file_exist[200:240]) in res[0].log)
示例#28
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running stat
regression = Regression(loader, "getcwd")

regression.add_check(name="Getcwd syscall",
    check=lambda res: "[bss_cwd_buf] getcwd succeeded: /" in res[0].out and \
                      "[mmapped_cwd_buf] getcwd succeeded: /" in res[0].out)

rv = regression.run_checks()
if rv:
    sys.exit(rv)
示例#29
0
import sys
from regression import Regression

loader = sys.argv[1]

# Running sigaltstack
regression = Regression(loader, "sigaltstack")

messages = (
    "OK on sigaltstack in main thread before alarm",
    "&act == 0x",
    "sig 14 count 1 goes off with sp=0x",
    "OK on signal stack",
    "OK on sigaltstack in handler",
    "sig 14 count 2 goes off with sp=0x",
    "OK on signal stack",
    "OK on sigaltstack in handler",
    "sig 14 count 3 goes off with sp=0x",
    "OK on signal stack",
    "OK on sigaltstack in handler",
    "OK on sigaltstack in main thread",
    "done exiting",
)

regression.add_check(name="Sigaltstack Test",
    check=lambda res: all([x in res[0].out for x in messages]))

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#30
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running Bootstrap
regression = Regression(loader, "bootstrap")

regression.add_check(name="Basic Bootstrapping",
    check=lambda res: "User Program Started" in res[0].out)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].out and \
            "argv[0] = file:bootstrap" in res[0].out)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].out and \
           "argv[0] = file:bootstrap" in res[0].out and \
           "argv[1] = a" in res[0].out and "argv[2] = b" in res[0].out and \
           "argv[3] = c" in res[0].out and "argv[4] = d" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)

# Running Exec
regression = Regression(loader, "exec")

regression.add_check(name="2 page child binary",
    check=lambda res: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 " in res[0].out)
示例#31
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]
sgx = os.environ.get('SGX_RUN') == '1'

# Running Bootstrap
regression = Regression(loader, "mmap-file", None, 60000)

regression.add_check(name="Private mmap beyond file range",
    check=lambda res: "mmap test 6 passed" in res[0].out and \
                      "mmap test 7 passed" in res[0].out)

regression.add_check(name="Private mmap beyond file range (after fork)",
    check=lambda res: "mmap test 1 passed" in res[0].out and \
                      "mmap test 2 passed" in res[0].out and \
                      "mmap test 3 passed" in res[0].out and \
                      "mmap test 4 passed" in res[0].out)

# On SGX, SIGBUS isn't always implemented correctly, for lack
# of memory protection.  For now, some of these cases won't work.
if not sgx:
    regression.add_check(name="SIGBUS test",
                         check=lambda res: "mmap test 5 passed" in res[0].out and \
                         "mmap test 8 passed" in res[0].out)


rv = regression.run_checks()
if rv: sys.exit(rv)
示例#32
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']
try:
    sgx = os.environ['SGX_RUN']
except KeyError:
    sgx = 0

regression = Regression(loader, "Memory")

regression.add_check(name="Memory Allocation",
                     check=lambda res: "Memory Allocation OK" in res[0].log)

regression.add_check(
    name="Memory Allocation with Address",
    check=lambda res: "Memory Allocation with Address OK" in res[0].log)

regression.add_check(
    name="Memory Protection",
    flaky=sgx,
    check=lambda res: "Memory Allocation Protection (RW) OK" in res[
        0].log and "Memory Protection (R) OK" in res[0].log)

regression.add_check(name="Memory Deallocation",
                     flaky=sgx,
                     check=lambda res: "Memory Deallocation OK" in res[0].log)

示例#33
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running Bootstrap
regression = Regression(loader, "proc")

regression.add_check(name="Base /proc files present",
    check=lambda res: "/proc/1/.." in res[0].out and \
                      "/proc/1/cwd" in res[0].out and \
                      "/proc/1/exe" in res[0].out and \
                      "/proc/1/root" in res[0].out and \
                      "/proc/1/fd" in res[0].out and \
                      "/proc/1/maps" in res[0].out and \
                      "/proc/." in res[0].out and \
                      "/proc/1" in res[0].out and \
                      "/proc/self" in res[0].out and \
                      "/proc/meminfo" in res[0].out and \
                      "/proc/cpuinfo" in res[0].out)


rv = regression.run_checks()
if rv: sys.exit(rv)
示例#34
0
#!/usr/bin/env python2

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_SEC']

if not os.path.exists(loader):
    print "Reference monitor mode is not available on this platform"
    exit(0)

regression = Regression(loader, "Process")


def check_times(target, lines, times):
    count = 0
    for line in lines:
        if target == line:
            count += 1
    return count == times


regression.add_check(
    name="Process Creation",
    check=lambda res: check_times("Child Process Created", res[0].log, 3))

regression.add_check(
    name="Process Creation Arguments",
    check=lambda res: check_times("argv[0] = Process", res[
        0].log, 3) and check_times("argv[1] = Child", res[0].log, 3))
示例#35
0
import os, sys
from regression import Regression

loader = sys.argv[1]

# Running Bootstrap
regression = Regression(loader, "proc-path")

regression.add_check(name="Base /proc path present",
    check=lambda res: "proc path test success" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#36
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running stat
regression = Regression(loader, "stat_invalid_args")

regression.add_check(name="Stat with invalid arguments",
    check=lambda res: "stat(invalid-path-ptr) correctly returned error" in res[0].out and \
                      "stat(invalid-buf-ptr) correctly returned error" in res[0].out and \
                      "lstat(invalid-path-ptr) correctly returned error" in res[0].out and \
                      "lstat(invalid-buf-ptr) correctly returned error" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)

# Running fstat
regression = Regression(loader, "fstat_cwd")

regression.add_check(
    name="Fstat on a directory",
    check=lambda res: "fstat returned the fd type as S_IFDIR" in res[0].out)

rv = regression.run_checks()
if rv:
    sys.exit(rv)
示例#37
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = '../src/pal'

regression = Regression(loader, "Thread")

regression.add_check(name="Thread Creation",
    check=lambda res: "Child Thread Created" in res[0].log and
                      "Run in Child Thread: Hello World" in res[0].log)

regression.add_check(name="Multiple Threads Run in Parallel",
    check=lambda res: "Threads Run in Parallel OK" in res[0].log)

regression.add_check(name="Set Thread Private Segment Register",
    check=lambda res: "Private Message (FS Segment) 1: Hello World 1" in res[0].log and
                      "Private Message (FS Segment) 2: Hello World 2" in res[0].log)

regression.add_check(name="Thread Exit",
    check=lambda res: "Child Thread Exited" in res[0].log)

regression.run_checks()
示例#38
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Exception")

regression.add_check(name="Exception Handling (Div-by-Zero)",
                     check=lambda res: any([
                         line.startswith("Div-by-Zero Exception Handler")
                         for line in res[0].log
                     ]))

regression.add_check(name="Exception Handling (Memory Fault)",
                     check=lambda res: any([
                         line.startswith("Memory Fault Exception Handler")
                         for line in res[0].log
                     ]))

regression.add_check(name="Exception Handler Swap",
                     check=lambda res: any([
                         line.startswith("Div-by-Zero Exception Handler 1")
                         for line in res[0].log
                     ]) and any([
                         line.startswith("Div-by-Zero Exception Handler 2")
                         for line in res[0].log
                     ]))

regression.add_check(name="Exception Handling (Set Context)",
示例#39
0
import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running Bootstrap
regression = Regression(loader, "bootstrap")

regression.add_check(name="Basic Bootstrapping",
    check=lambda res: "User Program Started" in res[0].out)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].out and \
            "argv[0] = file:bootstrap" in res[0].out)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].out and \
           "argv[0] = file:bootstrap" in res[0].out and \
           "argv[1] = a" in res[0].out and "argv[2] = b" in res[0].out and \
           "argv[3] = c" in res[0].out and "argv[4] = d" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)

# Running Exec
regression = Regression(loader, "exec")

regression.add_check(name="2 page child binary",
    check=lambda res: "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 " in res[0].out)
示例#40
0
    print("Bulk IPC not supported on SGX")
    exit(0)

## XXX Should really be running these tests as part of CI
if not os.path.exists('/dev/gipc'):
    print("GIPC not loaded; skipping these tests\n")
    exit(0)


def prepare_files(args):
    with open("ipc_mapping.tmp", "w") as f:
        f.write("Hello World")
        os.ftruncate(f.fileno(), mmap.PAGESIZE)


regression = Regression(loader, "Ipc", prepare_files)


def check_times(target, lines, times):
    count = 0
    for line in lines:
        if target == line:
            count += 1
    return count == times


regression.add_check(
    name="Create and Join Physical Memory Bulk Copy Store",
    check=lambda res: check_times(
        "Create Physical Memory Store OK", res[0].log, 5) and check_times(
            "Join Physical Memory Store OK", res[0].log, 5))
示例#41
0
import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Socket")

regression.add_check(name="TCP Socket Creation",
    check=lambda res: "TCP Creation 1 OK" in res[0].log)

regression.add_check(name="TCP Socket Connection",
    check=lambda res: "TCP Connection 1 OK" in res[0].log)

regression.add_check(name="TCP Socket Transmission",
    check=lambda res: "TCP Write 1 OK" in res[0].log and
                      "TCP Read 1: Hello World 1" in res[0].log and
                      "TCP Write 2 OK" in res[0].log and
                      "TCP Read 2: Hello World 2" in res[0].log)

regression.add_check(name="UDP Socket Creation",
    check=lambda res: "UDP Creation 1 OK" in res[0].log)

regression.add_check(name="UDP Socket Connection",
    check=lambda res: "UDP Connection 1 OK" in res[0].log)

regression.add_check(name="UDP Socket Transmission",
    check=lambda res: "UDP Write 1 OK" in res[0].log and
                      "UDP Read 1: Hello World 1" in res[0].log and
                      "UDP Write 2 OK" in res[0].log and
                      "UDP Read 2: Hello World 2" in res[0].log)
import pandas as pd
from regression import Regression

x = Regression()

df = pd.read_csv("trips_summary_covid_pub.csv")
df['date'] = pd.to_datetime(df['date'])
df['pre_covid'] = (df.date < "12/19/2020").astype("int")
df = df.loc[df.pre_covid == 0]

# TOTAL REVENUE MODEL
x.select_cols = [
    'pickup_community_area', 'hour', 'week_day', 'cases', 'total_revenue'
]
x.dummy_cols = ['pickup_community_area', 'hour', 'week_day']
x.y_col = 'total_revenue'

res_revenue = x.time_split(df)

# TOTAL COUNT MODEL
x.select_cols = ['pickup_community_area', 'hour', 'week_day', 'cases', 'count']
x.y_col = 'count'

res_count = x.time_split(df)

# TOTAL SECONDS MODEL
x.select_cols = [
    'pickup_community_area', 'hour', 'week_day', 'cases', 'trip_seconds_tot'
]
x.y_col = 'trip_seconds_tot'
示例#43
0
import sys
from regression import Regression

loader = sys.argv[1]

regression = Regression(loader, "proc_cpuinfo", None, 50000)

regression.add_check(name="proc/cpuinfo Linux-based formatting",
    check=lambda res: "cpuinfo test passed" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#44
0
import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "SendHandle")

def check_times(target, lines, times):
    count = 0
    for line in lines:
        if target == line:
            count += 1
    return count == times

regression.add_check(name="Send and Receive Handles across Processes",
    check=lambda res: check_times("Send Handle OK", res[0].log, 3) and
                      check_times("Receive Handle OK", res[0].log, 3))

regression.add_check(name="Send Pipe Handle",
        check=lambda res: check_times("Receive Pipe Handle: Hello World", res[0].log, 1))

regression.add_check(name="Send Socket Handle",
        check=lambda res: check_times("Receive Socket Handle: Hello World", res[0].log, 1))

regression.add_check(name="Send File Handle",
        check=lambda res: check_times("Receive File Handle: Hello World", res[0].log, 1))

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#45
0
    if os.path.exists("dir_delete.tmp"):
        shutil.rmtree("dir_delete.tmp")

    global dir_files
    os.mkdir("dir_exist.tmp")
    dir_files = []
    for i in range(5):
        file = ''.join([random.choice(string.ascii_letters) for i in range(8)])
        f = open("dir_exist.tmp/" + file, "w")
        f.close()
        dir_files.append(file)

    os.mkdir("dir_delete.tmp")


regression = Regression(loader, "Directory", prepare_dirs)

regression.add_check(name="Basic Directory Opening",
    check=lambda res: "Directory Open Test 1 OK" in res[0].log and
                      "Directory Open Test 2 OK" in res[0].log and
                      "Directory Open Test 3 OK" in res[0].log)

regression.add_check(name="Basic Directory Creation",
    check=lambda res: "Directory Creation Test 1 OK" in res[0].log and
                      "Directory Creation Test 2 OK" in res[0].log and
                      "Directory Creation Test 3 OK" in res[0].log)

def check_read(res):
    global dir_files
    for file in dir_files:
        if ("Read Directory: " + file) not in res[0].log:
示例#46
0
import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Pipe")

regression.add_check(name="Pipe Creation",
    check=lambda res: "Pipe Creation 1 OK" in res[0].log)

regression.add_check(name="Pipe Attributes",
    check=lambda res: "Pipe Attribute Query 1 on pipesrv returned OK" in res[0].log)

regression.add_check(name="Pipe Connection",
    check=lambda res: "Pipe Connection 1 OK" in res[0].log)

regression.add_check(name="Pipe Transmission",
    check=lambda res: "Pipe Write 1 OK" in res[0].log and
                      "Pipe Read 1: Hello World 1" in res[0].log and
                      "Pipe Write 2 OK" in res[0].log and
                      "Pipe Read 2: Hello World 2" in res[0].log)

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#47
0
import sys
from regression import Regression

loader = sys.argv[1]

# Running udp
regression = Regression(loader, "udp", None)

regression.add_check(name="udp",
    check=lambda res:
      "Data: This is packet 0" in res[0].out and
      "Data: This is packet 1" in res[0].out and
      "Data: This is packet 2" in res[0].out and
      "Data: This is packet 3" in res[0].out and
      "Data: This is packet 4" in res[0].out and
      "Data: This is packet 5" in res[0].out and
      "Data: This is packet 6" in res[0].out and
      "Data: This is packet 7" in res[0].out and
      "Data: This is packet 8" in res[0].out and
      "Data: This is packet 9" in res[0].out)

rv = regression.run_checks()
if rv:
    sys.exit(rv)
示例#48
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Misc", timeout=5000)

regression.add_check(name="Query System Time",
                     check=lambda res: "Query System Time OK" in res[0].log)

regression.add_check(name="Delay Execution for 10000 Microseconds",
                     check=lambda res:
                     "Delay Execution for 10000 Microseconds OK" in res[0].log)

regression.add_check(
    name="Delay Execution for 3 Seconds",
    check=lambda res: "Delay Execution for 3 Seconds OK" in res[0].log)

regression.add_check(name="Generate Random Bits",
                     check=lambda res: "Generate Random Bits OK" in res[0].log)

rv = regression.run_checks()
if rv:
    sys.exit(rv)
示例#49
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = '../src/pal'

regression = Regression(loader, "Exception")

def check_exception1(res):
    for line in res[0].log:
        if not line:
            continue
        if line.startswith('Div-by-Zero Exception Handler'):
            return True
    return False

regression.add_check(name="Exception Handling (Div-by-Zero)", check=check_exception1)

def check_exception2(res):
    for line in res[0].log:
        if not line:
            continue
        if line.startswith('Memory Fault Exception Handler'):
            return True
    return False

regression.add_check(name="Exception Handling (Memory Fault)", check=check_exception2)

def check_exception3(res):
    found1 = False
示例#50
0
import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "Process", timeout=8000)


def check_times(target, lines, times):
    count = 0
    for line in lines:
        if target == line:
            count += 1
    return count == times


regression.add_check(
    name="Process Creation",
    check=lambda res: check_times("Child Process Created", res[0].log, 3))

regression.add_check(
    name="Process Creation Arguments",
    check=lambda res: check_times("argv[0] = Process", res[
        0].log, 3) and check_times("argv[1] = Child", res[0].log, 3))

regression.add_check(
    name="Process Channel Transmission",
    check=lambda res: check_times("Process Write 1 OK", res[
        0].log, 3) and check_times("Process Read 1: Hello World 1", res[
            0].log, 3) and check_times("Process Write 2 OK", res[0].log, 3) and
    check_times("Process Read 2: Hello World 2", res[0].log, 3))
示例#51
0
import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']
sgx = os.environ.get('SGX_RUN') == '1'

def manifest_file(file):
    if sgx:
        return file + '.manifest.sgx'
    else:
        return file + '.manifest'

# Running Bootstrap
regression = Regression(loader, "Bootstrap")

regression.add_check(name="Basic Bootstrapping",
    check=lambda res: "User Program Started" in res[0].log)

regression.add_check(name="Control Block: Executable Name",
    check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].log and \
            "argv[0] = file:Bootstrap" in res[0].log)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].log and \
           "argv[1] = a" in res[0].log and "argv[2] = b" in res[0].log and \
           "argv[3] = c" in res[0].log and "argv[4] = d" in res[0].log)
示例#52
0
    def run(self):
        io = FileIO()
        will_update = self.update

        if self.csvfile:
            stock_tse = io.read_from_csv(self.code, self.csvfile)

            msg = "".join([
                "Read data from csv: ", self.code, " Records: ",
                str(len(stock_tse))
            ])
            print(msg)

            if self.update and len(stock_tse) > 0:
                index = pd.date_range(start=stock_tse.index[-1],
                                      periods=2,
                                      freq='B')
                ts = pd.Series(None, index=index)
                next_day = ts.index[1]
                t = next_day.strftime('%Y-%m-%d')
                newdata = io.read_data(self.code, start=t, end=self.end)

                msg = "".join([
                    "Read data from web: ", self.code, " New records: ",
                    str(len(newdata))
                ])
                print(msg)
                if len(newdata) < 1:
                    will_update = False
                else:
                    print(newdata.ix[-1, :])

                stock_tse = stock_tse.combine_first(newdata)
                io.save_data(stock_tse, self.code, 'stock_')
        else:
            stock_tse = io.read_data(self.code, start=self.start, end=self.end)

            msg = "".join([
                "Read data from web: ", self.code, " Records: ",
                str(len(stock_tse))
            ])
            print(msg)

        if stock_tse.empty:
            msg = "".join(["Data empty: ", self.code])
            print(msg)
            return None

        if not self.csvfile:
            io.save_data(stock_tse, self.code, 'stock_')

        try:
            stock_d = stock_tse.asfreq('B').dropna()[self.days:]

            ti = TechnicalIndicators(stock_d)

            ti.calc_sma()
            ti.calc_sma(timeperiod=5)
            ti.calc_sma(timeperiod=25)
            ti.calc_sma(timeperiod=50)
            ti.calc_sma(timeperiod=75)
            ewma = ti.calc_ewma(span=5)
            ewma = ti.calc_ewma(span=25)
            ewma = ti.calc_ewma(span=50)
            ewma = ti.calc_ewma(span=75)
            bbands = ti.calc_bbands()
            sar = ti.calc_sar()
            draw = Draw(self.code, self.fullname)

            ret = ti.calc_ret_index()
            ti.calc_vol(ret['ret_index'])
            rsi = ti.calc_rsi(timeperiod=9)
            rsi = ti.calc_rsi(timeperiod=14)
            mfi = ti.calc_mfi()
            roc = ti.calc_roc(timeperiod=10)
            roc = ti.calc_roc(timeperiod=25)
            roc = ti.calc_roc(timeperiod=50)
            roc = ti.calc_roc(timeperiod=75)
            roc = ti.calc_roc(timeperiod=150)
            ti.calc_cci()
            ultosc = ti.calc_ultosc()
            stoch = ti.calc_stoch()
            ti.calc_stochf()
            ti.calc_macd()
            willr = ti.calc_willr()
            ti.calc_momentum(timeperiod=10)
            ti.calc_momentum(timeperiod=25)
            tr = ti.calc_tr()
            ti.calc_atr()
            ti.calc_natr()
            vr = ti.calc_volume_rate()

            ret_index = ti.stock['ret_index']
            clf = Classifier(self.clffile)
            train_X, train_y = clf.train(ret_index, will_update)
            msg = "".join(["Train Records: ", str(len(train_y))])
            print(msg)
            clf_result = clf.classify(ret_index)[0]
            msg = "".join(["Classified: ", str(clf_result)])
            print(msg)
            ti.stock.ix[-1, 'classified'] = clf_result

            reg = Regression(self.regfile, alpha=1, regression_type="Ridge")
            train_X, train_y = reg.train(ret_index, will_update)
            msg = "".join(["Train Records: ", str(len(train_y))])
            base = ti.stock_raw['Adj Close'][0]
            reg_result = int(reg.predict(ret_index, base)[0])
            msg = "".join(["Predicted: ", str(reg_result)])
            print(msg)
            ti.stock.ix[-1, 'predicted'] = reg_result

            if len(self.reference) > 0:
                ti.calc_rolling_corr(self.reference)
                ref = ti.stock['rolling_corr']
            else:
                ref = []

            io.save_data(io.merge_df(stock_d, ti.stock), self.code, 'ti_')

            draw.plot(stock_d,
                      ewma,
                      bbands,
                      sar,
                      rsi,
                      roc,
                      mfi,
                      ultosc,
                      willr,
                      stoch,
                      tr,
                      vr,
                      clf_result,
                      reg_result,
                      ref,
                      axis=self.axis,
                      complexity=self.complexity)

            return ti

        except (ValueError, KeyError) as e:
            print("Error occured in", self.code, "at analysis.py")
            print('ErrorType:', str(type(e)))
            print('ErrorMessage:', str(e))
            return None
示例#53
0
import sys
from regression import Regression

loader = sys.argv[1]

# Running udp
regression = Regression(loader, "udp", None, 50000)

regression.add_check(
    name="udp",
    check=lambda res: "Data: This is packet 0" in res[0].out and
    "Data: This is packet 1" in res[0].out and "Data: This is packet 2" in res[
        0].out and "Data: This is packet 3" in res[0].out and
    "Data: This is packet 4" in res[0].out and "Data: This is packet 5" in res[
        0].out and "Data: This is packet 6" in res[0].out and
    "Data: This is packet 7" in res[0].out and "Data: This is packet 8" in res[
        0].out and "Data: This is packet 9" in res[0].out)

rv = regression.run_checks()
if rv:
    sys.exit(rv)
示例#54
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_SEC']

if not os.path.exists(loader):
    print "Reference monitor mode is not available on this platform"
    exit(0)

# Running Bootstrap
regression = Regression(loader, "Bootstrap")

regression.add_check(name="Basic Bootstrapping",
                     check=lambda res: "User Program Started" in res[0].log)

regression.add_check(
    name="Control Block: Executable Name",
    check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)

regression.add_check(
    name="Control Block: Default Manifest",
    check=lambda res: "Loaded Manifest: file:manifest" in res[0].log)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].log and \
            "argv[0] = file:Bootstrap" in res[0].log)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
示例#55
0
#!/usr/bin/env python2

import os, sys, mmap
from regression import Regression

loader = sys.argv[1]

# Running Bootstrap
regression = Regression(loader, "getdents", None, 10000)

# This doesn't catch extraneous entries, but should be fine
# until the LTP test can be run (need symlink support)
regression.add_check(name="Directory listing (32-bit)",
    check=lambda res: "getdents: setup ok" in res[0].out and \
                      "getdents32: . [0x4]" in res[0].out and \
                      "getdents32: .. [0x4]" in res[0].out and \
                      "getdents32: file1 [0x8]" in res[0].out and \
                      "getdents32: file2 [0x8]" in res[0].out and \
                      "getdents32: dir3 [0x4]" in res[0].out)

regression.add_check(name="Directory listing (64-bit)",
    check=lambda res: "getdents: setup ok" in res[0].out and \
                      "getdents64: . [0x4]" in res[0].out and \
                      "getdents64: .. [0x4]" in res[0].out and \
                      "getdents64: file1 [0x8]" in res[0].out and \
                      "getdents64: file2 [0x8]" in res[0].out and \
                      "getdents64: dir3 [0x4]" in res[0].out)

rv = regression.run_checks()
if rv: sys.exit(rv)
示例#56
0
loader = os.environ['PAL_LOADER']
try:
    sgx = os.environ['SGX_RUN']
except KeyError:
    sgx = False


def manifest_file(file):
    if 'SGX_RUN' in os.environ and os.environ['SGX_RUN'] == '1':
        return file + '.manifest.sgx'
    else:
        return file + '.manifest'


# Running Bootstrap
regression = Regression(loader, "Bootstrap")

regression.add_check(name="Basic Bootstrapping",
                     check=lambda res: "User Program Started" in res[0].log)

regression.add_check(
    name="Control Block: Executable Name",
    check=lambda res: "Loaded Executable: file:Bootstrap" in res[0].log)

regression.add_check(name="One Argument Given",
    check=lambda res: "# of Arguments: 1" in res[0].log and \
            "argv[0] = file:Bootstrap" in res[0].log)

regression.add_check(name="Five Arguments Given",
    args = ['a', 'b', 'c', 'd'],
    check=lambda res: "# of Arguments: 5" in res[0].log and \
示例#57
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = "../src/pal"

regression = Regression(loader, "Pipe")

regression.add_check(name="Pipe Creation", check=lambda res: "Pipe Creation 1 OK" in res[0].log)

regression.add_check(name="Pipe Connection", check=lambda res: "Pipe Connection 1 OK" in res[0].log)

regression.add_check(
    name="Pipe Transmission",
    check=lambda res: "Pipe Write 1 OK" in res[0].log
    and "Pipe Read 1: Hello World 1" in res[0].log
    and "Pipe Write 2 OK" in res[0].log
    and "Pipe Read 2: Hello World 2" in res[0].log,
)

regression.run_checks()
示例#58
0
#!/usr/bin/env python2

import os, sys, mmap
from regression import Regression

loader = os.environ['PAL_LOADER']

regression = Regression(loader, "SendHandle")


def check_times(target, lines, times):
    count = 0
    for line in lines:
        if target == line:
            count += 1
    return count == times


regression.add_check(
    name="Send and Receive Handles across Processes",
    check=lambda res: check_times("Send Handle OK", res[
        0].log, 3) and check_times("Receive Handle OK", res[0].log, 3))

regression.add_check(name="Send Pipe Handle",
                     check=lambda res: check_times(
                         "Receive Pipe Handle: Hello World", res[0].log, 1))

regression.add_check(name="Send Socket Handle",
                     check=lambda res: check_times(
                         "Receive Socket Handle: Hello World", res[0].log, 1))
示例#59
0
#!/usr/bin/python

import os, sys, mmap
from regression import Regression

loader = '../src/pal'

regression = Regression(loader, "Exception")

regression.add_check(name="Exception Handling (Div-by-Zero)",
    check=lambda res: any([line.startswith("Div-by-Zero Exception Handler") for line in res[0].log]))

regression.add_check(name="Exception Handling (Memory Fault)",
    check=lambda res: any([line.startswith("Memory Fault Exception Handler") for line in res[0].log]))

regression.add_check(name="Exception Handler Swap",
    check=lambda res: any([line.startswith("Div-by-Zero Exception Handler 1") for line in res[0].log]) and
                      any([line.startswith("Div-by-Zero Exception Handler 2") for line in res[0].log]))

regression.add_check(name="Exception Handling (Set Context)",
    check=lambda res: any([line.startswith("Div-by-Zero Exception Handler 1") for line in res[0].log]))

regression.run_checks()
        plt.ylabel("")
        data.plot.scatter(x=x, y=y, ax=ax, s=10, label=y_label2, legend=False)
        # call teh legend, place atop the image on the left
        # bbox_to_anchor used to specify exact placement of label
        plt.legend(loc="upper left",
                   labels=[y_label1, y_label2],
                   bbox_to_anchor=(0, 1.17))
        # remove lines marking units on the axis
        ax.xaxis.set_ticks_position('none')
        ax.yaxis.set_ticks_position('none')
        plt.show()
        plt.close()


data = pd.read_csv("cleanedEconFreedomData.csv")
reg = Regression()

y_var = ["GDP per Capita (PPP)"]
x_vars = [
    "Judical Effectiveness", "Property Rights", "Fiscal Health",
    "Investment Freedom ", "Financial Freedom", "Inflation (%)",
    "Public Debt (% of GDP)"
]
reg.OLS("GDP Per Capita Restricted", data, y_var, x_vars)
plot_scatter_with_estimator(reg.data, x_vars, y_var)

x_vars_unrestricted = x_vars
x_vars_restricted = ["Judical Effectiveness", "Property Rights"]
reg.OLS("GDP Per Capita Restricted", data, y_var, x_vars_restricted)
reg.OLS("GDP Per Capita Unrestricted", data, y_var, x_vars_unrestricted)