def test_missing_output_warning(tmpdir):
""" Test missing output file exception """
input_file = tmpdir.join(u'in1')
assert not input_file.check()
input_file.write(u'')
try:
regress(u'Get-Content' if on_windows() else u'cat',
path=tmpdir.strpath,
error=True)
raise AssertionError
except OutputNotFound:
pass
def test_multiple_files(tmpdir):
""" Simple regress test with multiple input files """
testing_string = u'testing\n'
input_file = tmpdir.join(u'in1')
assert not input_file.check()
input_file.write(testing_string)
output_file = tmpdir.join(u'asd1')
assert not output_file.check()
output_file.write(testing_string)
try:
regress(u'somethingnonexistent',
in_prefix=u'abc',
out_prefix=u'asd',
path=tmpdir.strpath)
raise AssertionError
except CommandNotFound:
pass
def test_command_not_found(tmpdir):
""" Test command not found exception """
testing_string = u'testing\n'
input_file = tmpdir.join(u'in1')
assert not input_file.check()
input_file.write(testing_string)
output_file = tmpdir.join(u'asd1')
assert not output_file.check()
output_file.write(testing_string)
try:
regress(u'somethingnonexistent',
in_prefix=u'abc',
out_prefix=u'asd',
path=tmpdir.strpath)
raise AssertionError
except CommandNotFound as err:
assert (u'somethingnonexistent' in str(err))
def curve(train_data, train_labels, test_data, test_labels, lagrange):
avg_errors = np.zeros(train_data.shape[DATA_AXIS])
for trial in range(10):
indices = [i for i in range(train_data.shape[DATA_AXIS])]
shuffle(indices)
for num_samples in range(1, train_data.shape[DATA_AXIS] + 1):
data = train_data[indices[:num_samples]]
labels = train_labels[indices[:num_samples]]
coefs = regress(data, labels, lagrange)
predicted = predict(test_data, coefs)
avg_errors[num_samples - 1] += mse(test_labels, predicted)
return avg_errors / 10
def test_simple_regression(tmpdir):
""" Run simple regress test with cat and 1 input file """
testing_string = u'testing\n'
input_file = tmpdir.join(u'in1')
assert not input_file.check()
input_file.write(testing_string)
output_file = tmpdir.join(u'out1')
assert not output_file.check()
output_file.write(testing_string)
fails = regress(u'Get-Content' if on_windows() else u'cat',
path=tmpdir.strpath)
print(fails)
assert not fails
def main():
args = parseArgs()
map = Map()
for line in sys.stdin.readlines():
line = line.rstrip('\n')
map.addLine(line)
map.init()
print "Loaded", map.width, "x", map.height
print map
if args.regress:
print "Regressing..."
regress(map)
elif args.aggress:
print "Aggressing..."
aggress(map)
elif args.svm:
svm(map)
elif args.eval:
eval(args.eval, map)
def test_changed_prefix_regression(tmpdir):
""" Simple regress test with modified input and output prefixes """
testing_string = u'testing\n'
input_file = tmpdir.join(u'abc1')
assert not input_file.check()
input_file.write(testing_string)
output_file = tmpdir.join(u'asd1')
assert not output_file.check()
output_file.write(testing_string)
fails = regress(u'Get-Content' if on_windows() else u'cat',
in_prefix=u'abc',
out_prefix=u'asd',
path=tmpdir.strpath)
print(fails)
assert not fails
def test_awk_with_second_column_options(tmpdir):
""" Test regress passing multiple extra options/arguments to awk """
testing_string = u'a\tregress\tb'
expected_output = u'regress\n'
input_file1 = tmpdir.join(u'in1')
assert not input_file1.check()
input_file1.write(testing_string)
output_file1 = tmpdir.join(u'out1')
assert not output_file1.check()
output_file1.write(expected_output)
fails = regress(u'awk',
path=tmpdir.strpath,
options=[u'-F', u'\t', u'{print $2}'])
print(fails)
assert not fails
def test_cat_with_non_printing_options(tmpdir):
""" Test regress passing one extra option/argument to cat """
testing_string1 = u'\t\ttesting\n'
input_file1 = tmpdir.join(u'in1')
assert not input_file1.check()
input_file1.write(testing_string1)
output_file1 = tmpdir.join(u'out1')
assert not output_file1.check()
output_file1.write(u'^I^Itesting\n')
fails = regress(u'Get-Content' if on_windows() else u'cat',
path=tmpdir.strpath,
error=True,
options=[u'-t'])
print(fails)
assert not fails
def cross_validate(name, file):
data, labels = load_data(file, dummy=1.0)
log = open('logs/q3/%s.log' % name, 'w')
stdout.write("Evaluating data set '%s'..." % name)
stdout.flush()
# split the data into folds
indices = [i for i in range(data.shape[DATA_AXIS])]
shuffle(indices)
fold_size = ceil(float(data.shape[DATA_AXIS]) / NUM_FOLDS)
# evaluate each lagrange
best_error = maxint
for lagrange in range(0, 151):
avg_error = 0.0
# try each fold average errors
for i in range(NUM_FOLDS):
low = int(i * fold_size)
high = int((i + 1) * fold_size)
train_indices = indices[:low] + indices[high:]
test_indices = indices[low:high]
coefs = regress(data[train_indices], labels[train_indices],
lagrange)
predicted = predict(data[test_indices], coefs)
error = mse(labels[test_indices], predicted)
avg_error += error / NUM_FOLDS
message = 'lagrange=%d fold=%d error=%.3f\n'
log.write(message % (lagrange, i, error))
# update best error and lagrange if result is better
if avg_error < best_error:
best_error = avg_error
best_lagrange = lagrange
# report the results
stdout.write('done.\n')
stdout.write('Best Lagrange value is %d.\n' % best_lagrange)
stdout.write('Best error is %.3f.\n' % best_error)
stdout.write("Logs written to '%s'.\n" % log.name)
stdout.flush()
log.close()
def test_one_fail(tmpdir):
""" Test regress failing 1/2 test and make sure the right one
failed and the actual output is right """
testing_string = u'testing\n'
input_file1 = tmpdir.join(u'in1')
assert not input_file1.check()
input_file1.write(testing_string)
input_file2 = tmpdir.join(u'in2')
assert not input_file2.check()
input_file2.write(testing_string)
output_file1 = tmpdir.join(u'out1')
assert not output_file1.check()
output_file1.write(testing_string)
output_file2 = tmpdir.join(u'out2')
assert not output_file2.check()
output_file2.write(testing_string + u'fail')
fails = regress(u'Get-Content' if on_windows() else u'cat',
path=tmpdir.strpath)
print(fails)
assert len(fails) == 1
assert fails[0][0].endswith(u'in2')
assert fails[0][1] == testing_string
def sigmoid_est(x,y, five=False):
try:
slope, alpha=regress.regress(y, x) #get slope
if slope > 0:
p1 = np.min(y, axis=0)
p2 = np.max(y, axis=0) - p1
sign=1.0
else:
sign = -1.0
p1 = np.max(y, axis=0)
p2 = np.min(y, axis=0) - p1
p4 = np.mean(x, axis=0)
#print xprime, yprime,p2/(p1-yprime) -1,p4 - xprime, slope
p3 = sign * np.log(np.abs(slope))
except Exception as e:
print "Sig Est ", p1, p2, p3, p4
raise e
if five:
#print "Est", (p1, p2, p3, p4, 1.0)
return (p1, p2, p3, p4, 1.0)
else:
return (p1, p2, p3, p4)
def plot_regression(name, train_file, test_file):
stdout.write("Drawing plot for data set '%s'... " % name)
stdout.flush()
train_data, train_labels = load_data(train_file, dummy=1.0)
test_data, test_labels = load_data(test_file, dummy=1.0)
lagranges = [lagrange for lagrange in range(151)]
train_errors = []
test_errors = []
log = open('logs/q1/%s.log' % name, 'w')
# for each lagrange regress and calculate error
for lagrange in lagranges:
coefs = regress(train_data, train_labels, lagrange)
predicted = predict(train_data, coefs)
train_error = mse(train_labels, predicted)
train_errors.append(train_error)
predicted = predict(test_data, coefs)
test_error = mse(test_labels, predicted)
test_errors.append(test_error)
message = 'lagrange=%d train_error=%.3f test_error=%.3f\n'
log.write(message % (lagrange, train_error, test_error))
# plot errors as a function of the lagrange
pyplot.figure()
pyplot.xlim(0, 150)
pyplot.title("Data set '%s'" % name)
pyplot.xlabel('Lagrange multiplier')
pyplot.ylabel('Mean squared error')
pyplot.plot(lagranges, train_errors, label="Training")
pyplot.plot(lagranges, test_errors, label="Testing")
pyplot.legend(loc='lower right')
pyplot.savefig('plots/q1/%s.png' % name)
stdout.write("done.\n")
stdout.write("Plot image written to 'plots/q1/%s.png'.\n" % name)
stdout.write("Plot data written to '%s'.\n" % log.name)
stdout.flush()
log.close()
bre=eval(brl)
for Y in range(len(bre[1:-1])):
Year=bre[1:-1][Y]
lowyr= list(b.data().years()).index(bre[Y])
highyr=list(b.data().years()).index(bre[Y+2])+1
midyr=list(b.data().years()).index(bre[Y+1])+1
print >>outf, '"'+str(brl)+'"', breaklist[brl], Year, bre[Y], bre[Y+2], yearseglist[str(bre[Y:Y+3])],
pcb=convergent_breaks.resample_break(b.data().ys()[lowyr:highyr],b.data().years()[lowyr:highyr], N=100,withmode=True)
print >>outf,pcb[1][0], pcb[2][0], yearlist[Year], pcb[3][0][0], pcb[3][0][1],
if pcb[3][1] == None:
print >>outf,0, 0,
else:
print >>outf,pcb[3][1][0], pcb[3][1][1],
betaseg, alphaseg=regress.regress(b.data().ys()[lowyr:highyr],b.data().years()[lowyr:highyr])
betalow, alphalow=regress.regress(b.data().ys()[lowyr:midyr],b.data().years()[lowyr:midyr])
betahigh, alphahigh=regress.regress(b.data().ys()[midyr:highyr],b.data().years()[midyr:highyr])
print >>outf,betaseg, betalow, betahigh,
#now do a diagnostic on 15 year enclosed
low15 = max(0, midyr-8)
hi15=midyr+8
beta15, alpha15=regress.regress(b.data().ys()[low15:hi15],b.data().years()[low15:hi15])
print >>outf,(alphahigh+betahigh * bre[Y+1])-(alphalow+betalow * bre[Y+1]), beta15,
pcb15=convergent_breaks.resample_break(b.data().ys()[low15:hi15],b.data().years()[low15:hi15], N=100,withmode=True)
print >>outf,pcb15[1][0], pcb15[2][0], pcb15[3][0][0], pcb15[3][0][1],
if pcb15[3][1] == None:
print >>outf,0, 0
else:
self.packets.append(ip)
# Main
def usage():
print "Usage: %s [-dg]" % sys.argv[0]
try:
opts, args = getopt.getopt(sys.argv[1:],"dg", ["debug", "generate"])
except getopt.GetoptError:
usage()
sys.exit(2)
debug = 0
generate = 0
for o, a in opts:
if o in ("-d", "--debug"):
debug = 1
if o in ("-g", "--generate"):
generate = 1
if o in ("-h", "--help"):
usage()
sys.exit(1)
reg = regress.regress("detect probe", "../honeyd", "config.1", debug)
reg.generate = generate
reg.run(DetectSFSROpen())
reg.run(DetectSAAROpen())
reg.run(DetectSAARClose())
reg.run(DetectSFSRClose())
reg.finish()
ip.len += len(ip.data)
self.packets.append(ip)
# Main
def usage():
print "Usage: %s [-dg]" % sys.argv[0]
try:
opts, args = getopt.getopt(sys.argv[1:], "dg", ["debug", "generate"])
except getopt.GetoptError:
usage()
sys.exit(2)
debug = 0
generate = 0
for o, a in opts:
if o in ("-d", "--debug"):
debug = 1
if o in ("-g", "--generate"):
generate = 1
if o in ("-h", "--help"):
usage()
sys.exit(1)
reg = regress.regress("routing behavior", "../honeyd", "config.2", debug)
reg.generate = generate
reg.run(RouteOne())
reg.finish()
ip.data = payload
ip.len += len(ip.data)
self.packets.append(ip)
# Main
def usage():
print "Usage: %s [-dg]" % sys.argv[0]
try:
opts, args = getopt.getopt(sys.argv[1:],"dg", ["debug", "generate"])
except getopt.GetoptError:
usage()
sys.exit(2)
debug = 0
generate = 0
for o, a in opts:
if o in ("-d", "--debug"):
debug = 1
if o in ("-g", "--generate"):
generate = 1
if o in ("-h", "--help"):
usage()
sys.exit(1)
reg = regress.regress("general networking tests", "../honeyd", "config.1", debug)
reg.generate = generate
reg.run(Ping())
reg.run(TCPOpen())
reg.finish()
count += 1
output.close()
input.close()
return count
# Main
failures = []
prints = {}
number = make_configuration("config.nmap", "../nmap.prints")
reg = regress.regress("Nmap fingerprints", "../honeyd", "config.nmap")
reg.start_honeyd(reg.configuration)
reg.fe.read()
success = 0
partial = 0
nothing = 0
for count in range(0, number):
res = nmap(count)
if res == 1:
success += 1
elif res == 2:
partial += 1
else:
nothing += 1
#getting indicators
wbdata.get_indicator(source=14)
#'SG.GEN.PARL.ZS' = % of women in national parliament
#'NY.GDP.MKTP.CD' = GDP (current US$)
#'SH.STA.MMRT2' = Maternal mortality ratio (modeled estimate, per 100,000 live births)
indicators = {
"SG.GEN.PARL.ZS": "Female_MPs",
"NY.GDP.MKTP.CD": "GDP",
"SH.STA.MMRT": "Maternal_Mortality"
}
#getting data
countries = [
i['id'] for i in wbdata.get_country(incomelevel="LIC", display=False)
]
Gender = wbdata.get_dataframe(indicators,
country=countries,
convert_date=True,
keep_levels=True)
Gender.to_csv('C:/Users/Ecem/class/hwdata.csv')
data = Gender
x1 = Gender.iloc[:, 0:1]
x2 = Gender.iloc[:, 1:2]
y = Gender.iloc[:, 2:3]
x = np.hstack((x1, x2))
from regress import regress
regress(Gender)
y2_HS = mat(y_HS[m1:]) y2_HS = y2_HS.T y2_IoHS = mat(y_IoHS[m1:]) y2_IoHS = y2_IoHS.T y2_DNF = mat(y_DNF[m1:]) y2_DNF = y2_DNF.T print "\n\nFinished preparing data!\n\n" sys.stdout.flush() regularizers = [0.00000001, 0.0000001, 0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000, 2000, 5000 ] for l_coeff in regularizers: print "\nl_coeff = ",l_coeff print "\nDT" reg.regress(x1, y1_DT, x2, y2_DT, l_coeff) reg.regress(x1_deg2, y1_DT, x2_deg2, y2_DT, l_coeff) # reg.regress(x1_deg3, y1_DT, x2_deg3, y2_DT, l_coeff) reg.regress(eig_fea1, y1_DT, eig_fea2, y2_DT, l_coeff) reg.regress(eig_fea1_deg2, y1_DT, eig_fea2_deg2, y2_DT, l_coeff) print "\nHS" reg.regress(x1, y1_HS, x2, y2_HS, l_coeff) reg.regress(x1_deg2, y1_HS, x2_deg2, y2_HS, l_coeff) # reg.regress(x1_deg3, y1_HS, x2_deg3, y2_HS, l_coeff) reg.regress(eig_fea1, y1_HS, eig_fea2, y2_HS, l_coeff) reg.regress(eig_fea1_deg2, y1_HS, eig_fea2_deg2, y2_HS, l_coeff) print "\nIoHS" reg.regress(x1, y1_IoHS, x2, y2_IoHS, l_coeff) reg.regress(x1_deg2, y1_IoHS, x2_deg2, y2_IoHS, l_coeff)
import regress
import getData
import numpy as np
(xList, yList) = getData.getData()
# Feature scaling GDP data to achieve meaningful results
xArray = np.array(xList)
xMin = np.amin(xArray)
xMax = np.amax(xArray)
xScaled = (xArray - xMin) / (xMax - xMin)
x = xScaled.tolist()
# Converting percent values to decimals for urban population ratio - Normalizing
yArray = np.array(yList)
yNormalized = yArray / 100
y = yNormalized.tolist()
(alpha, beta, standardError, lowerBound, upperBound) = regress.regress(x, y)
print("With given format of Y = alpha + beta*X")
print("Alpha value is: " + str(alpha))
print("Beta value is: " + str(beta))
print("Standard error is: " + str(standardError))
print("95% Confidence interval for beta: " + str(lowerBound) + " - " +
str(upperBound))
regress.plotRegressionGraph(x, y, alpha, beta)
bp=brkrpt(os.environ["HOMEPATH"]+"\\Documents\\abrupt\\4Roger_Nature_SVN_264\\HadCRUT.4.2.0.0.annual_ns_avg//HadCRUT.4.2.0.0.annual_ns_avg.txt_0.trace")
yearsegs, ysegs, xsegs = bp.segments()
pre98Years=yearsegs[-2]
pre98temps=ysegs[-2]
post98Years=yearsegs[-1]
post98temps=ysegs[-1]
for i in range(len(pre98Years)):
print "Pre", i, pre98Years[i], pre98temps[i]
for i in range(len(post98Years)):
print "Post", i+len(pre98Years), post98Years[i], post98temps[i]
beta1,alpha1=regress.regress(pre98temps, pre98Years)
beta2,alpha2=regress.regress(post98temps, post98Years)
print beta1, alpha1, beta2, alpha2
allYears=[y for y in pre98Years]
allYears.extend(post98Years)
print allYears
alltemps=[t for t in pre98temps]
alltemps.extend(post98temps)
crossyhat1=-np.array([alpha1+beta1* y for y in allYears]) +alltemps
crossyhat2=-np.array([alpha2+beta2* y for y in allYears]) +alltemps
p=dnet.IP_PROTO_TCP)
ip.data = payload
ip.len += len(ip.data)
self.packets.append(ip)
# Main
def usage():
print "Usage: %s [-dg]" % sys.argv[0]
try:
opts, args = getopt.getopt(sys.argv[1:],"dg", ["debug", "generate"])
except getopt.GetoptError:
usage()
sys.exit(2)
debug = 0
generate = 0
for o, a in opts:
if o in ("-d", "--debug"):
debug = 1
if o in ("-g", "--generate"):
generate = 1
if o in ("-h", "--help"):
usage()
sys.exit(1)
reg = regress.regress("routing behavior", "../honeyd", "config.2", debug)
reg.generate = generate
reg.run(RouteOne())
reg.finish()
count += 1
output.close()
input.close()
return count
# Main
failures = []
prints = {}
number = make_configuration("config.nmap", "../nmap.prints")
reg = regress.regress("Nmap fingerprints", "../honeyd", "config.nmap")
reg.start_honeyd(reg.configuration)
reg.fe.read()
success = 0
partial = 0
nothing = 0
for count in range(0, number):
res = nmap(count)
if res == 1:
success += 1
elif res == 2:
partial += 1
else:
nothing += 1
