def load_train_test_data():
rating_train_data = os.path.join(DATA_DIR, 'ua.base')
rating_test_data = os.path.join(DATA_DIR, 'ua.test')
ratings_base, ratings_test = ProcessData.process_data(None, None, None, 'train_test_data',
rating_train_data, rating_test_data)
# ProcessData.check_loaded_data(users, ratings, items, ratings_base, ratings_test)
train_data, test_data = ProcessData.create_train_test_SFrame(ratings_base, ratings_test)
return train_data, test_data
def run():
if config.runMode.find('train') >= 0:
trainFeature = Feature(config.trainFile, 'train')
testFeature = Feature(config.testFile, 'test')
else:
testFeature = Feature(config.readFile, 'test')
if config.formatConvert:
df = dataFormat()
df.convert()
config.globalCheck()
# TODO
# directoryCheck()
config.swLog = open(config.outDir + config.fLog, 'w')
config.swResRaw = open(config.outDir + config.fResRaw, 'w')
config.swTune = open(config.outDir + config.fTune, 'w')
if config.runMode.find('tune') >= 0:
print('\nstart tune...')
config.swLog.write('\nstart tune...\n')
tuneStochasticOptimizer()
elif config.runMode.find('train') >= 0:
print('\nstart training...')
config.swLog.write('\nstart training...\n')
if config.runMode.find('rich') >= 0:
richEdge.train()
else:
train()
elif config.runMode.find('test') >= 0:
config.swLog.write('\nstart testing...\n')
if config.runMode.find('rich') >= 0:
richEdge.test()
else:
test()
ProcessData.tocrfoutput(config.outFolder + 'outputTag.txt',
config.outputFile,
config.tempFile + '/test.raw.txt')
elif config.rumMode.find('cv') >= 0:
print('\nstart cross validation')
config.swLog.write('\nstart cross validation\n')
crossValidation()
else:
raise Exception('error')
config.swLog.close()
config.swResRaw.close()
config.swTune.close()
if config.runMode.find('train') >= 0:
resSummarize.summarize()
#ClearDirectory(config.tempFile)
print('finished.')
def preprocess(datapath, outputpath, dsfactor=16):
filelist2d = glob.glob(datapath + "2d/*")
for filename in filelist2d:
fn3d = datapath + "3d/" + os.path.basename(filename).split(
'.2d.h5')[0] + ".3d.h5"
data, dataf = pd.loaddata(filename, key="features")
data3d = pd.loaddata(fn3d, is3D=True)
dsdata, samples = pd.downsample(data, dsfactor, *data.shape)
pd.savedata("2d", dataf, dsdata, "features", outputpath,
os.path.basename(filename))
def __init__(self):
self.dataset = ProcessData(train_ratio=0.8,process_raw_data=False,do_augment=False, image_type='OA',
get_scale_center=False, single_sample=True,
do_blur=False, do_crop=False, do_deform=False, do_flip=True)
self.model = cnn_skipC_OA_model.cnn_skipC_OA_model(
criterion=nn.MSELoss(),
optimizer= torch.optim.Adam,
learning_rate=0.001,
weight_decay=0
)
def apiAdd(clientID, sourceID, data, schemaOverRide):
if isinstance(data, dict):
dataList = [data]
else:
dataList = data
toInsert = []
for item in dataList:
item = ProcessData.flatten(data=item)
item = ProcessData.enrich(data=item, schemaOverRide=schemaOverRide)
date = ProcessData.getNow()
toInsert.append({'created': date, 'data': item})
SourceHandler.add(clientID=clientID, sourceID=sourceID, data=toInsert)
def CreateAccSummaryString(OutFile):
df1 = ProcessData.loadBehDataFile(OutFile)
# Process the file
df1 = ProcessData.processBehavioralFile(df1)
# Extract accuracy
ACCdata = df1.data.Acc.groupby(df1.data.LoadLevels).mean()
# Create a string of accuracy at each load level
SumString = ""
count = 1
for i in ACCdata:
SumString = "%s,%d:%2.0f"%(SumString,count,i*100)
count = count + 1;
return SumString[1:]
def CreateAccSummaryString(OutFile):
df1 = ProcessData.loadBehDataFile(OutFile)
# Process the file
df1 = ProcessData.processBehavioralFile(df1)
# Extract accuracy
ACCdata = df1.data.Acc.groupby(df1.data.LoadLevels).mean()
# Create a string of accuracy at each load level
SumString = ""
count = 1
for i in ACCdata:
SumString = "%s,%d:%2.0f" % (SumString, count, i * 100)
count = count + 1
return SumString[1:]
def main(portfolio):
pricedatadf = portfolio.pricedatadf
fixedday = pd.to_datetime(portfolio.inputdate_init).day
# Init wheights Dataframe and datetime index (weightsdfindex)
column_names = ['Weight' + x for x in portfolio.tickersymbolist]
weightsdf = pd.DataFrame(columns=column_names)
weightsdfindex = []
# first allocation day
nextallocday = portfolio.pricedatadf[portfolio.inputdate_init:].index[0]
i = 0
while nextallocday != 'Out':
print('++++++++++++++++++++++++++++++++++++++++')
# get DataFrame with price returns
# All Days full Year
#returnsdf = ProcessData.getdailyreturns_year(pricedatadf, nextallocday, fixedday)
# All Days full Month
#returnsdf = ProcessData.getdailyreturns_month(pricedatadf, nextallocday, fixedday)
# One Day per Month During one Year (example: fist day of each month)
returnsdf = ProcessData.getslicedreturns_month(pricedatadf, nextallocday, fixedday)
# get next day allocation day
nextallocday = ProcessData.nextallocationday_month(pricedatadf, nextallocday, fixedday)
# get covariance matrix and expected returnsdf
covariance, expectedreturns = getcovexpect(returnsdf)
# minimize
weights = minimizefunction(covariance, expectedreturns)
# get Standard Deviation
std = getstd(weights, covariance)
# save date index in wheighs DataFrame
weightsdfindex.append(returnsdf.index[-1])
weightsdf.loc[i] = weights
i += 1
print('Expected returnsdf:\t' + str(expectedreturns))
print('Weigths: \t' + str(weights*100))
print('Standard Deviation:\t' + str(std))
print('Next Alloc Day: \t' + str(nextallocday))
print()
# get index and save weightsdf to csv
weightsdf = getweightsdf(weightsdf, weightsdfindex)
return weightsdf
def ProcessAFile(inputFile,subid,Tag,col):
print inputFile
print subid
df1 = ProcessData.loadBehDataFile(inputFile)
df1 = ProcessData.processBehavioralFile(df1)
wkB = WriteToGoogleSpreadSheet.openWorkBook()
# Check to see if a worksheet exists
worksheetName = 'behData_%s'%(subid)
try:
wkS = wkB.worksheet(worksheetName)
except:
wkS = WriteToGoogleSpreadSheet.createWorksheet(wkB, subid)
ProcessData.writeSummaryBehavioralDataToFile(df1, wkS, Tag, col)
return df1
def builder(directed,weighted,window_size,inputDoc):
# inputDoc: String
# directed: True for directed, False for undirected
# weighted: True for weighted graph, False for unweighted graph
inputDoc = ProcessData.ProcessDoc(inputDoc)
# print inputDoc
# for direction
if directed:
G = nx.DiGraph()
else:
G = nx.Graph()
# add edges and nodes
for s in inputDoc:
# Ignore the sentence with less than window_size words
if len(s) >= window_size:
for word in s:
# print word
G.add_node(word,name = word)
window = list(s[:window_size])
for i in range(len(s)):
# print window
for word in window[1:]:
if word != window[0]:
if G.has_edge(window[0],word):
G[window[0]][word]['weight'] += 1
else:
G.add_edge(window[0],word,weight=1)
# print window[0],word
if i < (len(s) - window_size):
window.append(s[i+window_size])
del window[0]
return G
# graph = builder(directed,weighted,window_size,testDoc)
# nx.write_gml(graph,'test1.gml')
def auth(request, email, password, action):
action = action[0]
password = password[0]
email = email[0]
redirectURL = '/?'
validityCheck = action in [
'login', 'register'
] and ProcessData.typeCheck(typeName='email', value=email) is True
if validityCheck is True:
if action == 'login':
client = Client.get(email=email, password=password)
client.login(request=request)
try:
pass
except:
redirectURL += 'error=The entered email and password do not match'
elif action == 'register':
try:
client = Client.create(email=email, password=password)
client.login(request=request)
except DuplicateUser:
redirectURL += 'error=This email address already exist in our records'
else:
redirectURL += 'error=Enter a valid email address'
# return HttpResponse("WORKED")
return redirect(redirectURL)
def do_POST(self):
form = cgi.FieldStorage(
fp=self.rfile,
headers=self.headers,
environ={'REQUEST_METHOD':'POST',
'CONTENT_TYPE':self.headers['Content-Type'],
})
id = uuid.uuid4()
logfilepath = 'Data/' + str(id) + '.log'
with open(logfilepath, 'w') as logfile:
logfile.write(form['datafile'].value)
directory = 'SharedData/' + str(id)
if not os.path.exists(directory):
os.makedirs(directory)
jsonobj = ProcessData.fromFile(logfilepath)
jsonpath = directory + '/tasks.json'
with open(jsonpath, 'w') as jsonfile:
jsonfile.write(json.dumps(jsonobj)+"\n")
os.remove(logfilepath)
print
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
# This is hacky and horrible and I'll look into a better way.
self.wfile.write(
"""
<!DOCTYPE html><html lang="en"><html><head><meta charset="UTF-8"><title>LegionViz</title><link rel="stylesheet" href="webstyle.css" type="text/css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.6/d3.min.js" charset="utf-8"></script>
</head><body><div id="header"><div><a href="index.html" class="logo"><h1>LegionVis</h1></a></div>
<ul><li><a href="index.html">home</a></li><li class="selected"><a href="tool.html">tool</a></li><li><a href="contact.html">contact</a></li></ul></div>
<div id="body"><div class="header"><p><a href='http://"""+form.headers.get('host')+"/Shared/"+str(id)+"""/display.html'>Click here to access your visualization</a></p></div></div></body></html>
""")
def Boot(self, FNAME_process):
FP_process = open(FNAME_process, 'r')
for line in FP_process:
current_process = line.split(',')
process = ProcessData.Process(self.current_PID, current_process)
self.QueueProcess(self.input_queue, process)
self.current_PID += 1
FP_process.close()
def train():
user_data_file = os.path.join(DATA_DIR, 'u.user')
rating_data_file = os.path.join(DATA_DIR, 'u.data')
item_rating_data = os.path.join(DATA_DIR, 'u.item')
users, ratings, items = ProcessData.process_data(user_data_file, rating_data_file, item_rating_data,
type='metadata')
train_data, test_data = load_train_test_data()
popularity_model, item_sim_model = Recommender.train_models(train_data, users, items)
return popularity_model, item_sim_model, users, items
def Main():
config = Configuration('configuration.json')
if config.isProcessingDataFirst:
ProcessData.process_data_files()
if config.isGeneratingGraph:
GraphData.create_plots()
if config.isGeneratingClocklabFiles:
FormatDataForClockLab.create_clock_lab_formatted_bulked_out_with_zeros_text_file(
)
if config.isGeneratingChronosFitFile:
FormatDataForChronosFit.create_cronos_fit_formatted_file()
if config.isShowingIndividualFish and config.isGeneratingDistanceSums:
DataFormatter.generate_distance_sums_for_individual_fish()
def generate(reqId):
def getRequestParameters():
query = ("SELECT NODE_ID, SOURCE_ID FROM training_requests WHERE REQUEST_ID = %s")
cursor.execute(query,(reqId,))
out = cursor.fetchall()[0]
return out[0], out[1]
nodeId, srcId = getRequestParameters()
input, output,dates = process.getTrainingData(reqId)
numInputs = len(input[0])
numOutputs = len(output[0])
file = os.path.join(dir, "./inputs/input_{}".format(reqId))
def writeHeader():
with open(file, 'w') as f:
f.write("_H:\t$Name")
for i in range(numInputs):
if (i == 0):
f.write(inpFormat.format(i) + firstFormat.format(numInputs))
else:
f.write(inpFormat.format(i))
for i in range(numOutputs):
if (i == 0):
f.write(outpFormat.format(i) + firstFormat.format(numOutputs))
else:
f.write(outpFormat.format(i))
def writeBody():
with open(file, 'a') as f:
for i in range(len(input)):
f.write("\n")
f.write("_D:\t")
f.write("{}\t".format(dates[i]))
for j in input[i]:
f.write(str(j)+"\t")
for j in output[i]:
f.write(str(j))
def addToFile():
writeHeader()
writeBody()
addToFile()
def anomaly_detection(path):
"""
Anomaly detection process.
:return:
"""
origin_data, sample_data, sample_abnormal_index, sample_normal_index, abnormal \
= data.preprocess(path, sample_bound=SAMPLE_BOUND, abnormal_bound=ABNORMAL_BOUND, normal_bound=NORMAL_BOUND, abnormal_rate=ABNORMAL_RATE)
sample_num = sample_data.shape[0]
centers, result = graph.classify_abnormal_data(abnormal, K_ABNORMAL)
Y = init_detection_tag(sample_num, sample_abnormal_index,
sample_normal_index, result)
if USE_LEARNING:
U, Du, De, H, W, first_abnormal, first_normal = init_hyper_graph(
sample_data, abnormal, centers)
else:
U, first_abnormal, first_normal = init_hyper_graph_quick(
sample_data, abnormal, centers)
true_tag = origin_data[:, 0]
true_data_index = np.nonzero(true_tag > 0)[0].tolist()
false_data_index = np.nonzero(true_tag <= 0)[0].tolist()
tp = set(first_abnormal).intersection(set(true_data_index))
tn = set(first_normal).intersection(set(false_data_index))
fp = set(first_abnormal).intersection(set(false_data_index))
if len(fp) > 0:
fp_array = origin_data[list(fp), :]
fp_array = fp_array[fp_array[:, 1].argsort()]
np.savetxt("fp_data.txt", fp_array, delimiter=",")
accuracy = (len(tp) + len(tn)) * 1.0 / true_tag.shape[0]
abnormal_precision = len(tp) * 1.0 / len(first_abnormal)
abnormal_recall = len(tp) * 1.0 / len(true_data_index)
f_value = 2.0 * abnormal_precision * abnormal_recall / (
abnormal_precision + abnormal_recall)
print 'Accuracy:' + str(accuracy)
print 'First Abnormal precision:' + str(abnormal_precision)
print 'First Abnormal recall:' + str(abnormal_recall)
print 'First F value:' + str(f_value)
if USE_LEARNING:
F = learn.joint_learning(LAMBDA,
LEARNING_RATE,
U,
Y,
Du,
De,
H,
W,
MU,
joint=USE_JOINT_LEARNING)
measure_result(F, origin_data)
def CreateAccSummaryString(OutFile):
df1 = ProcessData.loadBehDataFile(OutFile)
# Process the file
df1 = ProcessData.processBehavioralFile(df1)
# Extract accuracy
ACCdata = df1.data.Acc.groupby(df1.data.LoadLevels).mean()
# Create a string of accuracy at each load level
SumString = ""
count = 1
if 'TrainFB' in df1.RunType:
# If there is feedback then the scoring doesn't work because there are
# extra rows in the data file.
DataScale = 2.6666666
else:
DataScale = 1
for i in ACCdata:
SumString = "%s,%d:%2.0f"%(SumString,count,i*100*DataScale)
count = count + 1;
return SumString[1:]
def GenerateModel(dictionary, id):
demand = ProcessData.ProcessData(dictionary)
Prediction.GenerateModel(dictionary, demand)
latestPrediction = Prediction.GetPrediction(dictionary, demand)
loadDict[id] = latestPrediction
with open("savedPredictions.txt", "a+") as outfile:
json.dump({"id": id, "predictions": latestPrediction}, outfile)
outfile.write("\n")
print("latestPrediction type: ", type(latestPrediction),
"\n latestPrediction: ", latestPrediction)
return "Model Generation completed."
def LoadData(filename):
xml = filename + ".xml"
trace = "Data\\" + filename + "\\"
title, labels, jconf, authors, FullName, organization = ProcessData.ProcessingRawData(
trace + xml)
title = ProcessData.Wipe_off_Punctuation(title)
title_vocab, title_split = ProcessData.Split_Title(title)
title_one_hot, Max_Sequence_Len, vocab_size = ProcessData.One_hot_encoding(
title_vocab, title_split)
title_one_hot_padding = ProcessData.Padding_One_hot(
title_one_hot, Max_Sequence_Len)
author_vocab, authors_split = ProcessData.Split_Authors(authors)
edge_type = np.load(trace + "edge_type.npy")
edge_list_src = np.load(trace + "edge_list_src.npy")
edge_list_dst = np.load(trace + "edge_list_dst.npy")
num_nodes = len(authors_split)
edge_norm = [1 for i in range(len(edge_type))]
print("Number of edges: ", len(edge_list_src))
print("Number of nodes: ", len(authors_split))
print("Number of class: ", max(labels) + 1)
train_idx = random.sample(range(len(authors_split)),
int(len(authors_split) * 0.8))
test_idx = []
for i in range(len(authors_split)):
if i not in train_idx:
test_idx.append(i)
inputs = title_one_hot_padding
labels = labels
return edge_type, edge_list_src, edge_list_dst, num_nodes, edge_norm, vocab_size, train_idx, test_idx, inputs, labels
def main():
# Process data
filename = "Data/pima-indians-diabetes.data.csv"
dataset = ProcessData.loadCsv(filename)
# print("Loaded data file {0} with {1} rows".format(filename,len(dataset)))
splitRatio = 0.67
train, test = ProcessData.splitDataset(dataset, splitRatio)
# print("Split {0} rows into train with {1} and test with {2}".format(len(dataset),train,test))
#Get data feature
separated = GetFeature.separateByClass(train)
# print("Separated instances: {0}".format(separated))
summaries = GetFeature.summarizeByClass(train)
# print("Summaries[0]: {0}".format(summaries[0]) +"\n"
# + "Summaries[1] {0} ".format(summaries[1]))
#Preciction
predictions = Prediction.getPredictions(summaries, test)
accuracy = Prediction.getAccuracy(test, predictions)
print("Accuracy:{0}%".format(accuracy))
def main():
args = len(sys.argv)
input_file = sys.argv[1]
src = sys.argv[2]
dest = sys.argv[3]
with open(input_file, 'r') as file:
data = file.read().split('\n')
prepared_data = P.prepareData(data)
if args == 4:
uniform_cost_search(prepared_data, src, dest)
elif args == 5:
heuristic_file = sys.argv[4]
with open(heuristic_file, 'r') as file:
data = file.read().split('\n')
heuristic_data = P.prepareHeuristicData(data)
a_star_search(prepared_data, src, dest, heuristic_data)
else:
print('Invalid number of arguments...')
sys.exit()
def dist_plot(self, file, column):
data = ProcessData.importData(file)
print('hi')
data['levels'] = (data[column] - data[column].min()) / (
data[column].max() - data[column].min())
sns.despine(fig=self.fig,
ax=self.axes,
top=True,
right=True,
left=True,
bottom=True)
plot = sns.distplot(data['levels'], ax=self.axes)
plot.set_yticks([])
def train():
fileName = "trainData.model"
test = ReadFile.QAData("training.data")
test.readFile()
ProD.wordSeg(test)
ProD.delHighFre_useless(test)
ProD.delHighFre_psg(test)
trainModel(test, fileName)
def getData():
if request.method == 'POST':
#extract parameters from request.
year = request.form['year']
month = request.form['month']
type = request.form['type']
#construct a sql date type.
date = "{0}-{1}-01".format(year, month)
# todo add a method to build a geoJson.
snowGeoJson = None
if (type == 'polygon'):
snowGeoJson = ProcessData.recoverData(date, type)
snowGeoJson = json.dumps(snowGeoJson)
# FOR TESTING SPATIOTEMPORAL SIMULATION FOR ECITY ONLY.
#-----------------------------
elif (type == 'ecity'):
snowGeoJson = ProcessData.recoverData(date, type)
snowGeoJson = json.dumps(snowGeoJson)
#------------------------------
elif (type == 'point'):
objList = ProcessData.recoverData(date, type)
snowGeoJson = ProcessData.fromListToJson(objList)
# Setting a response with json and header.
response = Response(snowGeoJson, mimetype='application/json')
# Adding ACAO to response header for security purpose, so it can match the header of client.
response.headers.add('Access-Control-Allow-Origin', '*')
# print(response.get_data())
return response
# the code below is executed if the request method
# was GET or the credentials were invalid
return 'failed'
def hhdetection(stop_event):
while not stop_event.is_set():
stop_event.wait(5)
num_distance = 0
array_i = []
array_j = []
distance = []
sum_distance = 0
max_avg = 0
sum_avg = 0
print()
print()
print("=====================================================")
print("NOW HH DETECTION!!!!")
for i in range(int(len(gportindex))):
if sum(gporttraffic[i]) / len(gporttraffic[i]) > max_avg:
max_avg = sum(gporttraffic[i]) / len(gporttraffic[i])
sum_avg += sum(gporttraffic[i]) / len(gporttraffic[i])
avg_avg = sum_avg / len(gportindex)
for i in range(len(gportindex)):
if sum(gporttraffic[i]) / len(gporttraffic[i]) > avg_avg:
for j in range(len(gportindex)):
if sum(gporttraffic[j]) / len(gporttraffic[j]) > avg_avg:
if gswitchid[i] != gswitchid[j]:
dis = ProcessData.euclidean_distance(
gporttraffic[i], gporttraffic[j])
array_i.append(i)
array_j.append(j)
distance.append(dis)
sum_distance += dis
num_distance += 1
if num_distance != 0:
average_distance = sum_distance / num_distance
sort_dis = sorted(range(len(distance)), key=lambda k: distance[k])
for i in range(len(sort_dis)):
k = sort_dis[i]
print("distance[k] is " + str(distance[k]))
#if distance[k] > average_distance:
print("Hevey Hitter is " + str(array_i[k]) + " " +
str(array_j[k]))
print("Distance is " + str(distance[k]))
print(gporttables[array_i[k]])
print(gporttables[array_j[k]])
else:
print("NO HH DETECTED!!!")
def Build_TF_IDF(self):
for i in range(self.Total_Mails):
Msg = ProcessData.Process_Msg(self.mails[i])
count = list()
for word in Msg:
if self.labels[i]:
self.TF_Spam[word] = self.TF_Spam.get(word, 0) + 1
else:
self.TF_Ham[word] = self.TF_Ham.get(word, 0) + 1
if word not in count:
count += [word]
for word in count:
if self.labels[i]:
self.IDF_Spam[word] = self.IDF_Spam.get(word, 0) + 1
else:
self.IDF_Ham[word] = self.IDF_Ham.get(word, 0) + 1
def editConvertTypes(cls, clientID, sourceID, key, fromType, toType):
source = cls.getSource(clientID=clientID, sourceID=sourceID)
changes = []
for doc in source.find({"data." + key: {"$type": fromType}}):
value = doc.get('data', {}).get(key, None)
if value is None: continue
value = ProcessData.convertType(value=value, toType=toType)
if value is None: continue
doc['data'][key] = value
changes.append(ReplaceOne({'_id': doc['_id']}, doc))
if len(changes) > 0:
source.bulk_write(changes, ordered=False)
cls.cacheSchema(clientID=clientID,
sourceID=sourceID,
result=[{
'key': key,
'type': toType
}])
def dummy(clientID, sourceID):
userIDs = [uuid.uuid4().hex for x in range(100)]
types = ['Successful', 'Failed']
eventTypes = ['purchase', 'refund']
dates = pd.date_range(start=datetime.datetime(2014, 1, 1),
end=datetime.datetime.utcnow(),
freq='H',
tz='UTC')
data = [{
'amount':
random.randint(10, 10000),
"timestamp":
(ProcessData.getDate(value=random.choice(dates).isoformat())),
'userID':
random.choice(userIDs),
'event':
random.choice(eventTypes),
"status":
random.choice(types),
} for x in range(10000)]
Add.apiAdd(clientID=clientID,
sourceID=sourceID,
data=data,
schemaOverRide={})
def do_GET(self):
print self.path
STATICS = ('/Data','/Interface')
if self.path.startswith('/Shared/'):
# The beginnings of the link sharing utility.
pathparts = self.path.split('/')
id = pathparts[2]
if pathparts[3] != 'tasks.json':
pathwithoutshared = self.path[8:]
index = pathwithoutshared.index('/')
newpath = pathwithoutshared[index:]
self.path = newpath
else:
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('SharedData/'+id+'/tasks.json', 'r'), self.wfile)
return
if self.path == "/" or self.path == '/index.html':
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/index.html', 'r'),self.wfile)
if self.path == "/tool.html":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/tool.html', 'r'),self.wfile)
if self.path == "/contact.html":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/contact.html', 'r'),self.wfile)
elif self.path == "/upload.html":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/upload.html', 'r'),self.wfile)
elif self.path == "/display.html":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/display.html', 'r'),self.wfile)
elif self.path == "/js/main.js":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/js/main.js', 'r'),self.wfile)
elif self.path == "/js/mainview.js":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/js/mainview.js', 'r'),self.wfile)
elif self.path == "/js/histogramview.js":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/js/histogramview.js', 'r'),self.wfile)
elif self.path == "/js/summaryview.js":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/js/summaryview.js', 'r'),self.wfile)
elif self.path == "/js/graphview.js":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/js/graphview.js', 'r'),self.wfile)
elif self.path == "/js/script.js":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/js/script.js', 'r'),self.wfile)
elif self.path == "/style.css":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/style.css', 'r'),self.wfile)
elif self.path == "/webstyle.css":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/webstyle.css', 'r'),self.wfile)
elif self.path == "/favicon.ico":
self.send_response(200, "OKAY")
self.end_headers()
copyfileobj(open('Interface/favicon.ico', 'r'),self.wfile)
elif self.path == "/tasks.json":
self.send_response(200, "OKAY")
self.end_headers()
self.json_out(ProcessData.fromFile("Data/PROF.log"))
elif reduce(lambda a, b: a or b, (self.path.startswith(k) for k in STATICS)):
SimpleHTTPRequestHandler.do_GET(self)
def test(inputFile,subid,Tag,col):
df1 = ProcessData.loadBehDataFile(inputFile)
df1 = ProcessData.processBehavioralFile(df1)
return df1
class CNN_skipCo_OA_trainer(object):
def __init__(self):
self.dataset = ProcessData(train_ratio=0.8,process_raw_data=False,do_augment=False, image_type='OA',
get_scale_center=False, single_sample=True,
do_blur=False, do_crop=False, do_deform=False, do_flip=True)
self.model = cnn_skipC_OA_model.cnn_skipC_OA_model(
criterion=nn.MSELoss(),
optimizer= torch.optim.Adam,
learning_rate=0.001,
weight_decay=0
)
#self.logger = Logger(self)
def fit(self):
# get scale and center parameters
scale_params_low, scale_params_high = utils.load_params(image_type=self.dataset.image_type,
param_type="scale_params")
mean_image_low, mean_image_high = utils.load_params(image_type=self.dataset.image_type,
param_type="mean_images")
epochs=1
for e in range(0, epochs):
# separate names into random batches and shuffle every epoch
self.dataset.batch_names(batch_size=6)
# in self.batch_number is the number of batches in the training set
for i in range(self.dataset.batch_number):
X, Y = self.dataset.create_train_batches(self.dataset.train_batch_chunks[i])
# scale and center the batch
scale_center_X = utils.scale_and_center(X, scale_params_low, mean_image_low,
image_type=self.dataset.image_type)
scale_center_Y = utils.scale_and_center(Y, scale_params_high, mean_image_high,
image_type=self.dataset.image_type)
scale_center_X = np.array(scale_center_X)
scale_center_Y = np.array(scale_center_Y)
#print(scale_center_Y.shape)
# (C, N, H, W) to (N, C, H, W)
scale_center_X = scale_center_X.reshape(scale_center_X.shape[0], scale_center_X.shape[3],
scale_center_X.shape[1], scale_center_X.shape[2])
scale_center_Y = scale_center_Y.reshape(scale_center_Y.shape[0], scale_center_Y.shape[3],
scale_center_Y.shape[1], scale_center_Y.shape[2])
#print(scale_center_X.shape)
#print(scale_center_Y.shape)
input_tensor, target_tensor = torch.from_numpy(scale_center_X), torch.from_numpy(scale_center_Y)
if torch.cuda.is_available():
#print('CUDA available')
#print('current device ' + str(cur_dev))
#print('device count ' + str(torch.cuda.device_count()))
#print('device name ' + torch.cuda.get_device_name(cur_dev))
cur_dev = torch.cuda.current_device()
input_tensor.cuda()
target_tensor.cuda()
self.model.train_model(input_tensor, target_tensor, e)
## how to undo the scaling:
#unscaled_X = utils.scale_and_center_reverse(scale_center_X, scale_params_low, mean_image_low, image_type = self.dataset.image_type)
#unscaled_Y = utils.scale_and_center_reverse(scale_center_Y, scale_params_high, mean_image_high, image_type=self.dataset.image_type)
def predict(self):
#self.model.predict()
# see self.dataset.X_val and self.dataset.Y_val
pass
def log_model(self):
#self.logger.log(self.model)
pass
#
# Instauro Connessione con l'agent snmp
#
session = Session(hostname=address, community=community, version=int(version))
#
# Creo il database
#
Database.create()
currentTime = time()
while True:
sleepTime = time() - currentTime
if sleepTime < interval:
sleepTime = interval - sleepTime
sleep(sleepTime)
currentTime = time()
values = {}
values = defaultdict(lambda: "", values)
for id in oids:
el = session.get(id)
values[el.oid] += "%" + el.value
values = ProcessData.process(values)
Database.update(values)
import os
import ProcessAFile
import datetime
import ProcessData
BaseDir='/home/jason/Dropbox/SteffenerColumbia/Scripts/ExperimentalStimuli/PartialTrialDIR/Scripts/PsychoPyTask_Win32/data'
subid = '1233'
Tag = 'tester'
#FileName = 'PartialTrial_62160613001_1_Run2_2016_06_13_1520.csv'
FileName = 'PartialTrial_62160613001_1_TrainOrderFB_2016_06_13_142.csv'
#FileName = 'PartialTrial_62160613001_1_TrainNoFB_2016_06_13_1422.csv'
inFile = os.path.join(BaseDir,FileName)
ProcessAFile.CreateAccSummaryString(inFile)
OutFile = file
df1 = ProcessData.loadBehDataFile(inFile)
# Process the file
df1 = ProcessData.processBehavioralFile(df1)
# Extract accuracy
ACCdata = df1.data.Acc.groupby(df1.data.LoadLevels).mean()
# Split the file name based
from ProcessData import PTdata
import ProcessData
import WriteToGoogleSpreadSheet
from FileSelectClass import Example
def ProcessAFile(inputFile,subid,Tag):
df1 = ProcessData.loadBehDataFile(inputFile)
df1 = ProcessData.processBehavioralFile(df1)
wkB = WriteToGoogleSpreadSheet.openWorkBook()
wkS = WriteToGoogleSpreadSheet.createWorksheet(wkB, subid)
writeSummaryBehavioralDataToFile(df1, wkS, Tag)
ex = Example()
# This will ask you to select on or more files
ex.showDialog()
df1 = ProcessData.loadBehDataFile(ex.fileName[0][0])
df1 = ProcessData.processBehavioralFile(df1)
ProcessData.createSPMDesignMatrix(df1, 'P00002001_S0001_Run1_DM')
df2 = ProcessData.loadBehDataFile(ex.fileName[0][1])
df2 = ProcessData.processBehavioralFile(df2)
ProcessData.createSPMDesignMatrix(df1, 'P00002001_S0001_Run2_DM')
wkB = WriteToGoogleSpreadSheet.openWorkBook()
wkS = WriteToGoogleSpreadSheet.createWorksheet(wkB, '2001')
writeSummaryBehavioralDataToFile(df1, wkS, 'Run1')
###########################################################################
import ProcessAFile
import ProcessData
import WriteToGoogleSpreadSheet
ex = Example()
inFile = ex.showDialog()
subid = '2003'
print inFile
Tag = 'TrNoFB'
col = 8
ProcessAFile.ProcessAFile(inFile[0][0],subid,Tag,col)
df1 = ProcessData.loadBehDataFile(inFile[0][0])
df1.onlyKeepDataRows()
# find load levels
df1.whatLoadIsThisRow()
# check for accuracy
df1.areResponsesCorrect()
df1 = ProcessData.processBehavioralFile(df1)
Tag = 'Run1'
col = 4
ProcessAFile.ProcessAFile(inFile[0][0],subid,Tag,col)
# Create SPM design matrices
def ProcessAFile(inputFile,subid,Tag):
df1 = ProcessData.loadBehDataFile(inputFile)
df1 = ProcessData.processBehavioralFile(df1)
wkB = WriteToGoogleSpreadSheet.openWorkBook()
wkS = WriteToGoogleSpreadSheet.createWorksheet(wkB, subid)
writeSummaryBehavioralDataToFile(df1, wkS, Tag)
