def arduinoPin(pinNumber, state):
for x in range(0, len(pinNumber)-1):
Pin = GET.arduinoSyntax(pinNumber[x], state[x])
s.connect((header.VS_IP, header.VS_PORT))
s.sendall(Pin)
pinState = int(s.recv(1024))
s.close()
## Check if pinState contains error information
Error[x] = GET.isError(pinState)
return (Error)
def annotations():
callback = request.GET.get('callback')
documentId = request.GET.get('document_id')
#classifier_name = request.GET.get('classifier_name')
classifier_dump = request.GET.get('classifier')
questionId = request.GET.get('question_id')
collection_id = request.GET.get('collection_id')
range = request.GET.get('range')
meh = StringIO(classifier_dump)
buff = meh.getvalue()
clf = pickle.loads(buff)
features_train, labels_train = wtf.getArrays(collection_id)
print("params :")
#print("classifier_name" + classifier_name)
print("classifier_dump" + classifier_dump)
print("question_id" + question_id)
print("collection_id" + collection_id)
print("range" + range)
print("document_id" + document_id)
preditions = clf.predict(features_train)
data = preds_to_send(questionId, documentId, preditions[0], range)
put.send("http://localhost:8080/dataprocessing/rest-api/annotations", data)
return '{0}({1})'.format(callback, {'a': 1, 'b': 2})
def event(message):
Message = "- ", message
(st, ts) = GET.time_stamp()
file = open('%s/EventLog.txt' % (header.filefolder), "a")
print >> file, st, Message
file.close
return
def annotations():
callback = request.GET.get('callback')
documentId = request.GET.get('document_id')
#classifier_name = request.GET.get('classifier_name')
classifier_dump = request.GET.get('classifier')
questionId = request.GET.get('question_id')
collection_id = request.GET.get('collection_id')
range = request.GET.get('range')
meh = StringIO(classifier_dump)
buff = meh.getvalue()
clf = pickle.loads(buff)
features_train, labels_train = wtf.getArrays(collection_id)
print("params :")
#print("classifier_name" + classifier_name)
print("classifier_dump" + classifier_dump)
print("question_id" + question_id)
print("collection_id" + collection_id)
print("range" + range)
print("document_id" + document_id)
preditions = clf.predict(features_train)
data = preds_to_send(questionId,documentId,preditions[0],range)
put.send("http://localhost:8080/dataprocessing/rest-api/annotations", data)
return '{0}({1})'.format(callback, {'a':1, 'b':2})
def emergency(message):
Message = "- ", message
(st, ts) = GET.time_stamp()
file = open('%s/EmergencyLog.txt' % (header.filefolder), "a")
print >> file, st, Message
file.close
sock.sendto(Message, (header.UDP_IP, header.UDP_PORT))
return
def test():
callback = request.GET.get('callback')
classifier_name = request.GET.get('classifier_name')
classifier_dump = request.GET.get('classifier')
collection_id = request.GET.get('collection_id')
print("Params :")
print(classifier_name)
features_train, labels_train = wtf.getArrays(collection_id)
clf = pickle.loads(classifier_dump)
preditions, accuracy, recall, precision = classifier.test(clf = clf, features = features_train, labels = labels_train)
print("Test :")
print("accuracy" + accuracy)
print("precision" + precision)
print("recall" + recall)
data1 = test_data_to_send(classifier_id, 0, " ", precision, accuracy, recall)
post.send("http://localhost:8080/dataprocessing/rest-api","/result_test_classifier","",data1)
return '{0}({1})'.format(callback, {'a':1, 'b':2})
def test():
callback = request.GET.get('callback')
classifier_name = request.GET.get('classifier_name')
classifier_dump = request.GET.get('classifier')
collection_id = request.GET.get('collection_id')
print("Params :")
print(classifier_name)
features_train, labels_train = wtf.getArrays(collection_id)
clf = pickle.loads(classifier_dump)
preditions, accuracy, recall, precision = classifier.test(
clf=clf, features=features_train, labels=labels_train)
print("Test :")
print("accuracy" + accuracy)
print("precision" + precision)
print("recall" + recall)
data1 = test_data_to_send(classifier_id, 0, " ", precision, accuracy,
recall)
post.send("http://localhost:8080/dataprocessing/rest-api",
"/result_test_classifier", "", data1)
return '{0}({1})'.format(callback, {'a': 1, 'b': 2})
if header.connect == 1:
## If header.connect = 1, then the DHM successfully connected to the host server via UDP so it will now listen for commands
## Begin waiting for commands from BRUIE
while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
if data:
message = "Datagram Received:", data
PRINT.event(message)
if c in data: # DHM_record
## TO BE WRITTEN
RUN.DAQ() ## Placeholder
if header.udpStatus in data: # Host queried DHM status
pump = GET.pump()
relays = GET.relay("all")
temp = GET.temp()
message = 'Pump status is: ', pump, '\n', 'Relay statuses are: ', relays, '\n', 'Temperatures are : ', temp
PRINT.udp(message)
if header.udpPumpOn in data: # Turn pump on
header.touch(header.pumpLock)
RUN.pump()
if header.udpPumpOff in data: # Turn pump off
os.remove(pumpLock)
if f in data: # run microscope on auto
auto_run()
def clientThread(clientSocket, address):
doc_type = ''
while True:
msg = recvall(clientSocket)
msg_data = msg.split('\r\n\r\n'.encode())
headers = msg_data[0]
try:
header = headers.decode()
except UnicodeDecodeError:
pass
try:
pieces = header.split('\n')
except AttributeError:
pass
if len(msg) != 0:
if len(pieces) > 0:
request = pieces[0].split(' ')
#GET request
if request[0] == 'GET':
for line in pieces:
header = line.split(':')
if header[0] == 'Accept-Encoding':
encoding = header[1].strip()
break
else:
encoding = ''
if request[1].find('.png') == -1 and request[1].find(
'.jpg') == -1:
response_html = GET.get(pieces, address)
try:
if ((encoding == 'zlib' or encoding == 'gzip')
and response_html.find('404') < 0
and response_html.find('406') < 0):
clientSocket.sendall(response_html)
else:
clientSocket.sendall(response_html.encode())
except (TypeError, BrokenPipeError):
try:
clientSocket.sendall(response_html)
except TypeError:
clientSocket.sendall(response_html.encode())
else:
try:
response_html, response_image = GET.get(
pieces, address)
try:
clientSocket.sendall(response_html.encode() +
response_image)
except BrokenPipeError:
pass
except ValueError:
response_html = GET.get(pieces, address)
try:
clientSocket.sendall(response_html.encode())
except BrokenPipeError:
pass
#HEAD request
elif request[0] == 'HEAD':
response_html = HEAD.head(pieces, address)
clientSocket.sendall(response_html.encode())
#DELETE request
elif request[0] == 'DELETE':
if request[1].find('.png') == -1 and request[1].find(
'.jpg') == -1:
response_html = DELETE.delete(pieces, address)
try:
clientSocket.sendall(response_html.encode())
except BrokenPipeError:
pass
else:
try:
response_html, response_image = DELETE.delete(
pieces, address)
try:
clientSocket.sendall(response_html.encode() +
response_image)
except BrokenPipeError:
pass
except ValueError:
response_html = DELETE.delete(pieces, address)
try:
clientSocket.sendall(response_html.encode())
except BrokenPipeError:
pass
elif request[0] == 'PUT':
for line in pieces:
header = line.split(':')
if header[0] == 'Content-Type':
doc_type = header[1]
break
if doc_type.find('html') >= 0 or doc_type.find('plain') >= 0:
payload = msg_data[1].decode()
response_html = PUT.put(pieces, address, payload)
try:
clientSocket.sendall(response_html.encode())
except BrokenPipeError:
pass
else:
try:
payload = msg_data[1]
except IndexError:
pass
try:
response_html, response_image = PUT.put(
pieces, address, payload)
try:
clientSocket.sendall(response_html.encode() +
response_image)
except BrokenPipeError:
pass
except ValueError:
response_html = PUT.put(pieces, address, '')
try:
clientSocket.sendall(response_html.encode())
except BrokenPipeError:
pass
elif request[0] == 'POST':
payload = msg_data[1].decode()
response_html = POST.post(pieces, address, payload)
try:
clientSocket.sendall(response_html.encode())
except BrokenPipeError:
pass
try:
clientSocket.shutdown(socket.SHUT_RDWR)
except OSError:
#Transport endpoint not connected
pass
else:
clientSocket.close()
remove(clientSocket)
break
#fweight[1][0] = random.random(); fweight[1][1] = random.random()
'''DATASETS'''
fileNames = ['itemCate', 'user_accepted_cate', 'user_accepted_item', 'user_action', 'user_sns',
'cateItem', 'user_key_word', 'keyword_class',
'rec_log_train','rec_log_test', 'class_cate', 'user_class', 'itemKey']
DATA = IMPORT.IMPORT(fileNames)
'''DATA REC_LOG_TRAIN DISTRIBUTION'''
NORM = int(math.ceil(float(len(DATA['rec_log_train']))/size))
if rank < size - 1:
DATA['rec_log_train'] = dict(DATA['rec_log_train'].items()[rank*NORM:(rank+1)*NORM])
if rank == size - 1:
DATA['rec_log_train'] = dict(DATA['rec_log_train'].items()[rank*NORM:])
'''GET THE LIST OF RECOMMENDATION IN TRAINING SET'''
train = GET.get_train(DATA['rec_log_train'])
'''TRAINING PROCESS'''
while k == 1:
if rank == 0:
print 'USER ' + str(k) + ', larst performance = ' + str(performance)
print 'USER ' + str(k) + ', larst weight = ' + str(weight[0]) + ' ' + str(weight[1])
performance = 0
weight = [0, 0]
weight[0] = random.random(); weight[1] = random.random()
oldweight = weight
Gweight = [0, 0]
#Gfweight = [[0, 0], [0, 0]]
performance = 0
'''TRAINING PROCESS IN COMPUTING SITES'''
for userID in train.keys():
Quit = os.system('cat |sudo tee /sys/class/gpio/gpio%s/value' %
(header.buttonQuit))
DAQ = os.system('cat |sudo tee /sys/class/gpio/gpio%s/value' %
(header.buttonDAQ))
if pump == 0:
## No need to check whether the sample side valves are open, the arduino does that
header.touch(pumpLock)
subprocess.Popen(["sudo", "%s/runPump.py" % (header.codeFolder)])
while pump == 0:
pump = os.system('cat |sudo tee /sys/class/gpio/gpio%s/value' %
(header.buttonPump))
os.remove(pumpLock)
if V1_2 == 0:
V1 = GET.arduinoPin(header.relayV1[0])
V2 = GET.arduinoPin(header.relayV2[0])
## If any of the valves are closed (on), then open them
if (V1 == 1 or V2 == 1):
state = ["off", "off"]
else:
state = ["on", "on"]
pinNumber = [header.relayV1[0], header.relayV2[0]]
Error = SET.arduinoRelay(pinNumber, state)
if (True in Error):
message = "An unknown error occured when setting valve 1 and/or 2"
PRINT.event(message)
if V3 == 0:
V3 = GET.arduinoPin(header.relayV3[0])
def GRADE(userID, itemID, weight, fweight, DATA, userClass):
# weight = w1
# fweight = [fw1,fw2]
#partial = [0,0]
W = 0
INFO = {}
INFO['GRADE'] = 0
INFO['WEIGHT'] = 0
INFO['h - s'] = 0
#INFO['partial'] = partial
'''SPECIFY EVERY DATASET'''
# itemCate = DATA['itemCate']
# user_accepted_cate = DATA['user_accepted_cate']
# user_action = DATA['user_action']
# user_sns = DATA['user_sns']
# cateItem = DATA['cateItem']
# user_key_word = DATA['user_key_word']
# keyword_class = DATA['keyword_class']
# class_cate = {class:{cate:weight,...},...}
'''GET CATEGORY OF THE ITEM'''
# function test OK
itemCate = GET.get_item_cate(itemID, DATA['itemCate'])
if itemCate == None:
print 'None in ITEMCATE at PHASE1' + ', userID: ' + userID + ', itemID: ' + itemID
'''MATCH THE USER CATEGORY IN PHASE 1 AND RETURN WEIGHT 1'''
# get userCate in phase 1
# userCate = [cate,...], funtion test OK
userCate = GET.get_accepted(userID, DATA['user_accepted_cate'])
if userCate == None:
print 'None in USERCATE at PHASE1' + ', userID: ' + userID + ', itemID: ' + itemID
else:
if itemCate in userCate:
W = 1
'''NO MATCH: MATCH THE USER CATEGORY IN PHASE 2 AND RETURN WEIGHT'''
if W == 0:
print 'START PHASE 2,', 'user', userID, 'item', itemID
# user_key_class = {class:{key:weight,...},...}, function test DANTE
user_key_class = PHASE2.get_user_key_class(userID, DATA['user_key_word'], DATA['keyword_class'])
if user_key_class == None:
print 'None in USER_KEY_CLASS in PHASE2' + ', userID: ' + userID + ', itemID: ' + itemID
else:
# userCate = {cate:weight,...}, function test DANTE
userCate = PHASE2.get_cate_weight(user_key_class, DATA['class_cate'])
#print userID, user_key_class
#print userCate
if userCate == None:
print 'None in USERCATE at PHASE2' + ', userID: ' + userID + ', itemID: ' + itemID
elif itemCate in userCate and W == 0:
W = userCate[itemCate]
'''NO MATCH AGAIN: MATCH THE USER CATEGORY IN PHASE 3 AND RETURN WEIGHT'''
if W == 0:
print 'START PHASE 3,', 'user', userID, 'item', itemID
# get followee
# Neighbor = [followee,...], function test OK
Neighbor = GET.get_followee(userID, DATA['user_sns'])
if Neighbor == None:
print 'None in NEIGHBOR at PHASE3' + ', userID: ' + userID + ', itemID: ' + itemID
return INFO
# get followee's category in phase 1(WRONG) and 2
NeighborNum = len(Neighbor) # average
for followeeID in Neighbor:
# get followeeCate in phase1(WRONG) and phase 2, function test OK
followeeCate1 = GET.get_accepted(followeeID, DATA['user_accepted_cate'])
# user_key_class = {class:{key:weight,...},...}, function test DANTE
user_key_class = PHASE2.get_user_key_class(userID, DATA['user_key_word'], DATA['keyword_class'])
# userCate = {cate:weight,...}, function test DANTE
followeeCate2 = PHASE2.get_cate_weight(user_key_class, DATA['class_cate'])
# weight return with caculation of familiarity, using fweight
if followeeCate2 == None and followeeCate1 == None:
print 'None in FOLLOWEECATE at PHASE3' + ', userID: ' + userID + ', itemID: ' + itemID
return INFO
if followeeCate1 != None and itemCate in followeeCate1:
wPhase2 = 1 # tbd: get weight in phase 1 and 2
elif followeeCate2 != None and itemCate in followeeCate2:
wPhase2 = followeeCate2[itemCate]
else:
print 'None ITEMCATE in FOLLOWEECATE at PHASE3' + ', userID: ' + userID + ', itemID: ' + itemID
return INFO
# function test OK
temp = GET.get_action(userID, followeeID, DATA['user_action'])
if temp == None:
W = wPhase2 # tbd
else:
at, retweet, comment = temp
W += (fweight[0] * sigmoid(at) + fweight[1] * sigmoid(retweet) + fweight[2] * sigmoid(comment)) * wPhase2 / NeighborNum
#partial[0] += (sigmoid(at) - sigmoid(comment)) * wPhase2 / NeighborNum
#partial[1] += (sigmoid(retweet) - sigmoid(comment)) * wPhase2 / NeighborNum
'''TO BE SPECIFIED: PHASE2 AND PHASE3 EXCLUSIVE?'''
if itemCate in userCate and W != 1:
if userCate[itemCate] >= W:
W = userCate[itemCate]
#partial = [0,0]
'''IF MATCH: RETURN THE HOT GRADE AND SIMILARITY'''
# get hot rank
# return the hot rank of item in category, function test OK
hot = get_hot_rank(itemID, DATA['itemCate'], DATA['cateItem'])
# get similarity
similarity = get_similarity(userID, itemID, DATA) # function test OK
'''GRADING: NO MATCH RETURN -1, ELSE GRADE = 2*W(w1*h+(1-w1)*s)-1'''
if userClass >= 0:
INFO['GRADE'] = 2 * zoom(W, 100) * (weight * hot + (1 - weight) * similarity) - 1 # no time info
INFO['WEIGHT'] = zoom(W, 100)
INFO['h - s'] = hot - similarity
#print 'user', userID, 'item', itemID, 'hot', hot, 'similarity', similarity, 'WEIGHT', INFO['WEIGHT'], 'W', W, 'GRADE', INFO['GRADE']
#INFO['partial'] = partial
else:
INFO['GRADE'] = (1 + zoom(W, 100)) * hot - 1
return INFO
import cPickle
import GET
import IMPORT
'''data initialization'''
DATA = IMPORT.IMPORT(['itemKey', 'itemCate', 'keyword_class'])
class_cate = {} # {class:{cate:weight,...},...}
'''form class_cate'''
item_num = len(DATA['itemKey'].keys()) # number of the items
flag = {}
for item in DATA['itemKey'].keys():
for key in DATA['itemKey'][item]:
'''get class No. of the key'''
CLASSSET = GET.get_class(key, DATA['keyword_class']) # tbd: keyword_class
if CLASSSET == -1:
'''no match: go to the next item'''
continue
'''get the categories of the item'''
CATE = DATA['itemCate'][item]
'''compute the AVERAGE weight of the CATE(divided by item_num)'''
for CLASS in CLASSSET:
if CLASS not in class_cate:
class_cate[CLASS] = {CATE: DATA['itemKey'][item][key]}
flag[CLASS] = {CATE: 1}
elif CATE not in class_cate[CLASS]:
class_cate[CLASS][CATE] = DATA['itemKey'][item][key]
flag[CLASS][CATE] = 1
else:
class_cate[CLASS][CATE] += DATA['itemKey'][item][key]
def udp(message):
Message = "- ", message
(st, ts) = GET.time_stamp()
sock.sendto(Message, (header.UDP_IP, header.UDP_PORT))
return
