import api
from log import NewLogger
logger = NewLogger("server")
import socket_server
socket_server.Run()
from flask import Flask, request, jsonify
app = Flask(__name__)
@app.route('/classify', methods=['POST'])
def classifyHandler():
logger.debug('In {0} {1}'.format(request.method, request.path))
payload = request.get_json()
results = api.classify(payload)
status_code = 200
if not results['success']:
status_code = 400
logger.debug('Out {0} {1} [{2}]'.format(request.method, request.path,
status_code))
return jsonify(results)
@app.route('/learn', methods=['POST'])
def learnHandler():
logger.debug('In {0} {1}'.format(request.method, request.path))
payload = request.get_json()
results = api.learn(payload)
status_code = 200
import os
import json
import time
import signal
import socket
import argparse
import traceback
import threading
import api
from log import NewLogger
logger = NewLogger("server")
BUFFER_SIZE = 1024
def on_new_client(conn, addr):
try:
payload = ''
while True:
logger.debug("pipe contains {0} bytes".format(len(payload)))
chunk = conn.recv(BUFFER_SIZE).decode()
if not chunk:
break
logger.debug("chunk recieved {0} bytes".format(len(chunk)))
payload += chunk
if not "\n" in payload:
import os
import json
import time
import signal
import socket
import threading
# import _thread
import api
from log import NewLogger
logger = NewLogger("tcp")
# tcp server
# TCP_IP = '127.0.0.1'
TCP_IP = 'localhost'
TCP_PORT = 7005
BUFFER_SIZE = 1024
def on_new_client(conn, addr):
try:
payload = ''
while True:
logger.debug("pipe contains {0} bytes".format(len(payload)))
chunk = conn.recv(BUFFER_SIZE).decode()
if not chunk:
break
def __init__(self, family):
# self.logger = logging.getLogger('learn.AI')
self.logger = NewLogger("learn.AI")
self.naming = {'from': {}, 'to': {}}
self.family = family
class AI(object):
def __init__(self, family):
# self.logger = logging.getLogger('learn.AI')
self.logger = NewLogger("learn.AI")
self.naming = {'from': {}, 'to': {}}
self.family = family
# self.path_to_data = path_to_data
def classify(self, sensor_data):
# print(self.header)
# print(sensor_data)
header = self.header[1:]
is_unknown = True
csv_data = numpy.zeros(len(header))
for sensorType in sensor_data['s']:
if sensor_data['s'][sensorType]:
for sensor in sensor_data['s'][sensorType]:
sensorName = sensorType + "-" + sensor
# print(sensorName, self.header)
if sensorName in header:
is_unknown = False
csv_data[header.index(
sensorName)] = sensor_data['s'][sensorType][sensor]
self.headerClassify = header
self.csv_dataClassify = csv_data.reshape(1, -1)
payload = {'location_names': self.naming['to'], 'predictions': []}
threads = [None] * len(self.algorithms)
self.results = [None] * len(self.algorithms)
for i, alg in enumerate(self.algorithms.keys()):
threads[i] = Thread(target=self.do_classification, args=(i, alg))
threads[i].start()
for i, _ in enumerate(self.algorithms.keys()):
threads[i].join()
# print(csv_data)
# print(self.results)
for result in self.results:
if result != None:
payload['predictions'].append(result)
payload['is_unknown'] = is_unknown
return payload
def do_classification(self, index, name):
"""
header = ['wifi-a', 'wifi-b']
csv_data = [-67 0]
"""
if name == 'Gaussian Process':
return
t = time.time()
try:
prediction = self.algorithms[name].predict_proba(
self.csv_dataClassify)
except Exception as e:
logger.error(self.csv_dataClassify)
logger.error(str(e))
return
predict = {}
for i, pred in enumerate(prediction[0]):
predict[i] = pred
predict_payload = {'name': name, 'locations': [], 'probabilities': []}
badValue = False
for tup in sorted(predict.items(),
key=operator.itemgetter(1),
reverse=True):
predict_payload['locations'].append(str(tup[0]))
predict_payload['probabilities'].append(round(float(tup[1]), 2))
if math.isnan(tup[1]):
badValue = True
break
if badValue:
return
# try:
# t2 = time.time()
# name = "Extended Naive Bayes"
# clf = ExtendedNaiveBayes(self.family,path_to_data=self.path_to_data)
# predictions = clf.predict_proba(header,csv_data)
# predict_payload = {'name': name,'locations': [], 'probabilities': []}
# for tup in predictions:
# predict_payload['locations'].append(str(self.naming['from'][tup[0]]))
# predict_payload['probabilities'].append(round(tup[1],2))
# payload['predictions'].append(predict_payload)
# self.logger.debug("{} {:d} ms".format(name,int(1000 * (t2 - time.time()))))
# except Exception as e:
# self.logger.error(str(e))
# try:
# t2 = time.time()
# name = "Extended Naive Bayes2"
# clf = ExtendedNaiveBayes2(self.family, path_to_data=self.path_to_data)
# predictions = clf.predict_proba(header, csv_data)
# predict_payload = {'name': name, 'locations': [], 'probabilities': []}
# for tup in predictions:
# predict_payload['locations'].append(
# str(self.naming['from'][tup[0]]))
# predict_payload['probabilities'].append(round(tup[1], 2))
# payload['predictions'].append(predict_payload)
# self.logger.debug("{} {:d} ms".format(
# name, int(1000 * (t2 - time.time()))))
# except Exception as e:
# self.logger.error(str(e))
# self.logger.debug("{} {:d} ms".format(
# name, int(1000 * (t - time.time()))))
self.results[index] = predict_payload
@timeout(10)
def train(self, clf, x, y):
return clf.fit(x, y)
def learn(self, fname, file_data=None):
csvfile = None
if file_data:
# base64 and gzipped file
data = base64.b64decode(file_data)
# data = zlib.decompress(data, 16 + zlib.MAX_WBITS)
data = gzip.decompress(data)
csvfile = StringIO(data.decode('utf-8'))
else:
csvfile = open(fname, 'r')
t = time.time()
# load CSV file
self.header = []
rows = []
naming_num = 0
# with open(fname, 'r') as csvfile:
reader = csv.reader(csvfile, delimiter=',')
for i, row in enumerate(reader):
self.logger.debug(row)
if i == 0:
self.header = row
else:
for j, val in enumerate(row):
if j == 0:
# this is a name of the location
if val not in self.naming['from']:
self.naming['from'][val] = naming_num
self.naming['to'][naming_num] = val
naming_num += 1
row[j] = self.naming['from'][val]
continue
if val == '':
row[j] = 0
continue
try:
row[j] = float(val)
except:
self.logger.error("problem parsing value " + str(val))
rows.append(row)
csvfile.close()
# first column in row is the classification, Y
y = numpy.zeros(len(rows))
x = numpy.zeros((len(rows), len(rows[0]) - 1))
# shuffle it up for training
record_range = list(range(len(rows)))
shuffle(record_range)
for i in record_range:
y[i] = rows[i][0]
x[i, :] = numpy.array(rows[i][1:])
names = [
"Nearest Neighbors",
"Linear SVM",
"RBF SVM",
# "Gaussian Process",
"Decision Tree",
"Random Forest",
"Neural Net",
"AdaBoost",
"Naive Bayes",
"QDA"
]
classifiers = [
KNeighborsClassifier(3),
SVC(kernel="linear", C=0.025, probability=True),
SVC(gamma=2, C=1, probability=True),
# GaussianProcessClassifier(1.0 * RBF(1.0), warm_start=True),
DecisionTreeClassifier(max_depth=5),
RandomForestClassifier(max_depth=5,
n_estimators=10,
max_features=1),
MLPClassifier(alpha=1),
AdaBoostClassifier(),
GaussianNB(),
QuadraticDiscriminantAnalysis()
]
self.algorithms = {}
# split_for_learning = int(0.70 * len(y))
for name, clf in zip(names, classifiers):
t2 = time.time()
self.logger.debug("learning {}".format(name))
try:
self.algorithms[name] = self.train(clf, x, y)
# score = self.algorithms[name].score(x,y)
# logger.debug(name, score)
self.logger.debug("learned {}, {:d} ms".format(
name, int(1000 * (t2 - time.time()))))
except Exception as e:
self.logger.error("{} {}".format(name, str(e)))
self.logger.debug("{:d} ms".format(int(1000 * (t - time.time()))))
def save(self, save_file, redis_cache=False):
if redis_cache:
print(redis_cache)
t = time.time()
f = gzip.open(save_file, 'wb')
pickle.dump(self.header, f)
pickle.dump(self.naming, f)
pickle.dump(self.algorithms, f)
pickle.dump(self.family, f)
f.close()
self.logger.debug("{:d} ms".format(int(1000 * (t - time.time()))))
def load(self, save_file, redis_cache=False):
if redis_cache:
print(redis_cache)
t = time.time()
f = gzip.open(save_file, 'rb')
self.header = pickle.load(f)
self.naming = pickle.load(f)
self.algorithms = pickle.load(f)
self.family = pickle.load(f)
f.close()
self.logger.debug("{:d} ms".format(int(1000 * (t - time.time()))))
import pickle
import zlib
import gzip
import operator
import time
# import logging
import math
from io import StringIO
from threading import Thread
import functools
import base64
import redis
import multiprocessing
from log import NewLogger
logger = NewLogger("learn")
import numpy
from sklearn.feature_extraction import DictVectorizer
from sklearn.pipeline import make_pipeline
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn import cluster, mixture
from sklearn.neighbors import kneighbors_graph
import os
import time
import base58
from expiringdict import ExpiringDict
from learn import AI
ai_cache = ExpiringDict(max_len=100000, max_age_seconds=60)
from log import NewLogger
logger = NewLogger("api")
def to_base58(family):
return base58.b58encode(family.encode('utf-8')).decode('utf-8')
def classify(payload):
if payload is None:
return {'success': False, 'message': 'must provide sensor data'}
if 'sensor_data' not in payload:
return {'success': False, 'message': 'must provide sensor data'}
t = time.time()
data_folder = '.'
if 'data_folder' in payload:
data_folder = payload['data_folder']
fname = os.path.join(data_folder,