def test_create_complete(self):
prediction = Prediction(self.minimum_data)
prediction.add_custom_data(self.additional_data)
eq_(15, prediction.interval)
eq_('alias/project', prediction.project)
eq_('8.8.8.8', prediction.ip_address)
eq_(1415823240, prediction.period)
eq_('gaussian_multivariate', prediction.model)
eq_(1e-35, prediction.pvalue)
eq_(2e-31, prediction.epsilon)
eq_('outlier', prediction.label)
def test_create_complete(self):
prediction = Prediction(self.minimum_data)
prediction.add_custom_data(self.additional_data)
eq_(15, prediction.interval)
eq_('alias/project', prediction.project)
eq_('8.8.8.8', prediction.ip_address)
eq_(1415823240, prediction.period)
eq_('gaussian_multivariate', prediction.model)
eq_(1e-35, prediction.pvalue)
eq_(2e-31, prediction.epsilon)
eq_('outlier', prediction.label)
def test_create_minimum_data(self):
prediction = Prediction(self.minimum_data)
eq_(15, prediction.interval)
eq_('alias/project', prediction.project)
eq_('8.8.8.8', prediction.ip_address)
eq_(1415823240, prediction.period)
eq_('gaussian_multivariate', prediction.model)
def execute(self, *args, **kwargs):
"""Run outlier detection table
"""
project_interval = "{0}_{1}".format(self.project.full_name,
self.interval)
t1 = time.time()
labels, X = self.create_ndarray(
self.session.query(Example).filter_by(p_interval=project_interval)
# self.session.query(Example).filter_by(ip_address='84.120.211.34')
)
t2 = time.time()
np.random.shuffle(X)
# multivariate gauss
t3 = time.time()
columns = ['status_200', 'as_bot', 'as_badbot']
columns_idx = []
for c in columns:
columns_idx.append(labels.index(c))
p = self.density(X[:, columns_idx])
t4 = time.time()
outliers, epsilon = self.find_outliers(X, p,
self.project.list_epsilons)
t5 = time.time()
self.logger.info("\tDimensions are {0}".format(X.shape))
self.logger.info("\t{0}MB used in data".format(X.nbytes / 1024 / 1024))
self.logger.info("\tshuffle done ({0:.3f}s):".format(t3 - t2))
self.logger.info("\tmultivariate done ({0:.3f}s):".format(t4 - t3))
self.logger.info("\tfind outliers done ({0:.3f}s):".format(t5 - t4))
# TAKE CARE, loop function not vectorized internally
Xp = np.c_[X, p]
for x in Xp:
if x[-1] < self.project.cur_epsilon:
prediction = Prediction({
'interval':
self.interval,
'ip_address':
self.int2ip(int(x[labels.index('ip_address')])),
'period':
x[labels.index('period')],
'model':
'gauss_multivariate',
'project':
self.project.full_name,
'epsilon':
float(self.project.cur_epsilon),
'pvalue':
float(x[-1])
})
self.session.add(prediction)
def __init__(self, enabled: bool, target_od: float,
continuous_pump_time: float,
reactors: List[ReactorTemperatureControl],
sensors: List[OpticalDensitySensor],
pumps: List[PeristalticPump]):
"""
:param enabled: enable / disable control loop
:param reactors: reactor[0] = bacteria only (control constant growth rate), reactor[2] = bacteria & phages
:param sensors: sensor[0] = sensor for reactor[0], sensor[1] = sensor for reactor[2]
:param pumps: pump[0] = lb influx to reactor[0], pump[1] = pump from reactor[0] to reactor[1], pump[3] = from reactor[1] to the waste
:param interval: control loop interval
"""
self.enabled = enabled
self.target_od = target_od
self.reactors = reactors
self.sensors = sensors
self.pumps = pumps
self.tol = float(
self.config["prediction_constant_concentration"]["tol"])
self.continuous_pump_time = continuous_pump_time
self.predictor = Prediction(verbose=False)
with open("params.json") as file:
self.config = json.load(file)
def test_to_primitive(self):
prediction = Prediction(self.minimum_data)
prediction.add_custom_data(self.additional_data)
primitive = prediction.to_primitive()
def test_validate_complete_data(self):
prediction = Prediction(self.minimum_data)
prediction.add_custom_data(self.additional_data)
prediction.validate()
def test_validate_minimum_data(self):
prediction = Prediction(self.minimum_data)
prediction.validate()
def test_tablename(self):
eq_('prediction', Prediction.__tablename__)
prediction = Prediction(self.minimum_data)
eq_('prediction', prediction.__tablename__)
def test_key(self):
prediction = Prediction(self.minimum_data)
eq_('alias/project-15-1415823240-8.8.8.8-gaussian_multivariate',
prediction.__key__)
def test_to_primitive(self):
prediction = Prediction(self.minimum_data)
prediction.add_custom_data(self.additional_data)
primitive = prediction.to_primitive()
def test_validate_complete_data(self):
prediction = Prediction(self.minimum_data)
prediction.add_custom_data(self.additional_data)
prediction.validate()
def test_validate_minimum_data(self):
prediction = Prediction(self.minimum_data)
prediction.validate()
# convert the labels from integers to vectors
trainY = LabelBinarizer().fit_transform(trainY)
vatY = LabelBinarizer().fit_transform(vatY)
# train
Train.train(ModelVGGNet, trainX, vatX, trainY, vatY, "mini_vggnet")
#Train.train(ModelInception, trainX, vatX, trainY, vatY, "mini_inception")
'''Train.train(ModelResnet, trainX, vatX, trainY, vatY, "resnet", {
"learning_rate": 0.1,
"max_epochs": 100,
"batch_size": 32
})'''
# prediction
model_path = "D:/s5/cv_python/es_face/output/ckpt_resnet/ok/resnet_88_97.78.ckpt"
prediction = Prediction(ModelResnet, w, w, 1, 5, model_path)
r = prediction.check_model(vatX, vatY)
if r:
num_acc = 0
num_ok = 0
num_ok_acc = 0
num_sample = vatY.shape[0]
np.set_printoptions(suppress=True)
for i, sample in enumerate(vatX):
sample_ = np.expand_dims(sample, axis=0)
pred = prediction.prediction(sample_)
pred_max = np.max(pred)
# print("----- ----- ----- ----- ----- {}".format(np.around(pred_max, decimals=6)))
if pred_max > 0.9:
num_ok += 1
