def __init__(self):
super().__init__()
self.is_running = True
self.monitor_pool = MonitorPool()
self.tolerance = 0.3
self.database = Database.Database()
self.students = []
self.machine_learning = MachineLearning()
self.face_capture = FaceCapture(self.database, self.machine_learning,
self.students)
self.delay = 0.01
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
def train_model():
success_status = False
if request.method == 'POST':
success_status = True
global trained_models
ml_model = MachineLearning('diabetes.csv')
ml_model.svm()
ml_model.logistic_regression()
ml_model.decision_tree()
ml_model.persist_model()
trained_models = ml_model.filenames
return render_template("trainmodels.html",
success_status=success_status)
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
import traceback
from MachineLearning import MachineLearning
import urllib
import lemmatizer
# loadfile = input("Input file name you would like to load: ")
loadfile = "model.sav"
# ml_trainer = MachineLearning("lemmatizedsites.txt", num_features=20000, num_estimators=100, testsplit=0.25,
# seed=1572310670, ngram_range=(1, 1))
ml_trainer = MachineLearning("../docs/lemmatizedsites.txt",
num_features=13400,
num_estimators=100,
testsplit=0.25,
seed=1324640185,
ngram_range=(1, 1),
n_jobs=2)
ml_trainer.run(loadfile)
ml_trainer.print_stats()
ml_trainer.print_confusion_matrix()
# input("Press Enter to continue")
webpage = ""
while webpage != "EXIT":
webpage = input("Enter a webpage: ")
try:
if webpage.lower() == "save":
filename = input("Enter file name:")
ml_trainer.save(filename)
class Log(object):
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
def write_data_to_file(self, file_name, line):
file_name.write(line)
file_name.write("\n")
file_name.flush()
def create_log_files(self, prefix):
self.prefix = prefix
if not os.path.exists("Metric\\" + self.prefix):
os.makedirs("Metric\\"+self.prefix)
return open(os.path.join("Metric\\"+self.prefix, "log.txt"), "a+")
def check_file(self, file_name, prefix):
self.prefix = prefix
self.filename = file_name+".txt"
filepath = os.path.join("Metric\\"+self.prefix, self.filename)
if ((os.path.exists(filepath)) == False):
open(os.path.join("Metric\\"+self.prefix, self.filename),"a")
else:
pass
def setmessagelevel(self,
line_data,
local_time,
storage_nodes,
normal_value_count,
metrics, anomaly_value_count,
thresholds_values, thresholds_lines, fo, lo,
metric_name,
prefix,
):
self.metric_name = metric_name
self.fo = fo
self.log_files = lo
self.anomaly_value_count = anomaly_value_count
self.line_data = line_data
self.local_time = local_time
self.storage_nodes = storage_nodes
self.normal_value_count = normal_value_count
self.thresholds_values = thresholds_values
self.thresholds_lines = thresholds_lines
self.metrics = metrics
self.prefix = prefix
if self.line_data.find("errors.timing") > 0:
str_data = self.local_time, 'E: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
elif (((self.line_data.find("errors")) > 0) | ((self.line_data.find("quarantines")) > 0)):
str_data = self.local_time, 'C: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files,
self.join_line)
elif self.line_data.find("timing") > 0:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.parsing_line(self.join_line)
if len(self.storage_nodes) == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append([0*x for x in range(
0, len(self.metrics))])
self.anomaly_value_count.append([0*x for x in range(0, len(self.metrics))])
self.thresholds_values.append([0 for x in range(0, len(self.metrics))])
self.thresholds_lines.append([self.no_of_lines for x in range(0,len(self.metrics))])
self.mach_learn.anomaly_detection(0, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time, self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values
self.storage_nodes_count = 1
else:
for i in range(0, len(self.storage_nodes)):
if self.prefix == self.storage_nodes[i]:
self.mach_learn.anomaly_detection(i, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time,self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values
self.storage_nodes_count = 1
if self.storage_nodes_count == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append([0*x for x in range(0,len(self.metrics))])
self.anomaly_value_count.append([0*x for x in range(0,len(self.metrics))])
self.thresholds_values.append([0 for x in range(0,len(self.metrics))])
self.thresholds_lines.append([self.no_of_lines for x in range(0,len(self.metrics))])
self.mach_learn.anomaly_detection(len(self.storage_nodes)-1, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time,self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values
self.storage_nodes_count = 0
else:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
return self.storage_nodes, self.normal_value_count, self.anomaly_value_count, self.thresholds_values, self.thresholds_lines
def create_new_file(self):
self.timestr = time.strftime("%Y%m%d-%H%M%S")
self.filename = "logfile.txt"
if not os.path.exists("log"):
os.makedirs("log")
return open(os.path.join("log", self.filename), "a")
def main_menu():
# Instantiate the Top_k class
top_k = tk()
spark_run = False
command = 0
while command is not 'Q' and command is not 'q':
command = input("""
-------------------------------------------------------------------------------
| |
| |
| |
| |
| |
| |
| Alzheimer's -NCI |
| Top-K |
| |
| |
| |
| 1 - Print top 10 t-test values |
| 2 - Print cluster mean and std |
| 3 - Run machine learning |
| Q - Exit |
-------------------------------------------------------------------------------
Enter Command >>> """)
#command = input()
if command == '1':
#rosmap_file = input("Enter name of ROSMAP file: ")
#gene_cluster_file = input("Enter name of gene cluster file: ")
rosmap_file = "rosmap.csv"
gene_cluster_file = "gene_cluster.csv"
start = time.time()
top_k.find_top_k(rosmap_file, gene_cluster_file)
top_k.print_top_k()
end = time.time()
print("Time: ", end - start)
spark_run = True
#top_k, cluster_vals = tk.find_top_k(rosmap_file, gene_cluster_file)
#pprint.pprint(top_k)
elif command == '2':
if spark_run:
cluster = input("Cluster number: ")
top_k.print_cluster_values(cluster)
else:
load_top_ten()
elif command == '3':
if spark_run:
top_k.top_k_vals_for_patients()
ad, nci = top_k.get_all_ad_and_nci_vals()
sc = top_k.get_spark_context()
#import pdb; pdb.set_trace()
# Instantiate the MachineLearning class
ml = MachineLearning(ad, nci, sc)
# Create the labeled points for models with both
# life stats and no life stats
ml.set_labeled_points_data()
model_type = input("Include age stats? (y/n): ")
if model_type == 'y' or model_type == 'Y':
group = 'life_stats'
else:
group = 'genes'
num_rf_trees = input(
"Enter number of trees in random forest: ")
# Build model
ml.build_model(group, num_rf_trees)
ml.print_all_data()
else:
load_top_ten()
print("Bye")
class Log(object):
def __init__(self):
print "the log class"
self.metric = Metric()
self.mach_learn = MachineLearning()
self.storage_nodes = []
self.normal_value_count = []
self.metrics = []
self.anomaly_value_count = []
self.thresholds_values = []
self.thresholds_lines = []
self.no_of_lines = 200
def write_data_to_file(self, file_name, line):
file_name.write(line)
file_name.write("\n")
file_name.flush()
def create_log_files(self, prefix):
self.prefix = prefix
if not os.path.exists("Metric\\" + self.prefix):
os.makedirs("Metric\\" + self.prefix)
return open(os.path.join("Metric\\" + self.prefix, "log.txt"), "a+")
def check_file(self, file_name, prefix):
self.prefix = prefix
self.filename = file_name + ".txt"
filepath = os.path.join("Metric\\" + self.prefix, self.filename)
if ((os.path.exists(filepath)) == False):
open(os.path.join("Metric\\" + self.prefix, self.filename), "a")
else:
pass
def setmessagelevel(
self,
line_data,
local_time,
storage_nodes,
normal_value_count,
metrics,
anomaly_value_count,
thresholds_values,
thresholds_lines,
fo,
lo,
metric_name,
prefix,
):
self.metric_name = metric_name
self.fo = fo
self.log_files = lo
self.anomaly_value_count = anomaly_value_count
self.line_data = line_data
self.local_time = local_time
self.storage_nodes = storage_nodes
self.normal_value_count = normal_value_count
self.thresholds_values = thresholds_values
self.thresholds_lines = thresholds_lines
self.metrics = metrics
self.prefix = prefix
if self.line_data.find("errors.timing") > 0:
str_data = self.local_time, 'E: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
elif (((self.line_data.find("errors")) > 0) |
((self.line_data.find("quarantines")) > 0)):
str_data = self.local_time, 'C: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
elif self.line_data.find("timing") > 0:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.parsing_line(self.join_line)
if len(self.storage_nodes) == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.anomaly_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.thresholds_values.append(
[0 for x in range(0, len(self.metrics))])
self.thresholds_lines.append(
[self.no_of_lines for x in range(0, len(self.metrics))])
self.mach_learn.anomaly_detection(
0, self.metrics, self.fo, self.log_files, self.metric_name,
self.prefix, self.local_time, self.line_data,
self.normal_value_count, self.thresholds_lines,
self.thresholds_values
) # calling function code to detect anomaly values
self.storage_nodes_count = 1
else:
for i in range(0, len(self.storage_nodes)):
if self.prefix == self.storage_nodes[i]:
self.mach_learn.anomaly_detection(
i, self.metrics, self.fo, self.log_files,
self.metric_name, self.prefix, self.local_time,
self.line_data, self.normal_value_count,
self.thresholds_lines, self.thresholds_values
) # calling function code to detect anomaly values
self.storage_nodes_count = 1
if self.storage_nodes_count == 0:
self.storage_nodes.append(self.prefix)
self.normal_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.anomaly_value_count.append(
[0 * x for x in range(0, len(self.metrics))])
self.thresholds_values.append(
[0 for x in range(0, len(self.metrics))])
self.thresholds_lines.append(
[self.no_of_lines for x in range(0, len(self.metrics))])
self.mach_learn.anomaly_detection(
len(self.storage_nodes) - 1, self.metrics, self.fo,
self.log_files, self.metric_name, self.prefix,
self.local_time, self.line_data, self.normal_value_count,
self.thresholds_lines, self.thresholds_values
) # calling function code to detect anomaly values
self.storage_nodes_count = 0
else:
str_data = self.local_time, 'I: ', self.line_data
self.join_line = ''.join(str_data)
self.metric.write_data_to_file(self.fo, self.join_line)
self.metric.write_data_to_file(self.log_files, self.join_line)
return self.storage_nodes, self.normal_value_count, self.anomaly_value_count, self.thresholds_values, self.thresholds_lines
def create_new_file(self):
self.timestr = time.strftime("%Y%m%d-%H%M%S")
self.filename = "logfile.txt"
if not os.path.exists("log"):
os.makedirs("log")
return open(os.path.join("log", self.filename), "a")
class main:
print("Starting")
print("__name__ value: ", __name__)
if __name__ == '__main__':
print('[0]Testes')
print('[1]Projeto')
print('[2]Sinesp')
operacao = input()
if operacao is '0':
print("[0] Teste com classe")
print("[1] Invocando metodo")
print("[2] machine learning")
print("[3] matplotlib")
print("[4] plotly")
print("[5] pandas")
print("[6] seaborn")
print("[7] Open CV")
print("[8] MongoDB")
print("Comando ")
operacao = input()
if operacao is '0':
Controller.mainMet1(object)
if operacao is '1':
p1 = Person("John", 36)
print(p1.name)
print(p1.age)
if operacao is '2':
MachineLearning.DataSet(object)
if operacao is '3':
TestMatPlotLib.MatPlotLibPrinc(object)
if operacao is '4':
TestPlotly.PlotlyPrinc(object)
if operacao is '5':
TestPandas.PandasPrinc(object)
if operacao is '6':
TestSeaborn.SeabornPrinc(object)
if operacao is '7':
TestOpenCV.OpenCVPrinc(object)
if operacao is '8':
TestPymongo.PymongoTest(object)
if operacao is '1':
print("[0] OpenCV")
print("[1]Leitor camera")
operacao = input()
if operacao is '0':
OpenCV.OpenCV.EdgeDetectorVideo(object)
if operacao is '1':
MetodoStart.Start(object)
if operacao is '2':
print("[0] Sinespi")
operacao = input()
if operacao is '0':
TestSinesp.SinespPrinc(object)
def __init__(self,train_hidden_path=None,train_no_hidden_path=None,test_path=None,dest_data_path=None,dest_classes_path=None,process=None): self._config = Config(train_hidden_path,train_no_hidden_path,test_path,dest_data_path,dest_classes_path,process) self._image_processor = None self._image_statistics = None self._classificator = MachineLearning()
class StegoDetect: def __init__(self,train_hidden_path=None,train_no_hidden_path=None,test_path=None,dest_data_path=None,dest_classes_path=None,process=None): self._config = Config(train_hidden_path,train_no_hidden_path,test_path,dest_data_path,dest_classes_path,process) self._image_processor = None self._image_statistics = None self._classificator = MachineLearning() def _make_process(self): if self._config._get_process()==0: self.__preprocess_corpus() else: self.__detect_message() def __preprocess_corpus(self): f_data = open(self._config._get_dest_data_path(),"w") f_classes = open(self._config._get_dest_classes_path(),"w") for fd in listdir(self._config._get_train_hidden_path()): print fd dump(self.__preprocess_file(self._config._get_train_hidden_path()+fd),f_data) dump(1,f_classes) for fd in listdir(self._config._get_train_no_hidden_path()): print fd dump(self.__preprocess_file(self._config._get_train_no_hidden_path()+fd),f_data) dump(0,f_classes) f_data.close() f_classes.close() def __preprocess_file(self,fd): if self._image_processor == None: self._image_processor = ProcessImage(fd) else: self._image_processor._set_image(fd) self._image_processor._calc_histogram_rgb(self._image_processor._get_rgb_image()) histogram_rgb = self._image_processor._get_histogram_rgb() if self._image_statistics == None: self._image_statistics = ImageStatistics(histogram_rgb) else: self._image_statistics._set_histogram(histogram_rgb) mean_rgb = self._image_statistics._get_mean() variance_rgb = self._image_statistics._get_variance(mean_rgb) dft_histogram = self._image_processor._get_fourier_transformation() self._image_statistics._set_histogram(dft_histogram) mean_dft = self._image_statistics._get_mean() variance_dft = self._image_statistics._get_variance(mean_dft) desviation_dft = self._image_statistics._get_standard_desviation(variance_dft) skewness_dft = self._image_statistics._get_skewness(mean_dft,desviation_dft) curtosis_dft = self._image_statistics._get_curtosis(mean_dft,desviation_dft) energy_dft = self._image_processor._get_fourier_energy() mean_difference = self._image_processor._get_histogram_dft_mean_difference() feature_vector = self._image_processor._get_feature_vector(mean_rgb,variance_rgb,mean_dft,variance_dft,skewness_dft,curtosis_dft,energy_dft,mean_difference) feature_vector = self._image_processor._feature_scaling(feature_vector) return feature_vector def __unpickle(self,fd): res = [] while 1: try: res.append(load(fd)) except EOFError: break return res def __detect_message(self): f_data = open(self._config._get_dest_data_path(),"rb") f_classes = open(self._config._get_dest_classes_path(),"rb") features = self.__unpickle(f_data) classes = self.__unpickle(f_classes) f_data.close() f_classes.close() ## Train ml # self._classificator._train(features,classes) features = [] for fd in listdir(self._config._get_test_path()): res = self._classificator._test(self.__preprocess_file(self._config._get_test_path()+fd)) if res==[1]: print "File",fd,"has a hidden message" else: print "File",fd,"has not got any hidden message"
class Backend(threading.Thread):
def __init__(self):
super().__init__()
self.is_running = True
self.monitor_pool = MonitorPool()
self.tolerance = 0.3
self.database = Database.Database()
self.students = []
self.machine_learning = MachineLearning()
self.face_capture = FaceCapture(self.database, self.machine_learning,
self.students)
self.delay = 0.01
def read_config(self):
for e in self.database.monitor_read_all():
if e.type != "disabled":
self.monitor_pool.addMonitor(e)
print(e, end=',')
def read_encodings(self):
path = 'encoding'
person_list = os.listdir(path) # 列出文件夹下所有的目录与文件
for i, person_id in enumerate(person_list):
person_encoding_path = os.path.join(path, person_id)
stu = Student()
self.students.append(stu)
stu.set_id(person_id)
for i2, pics in enumerate(os.listdir(person_encoding_path)):
encoding_path = os.path.join(person_encoding_path, pics)
if os.path.isfile(encoding_path):
encoding = numpy.load(encoding_path)
stu.encodings.append(encoding)
def open_capture(self):
video_capture = cv2.VideoCapture(0)
cv2.namedWindow("Video")
frame_num = 0
face_locations = []
face_names = []
while True:
ret, frame = video_capture.read()
if frame_num % 5 == 0:
face_locations = face_recognition.face_locations(frame)
face_encodings = face_recognition.face_encodings(
frame, face_locations)
self.face_capture.read_img(frame, frame_num)
face_names = []
for face_encoding in face_encodings:
# See if the face is a match for the known face(s)
rst = self.machine_learning.face_compare(
self.students, face_encoding, tolerance=self.tolerance)
if rst:
face_names.append(str(rst.get_id()))
for (top, right, bottom,
left), name in zip(face_locations, face_names):
cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255),
2)
cv2.rectangle(frame, (left, bottom - 35), (right, bottom),
(0, 0, 255), cv2.FILLED)
font = cv2.FONT_HERSHEY_DUPLEX
cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0,
(255, 255, 255), 1)
# Display the resulting image
cv2.imshow('Video', frame)
frame_num += 1
# Hit 'q' on the keyboard to quit!
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# Release handle to the webcam
video_capture.release()
cv2.destroyAllWindows()
def run(self):
self.monitor_pool.startAll()
cv2.namedWindow("show")
frame_num = 0
while self.is_running:
for e in self.monitor_pool.getMonitors():
frame = e.get_cache()
self.face_capture.read_img(frame, frame_num, e)
cv2.imshow("show", frame)
time.sleep(self.delay)
if frame_num % 1000 == 0:
print("running")
frame_num += 1
