def __init__(self): super().__init__() self.is_running = True self.monitor_pool = MonitorPool() self.tolerance = 0.3 self.database = Database.Database() self.students = [] self.machine_learning = MachineLearning() self.face_capture = FaceCapture(self.database, self.machine_learning, self.students) self.delay = 0.01
def __init__(self): print "the log class" self.metric = Metric() self.mach_learn = MachineLearning() self.storage_nodes = [] self.normal_value_count = [] self.metrics = [] self.anomaly_value_count = [] self.thresholds_values = [] self.thresholds_lines = [] self.no_of_lines = 200
def train_model(): success_status = False if request.method == 'POST': success_status = True global trained_models ml_model = MachineLearning('diabetes.csv') ml_model.svm() ml_model.logistic_regression() ml_model.decision_tree() ml_model.persist_model() trained_models = ml_model.filenames return render_template("trainmodels.html", success_status=success_status)
import traceback from MachineLearning import MachineLearning import urllib import lemmatizer # loadfile = input("Input file name you would like to load: ") loadfile = "model.sav" # ml_trainer = MachineLearning("lemmatizedsites.txt", num_features=20000, num_estimators=100, testsplit=0.25, # seed=1572310670, ngram_range=(1, 1)) ml_trainer = MachineLearning("../docs/lemmatizedsites.txt", num_features=13400, num_estimators=100, testsplit=0.25, seed=1324640185, ngram_range=(1, 1), n_jobs=2) ml_trainer.run(loadfile) ml_trainer.print_stats() ml_trainer.print_confusion_matrix() # input("Press Enter to continue") webpage = "" while webpage != "EXIT": webpage = input("Enter a webpage: ") try: if webpage.lower() == "save": filename = input("Enter file name:") ml_trainer.save(filename)
class Log(object): def __init__(self): print "the log class" self.metric = Metric() self.mach_learn = MachineLearning() self.storage_nodes = [] self.normal_value_count = [] self.metrics = [] self.anomaly_value_count = [] self.thresholds_values = [] self.thresholds_lines = [] self.no_of_lines = 200 def write_data_to_file(self, file_name, line): file_name.write(line) file_name.write("\n") file_name.flush() def create_log_files(self, prefix): self.prefix = prefix if not os.path.exists("Metric\\" + self.prefix): os.makedirs("Metric\\"+self.prefix) return open(os.path.join("Metric\\"+self.prefix, "log.txt"), "a+") def check_file(self, file_name, prefix): self.prefix = prefix self.filename = file_name+".txt" filepath = os.path.join("Metric\\"+self.prefix, self.filename) if ((os.path.exists(filepath)) == False): open(os.path.join("Metric\\"+self.prefix, self.filename),"a") else: pass def setmessagelevel(self, line_data, local_time, storage_nodes, normal_value_count, metrics, anomaly_value_count, thresholds_values, thresholds_lines, fo, lo, metric_name, prefix, ): self.metric_name = metric_name self.fo = fo self.log_files = lo self.anomaly_value_count = anomaly_value_count self.line_data = line_data self.local_time = local_time self.storage_nodes = storage_nodes self.normal_value_count = normal_value_count self.thresholds_values = thresholds_values self.thresholds_lines = thresholds_lines self.metrics = metrics self.prefix = prefix if self.line_data.find("errors.timing") > 0: str_data = self.local_time, 'E: ', self.line_data self.join_line = ''.join(str_data) self.metric.write_data_to_file(self.fo, self.join_line) self.metric.write_data_to_file(self.log_files, self.join_line) elif (((self.line_data.find("errors")) > 0) | ((self.line_data.find("quarantines")) > 0)): str_data = self.local_time, 'C: ', self.line_data self.join_line = ''.join(str_data) self.metric.write_data_to_file(self.fo, self.join_line) self.metric.write_data_to_file(self.log_files, self.join_line) elif self.line_data.find("timing") > 0: str_data = self.local_time, 'I: ', self.line_data self.join_line = ''.join(str_data) self.metric.parsing_line(self.join_line) if len(self.storage_nodes) == 0: self.storage_nodes.append(self.prefix) self.normal_value_count.append([0*x for x in range( 0, len(self.metrics))]) self.anomaly_value_count.append([0*x for x in range(0, len(self.metrics))]) self.thresholds_values.append([0 for x in range(0, len(self.metrics))]) self.thresholds_lines.append([self.no_of_lines for x in range(0,len(self.metrics))]) self.mach_learn.anomaly_detection(0, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time, self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values self.storage_nodes_count = 1 else: for i in range(0, len(self.storage_nodes)): if self.prefix == self.storage_nodes[i]: self.mach_learn.anomaly_detection(i, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time,self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values self.storage_nodes_count = 1 if self.storage_nodes_count == 0: self.storage_nodes.append(self.prefix) self.normal_value_count.append([0*x for x in range(0,len(self.metrics))]) self.anomaly_value_count.append([0*x for x in range(0,len(self.metrics))]) self.thresholds_values.append([0 for x in range(0,len(self.metrics))]) self.thresholds_lines.append([self.no_of_lines for x in range(0,len(self.metrics))]) self.mach_learn.anomaly_detection(len(self.storage_nodes)-1, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time,self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values) # calling function code to detect anomaly values self.storage_nodes_count = 0 else: str_data = self.local_time, 'I: ', self.line_data self.join_line = ''.join(str_data) self.metric.write_data_to_file(self.fo, self.join_line) self.metric.write_data_to_file(self.log_files, self.join_line) return self.storage_nodes, self.normal_value_count, self.anomaly_value_count, self.thresholds_values, self.thresholds_lines def create_new_file(self): self.timestr = time.strftime("%Y%m%d-%H%M%S") self.filename = "logfile.txt" if not os.path.exists("log"): os.makedirs("log") return open(os.path.join("log", self.filename), "a")
def main_menu(): # Instantiate the Top_k class top_k = tk() spark_run = False command = 0 while command is not 'Q' and command is not 'q': command = input(""" ------------------------------------------------------------------------------- | | | | | | | | | | | | | Alzheimer's -NCI | | Top-K | | | | | | | | 1 - Print top 10 t-test values | | 2 - Print cluster mean and std | | 3 - Run machine learning | | Q - Exit | ------------------------------------------------------------------------------- Enter Command >>> """) #command = input() if command == '1': #rosmap_file = input("Enter name of ROSMAP file: ") #gene_cluster_file = input("Enter name of gene cluster file: ") rosmap_file = "rosmap.csv" gene_cluster_file = "gene_cluster.csv" start = time.time() top_k.find_top_k(rosmap_file, gene_cluster_file) top_k.print_top_k() end = time.time() print("Time: ", end - start) spark_run = True #top_k, cluster_vals = tk.find_top_k(rosmap_file, gene_cluster_file) #pprint.pprint(top_k) elif command == '2': if spark_run: cluster = input("Cluster number: ") top_k.print_cluster_values(cluster) else: load_top_ten() elif command == '3': if spark_run: top_k.top_k_vals_for_patients() ad, nci = top_k.get_all_ad_and_nci_vals() sc = top_k.get_spark_context() #import pdb; pdb.set_trace() # Instantiate the MachineLearning class ml = MachineLearning(ad, nci, sc) # Create the labeled points for models with both # life stats and no life stats ml.set_labeled_points_data() model_type = input("Include age stats? (y/n): ") if model_type == 'y' or model_type == 'Y': group = 'life_stats' else: group = 'genes' num_rf_trees = input( "Enter number of trees in random forest: ") # Build model ml.build_model(group, num_rf_trees) ml.print_all_data() else: load_top_ten() print("Bye")
class Log(object): def __init__(self): print "the log class" self.metric = Metric() self.mach_learn = MachineLearning() self.storage_nodes = [] self.normal_value_count = [] self.metrics = [] self.anomaly_value_count = [] self.thresholds_values = [] self.thresholds_lines = [] self.no_of_lines = 200 def write_data_to_file(self, file_name, line): file_name.write(line) file_name.write("\n") file_name.flush() def create_log_files(self, prefix): self.prefix = prefix if not os.path.exists("Metric\\" + self.prefix): os.makedirs("Metric\\" + self.prefix) return open(os.path.join("Metric\\" + self.prefix, "log.txt"), "a+") def check_file(self, file_name, prefix): self.prefix = prefix self.filename = file_name + ".txt" filepath = os.path.join("Metric\\" + self.prefix, self.filename) if ((os.path.exists(filepath)) == False): open(os.path.join("Metric\\" + self.prefix, self.filename), "a") else: pass def setmessagelevel( self, line_data, local_time, storage_nodes, normal_value_count, metrics, anomaly_value_count, thresholds_values, thresholds_lines, fo, lo, metric_name, prefix, ): self.metric_name = metric_name self.fo = fo self.log_files = lo self.anomaly_value_count = anomaly_value_count self.line_data = line_data self.local_time = local_time self.storage_nodes = storage_nodes self.normal_value_count = normal_value_count self.thresholds_values = thresholds_values self.thresholds_lines = thresholds_lines self.metrics = metrics self.prefix = prefix if self.line_data.find("errors.timing") > 0: str_data = self.local_time, 'E: ', self.line_data self.join_line = ''.join(str_data) self.metric.write_data_to_file(self.fo, self.join_line) self.metric.write_data_to_file(self.log_files, self.join_line) elif (((self.line_data.find("errors")) > 0) | ((self.line_data.find("quarantines")) > 0)): str_data = self.local_time, 'C: ', self.line_data self.join_line = ''.join(str_data) self.metric.write_data_to_file(self.fo, self.join_line) self.metric.write_data_to_file(self.log_files, self.join_line) elif self.line_data.find("timing") > 0: str_data = self.local_time, 'I: ', self.line_data self.join_line = ''.join(str_data) self.metric.parsing_line(self.join_line) if len(self.storage_nodes) == 0: self.storage_nodes.append(self.prefix) self.normal_value_count.append( [0 * x for x in range(0, len(self.metrics))]) self.anomaly_value_count.append( [0 * x for x in range(0, len(self.metrics))]) self.thresholds_values.append( [0 for x in range(0, len(self.metrics))]) self.thresholds_lines.append( [self.no_of_lines for x in range(0, len(self.metrics))]) self.mach_learn.anomaly_detection( 0, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time, self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values ) # calling function code to detect anomaly values self.storage_nodes_count = 1 else: for i in range(0, len(self.storage_nodes)): if self.prefix == self.storage_nodes[i]: self.mach_learn.anomaly_detection( i, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time, self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values ) # calling function code to detect anomaly values self.storage_nodes_count = 1 if self.storage_nodes_count == 0: self.storage_nodes.append(self.prefix) self.normal_value_count.append( [0 * x for x in range(0, len(self.metrics))]) self.anomaly_value_count.append( [0 * x for x in range(0, len(self.metrics))]) self.thresholds_values.append( [0 for x in range(0, len(self.metrics))]) self.thresholds_lines.append( [self.no_of_lines for x in range(0, len(self.metrics))]) self.mach_learn.anomaly_detection( len(self.storage_nodes) - 1, self.metrics, self.fo, self.log_files, self.metric_name, self.prefix, self.local_time, self.line_data, self.normal_value_count, self.thresholds_lines, self.thresholds_values ) # calling function code to detect anomaly values self.storage_nodes_count = 0 else: str_data = self.local_time, 'I: ', self.line_data self.join_line = ''.join(str_data) self.metric.write_data_to_file(self.fo, self.join_line) self.metric.write_data_to_file(self.log_files, self.join_line) return self.storage_nodes, self.normal_value_count, self.anomaly_value_count, self.thresholds_values, self.thresholds_lines def create_new_file(self): self.timestr = time.strftime("%Y%m%d-%H%M%S") self.filename = "logfile.txt" if not os.path.exists("log"): os.makedirs("log") return open(os.path.join("log", self.filename), "a")
class main: print("Starting") print("__name__ value: ", __name__) if __name__ == '__main__': print('[0]Testes') print('[1]Projeto') print('[2]Sinesp') operacao = input() if operacao is '0': print("[0] Teste com classe") print("[1] Invocando metodo") print("[2] machine learning") print("[3] matplotlib") print("[4] plotly") print("[5] pandas") print("[6] seaborn") print("[7] Open CV") print("[8] MongoDB") print("Comando ") operacao = input() if operacao is '0': Controller.mainMet1(object) if operacao is '1': p1 = Person("John", 36) print(p1.name) print(p1.age) if operacao is '2': MachineLearning.DataSet(object) if operacao is '3': TestMatPlotLib.MatPlotLibPrinc(object) if operacao is '4': TestPlotly.PlotlyPrinc(object) if operacao is '5': TestPandas.PandasPrinc(object) if operacao is '6': TestSeaborn.SeabornPrinc(object) if operacao is '7': TestOpenCV.OpenCVPrinc(object) if operacao is '8': TestPymongo.PymongoTest(object) if operacao is '1': print("[0] OpenCV") print("[1]Leitor camera") operacao = input() if operacao is '0': OpenCV.OpenCV.EdgeDetectorVideo(object) if operacao is '1': MetodoStart.Start(object) if operacao is '2': print("[0] Sinespi") operacao = input() if operacao is '0': TestSinesp.SinespPrinc(object)
def __init__(self,train_hidden_path=None,train_no_hidden_path=None,test_path=None,dest_data_path=None,dest_classes_path=None,process=None): self._config = Config(train_hidden_path,train_no_hidden_path,test_path,dest_data_path,dest_classes_path,process) self._image_processor = None self._image_statistics = None self._classificator = MachineLearning()
class StegoDetect: def __init__(self,train_hidden_path=None,train_no_hidden_path=None,test_path=None,dest_data_path=None,dest_classes_path=None,process=None): self._config = Config(train_hidden_path,train_no_hidden_path,test_path,dest_data_path,dest_classes_path,process) self._image_processor = None self._image_statistics = None self._classificator = MachineLearning() def _make_process(self): if self._config._get_process()==0: self.__preprocess_corpus() else: self.__detect_message() def __preprocess_corpus(self): f_data = open(self._config._get_dest_data_path(),"w") f_classes = open(self._config._get_dest_classes_path(),"w") for fd in listdir(self._config._get_train_hidden_path()): print fd dump(self.__preprocess_file(self._config._get_train_hidden_path()+fd),f_data) dump(1,f_classes) for fd in listdir(self._config._get_train_no_hidden_path()): print fd dump(self.__preprocess_file(self._config._get_train_no_hidden_path()+fd),f_data) dump(0,f_classes) f_data.close() f_classes.close() def __preprocess_file(self,fd): if self._image_processor == None: self._image_processor = ProcessImage(fd) else: self._image_processor._set_image(fd) self._image_processor._calc_histogram_rgb(self._image_processor._get_rgb_image()) histogram_rgb = self._image_processor._get_histogram_rgb() if self._image_statistics == None: self._image_statistics = ImageStatistics(histogram_rgb) else: self._image_statistics._set_histogram(histogram_rgb) mean_rgb = self._image_statistics._get_mean() variance_rgb = self._image_statistics._get_variance(mean_rgb) dft_histogram = self._image_processor._get_fourier_transformation() self._image_statistics._set_histogram(dft_histogram) mean_dft = self._image_statistics._get_mean() variance_dft = self._image_statistics._get_variance(mean_dft) desviation_dft = self._image_statistics._get_standard_desviation(variance_dft) skewness_dft = self._image_statistics._get_skewness(mean_dft,desviation_dft) curtosis_dft = self._image_statistics._get_curtosis(mean_dft,desviation_dft) energy_dft = self._image_processor._get_fourier_energy() mean_difference = self._image_processor._get_histogram_dft_mean_difference() feature_vector = self._image_processor._get_feature_vector(mean_rgb,variance_rgb,mean_dft,variance_dft,skewness_dft,curtosis_dft,energy_dft,mean_difference) feature_vector = self._image_processor._feature_scaling(feature_vector) return feature_vector def __unpickle(self,fd): res = [] while 1: try: res.append(load(fd)) except EOFError: break return res def __detect_message(self): f_data = open(self._config._get_dest_data_path(),"rb") f_classes = open(self._config._get_dest_classes_path(),"rb") features = self.__unpickle(f_data) classes = self.__unpickle(f_classes) f_data.close() f_classes.close() ## Train ml # self._classificator._train(features,classes) features = [] for fd in listdir(self._config._get_test_path()): res = self._classificator._test(self.__preprocess_file(self._config._get_test_path()+fd)) if res==[1]: print "File",fd,"has a hidden message" else: print "File",fd,"has not got any hidden message"
class Backend(threading.Thread): def __init__(self): super().__init__() self.is_running = True self.monitor_pool = MonitorPool() self.tolerance = 0.3 self.database = Database.Database() self.students = [] self.machine_learning = MachineLearning() self.face_capture = FaceCapture(self.database, self.machine_learning, self.students) self.delay = 0.01 def read_config(self): for e in self.database.monitor_read_all(): if e.type != "disabled": self.monitor_pool.addMonitor(e) print(e, end=',') def read_encodings(self): path = 'encoding' person_list = os.listdir(path) # 列出文件夹下所有的目录与文件 for i, person_id in enumerate(person_list): person_encoding_path = os.path.join(path, person_id) stu = Student() self.students.append(stu) stu.set_id(person_id) for i2, pics in enumerate(os.listdir(person_encoding_path)): encoding_path = os.path.join(person_encoding_path, pics) if os.path.isfile(encoding_path): encoding = numpy.load(encoding_path) stu.encodings.append(encoding) def open_capture(self): video_capture = cv2.VideoCapture(0) cv2.namedWindow("Video") frame_num = 0 face_locations = [] face_names = [] while True: ret, frame = video_capture.read() if frame_num % 5 == 0: face_locations = face_recognition.face_locations(frame) face_encodings = face_recognition.face_encodings( frame, face_locations) self.face_capture.read_img(frame, frame_num) face_names = [] for face_encoding in face_encodings: # See if the face is a match for the known face(s) rst = self.machine_learning.face_compare( self.students, face_encoding, tolerance=self.tolerance) if rst: face_names.append(str(rst.get_id())) for (top, right, bottom, left), name in zip(face_locations, face_names): cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2) cv2.rectangle(frame, (left, bottom - 35), (right, bottom), (0, 0, 255), cv2.FILLED) font = cv2.FONT_HERSHEY_DUPLEX cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1) # Display the resulting image cv2.imshow('Video', frame) frame_num += 1 # Hit 'q' on the keyboard to quit! if cv2.waitKey(1) & 0xFF == ord('q'): break # Release handle to the webcam video_capture.release() cv2.destroyAllWindows() def run(self): self.monitor_pool.startAll() cv2.namedWindow("show") frame_num = 0 while self.is_running: for e in self.monitor_pool.getMonitors(): frame = e.get_cache() self.face_capture.read_img(frame, frame_num, e) cv2.imshow("show", frame) time.sleep(self.delay) if frame_num % 1000 == 0: print("running") frame_num += 1