def getMinimalFeatureSet(): """ Returns a Processing Instance with minimal feature set """ proc = Processing() # channel features proc.addChannelFeature(MeanChannelValueFeature()) #proc.addChannelFeature(ChannelHistogramFeature()) # channel generators scale = 1.0 proc.addChannelGenerator(LaplaceChannelGenerator(scale)) proc.addChannelGenerator(GaussianGradientMagnitudeChannelGenerator(scale)) proc.addChannelGenerator(EVofGaussianHessianChannelGenerator(scale)) # supervoxel features proc.addSupervoxelFeature(SizeFeature()) return proc
def test_typeIsReadOnly(self): p = Processing("proc1", self.fixture1()) try: p.type = "AnotherType" self.fail("Exception expected") except AttributeError, e: self.assertEqual("Attribute 'type' is read only", e.args[0])
def Classification(Data, ID): Answers = [] Message = Data['message'] Answers.append(Data['message']) Result = bagOfWords(Message) if Result == 1: PreProcessing = Pre_Processing(Answers) Answers = PreProcessing.MainFunction() del Answers[0] Features = FeatureExtraction(Answers) Answers = Features.Test_TFIDF() Data1 = pd.DataFrame(Answers) del Answers Data1 = Data1.fillna(0) SentimentList = Features.Sentiment() Data1[315477] = SentimentList Processing_Object = Processing(Data1) Results = Processing_Object.Testing() print(Results[0]) else: Results = [] Results.append(0) Message = Data['message'] From = Data['from'] Time = Data['time'] ChatID = Data['ChatID'] Classified = db.collection(u"Chat").document(ChatID).collection(u"chating") Classified.add({ u"message": Message, u"from": From, u"time": Time, u"Class": str(Results[0]) })
def main(): lines = read_in() np_lines = np.array(lines) lines = list(np_lines) Message = lines[0] From = lines[1] Time = lines[2] ChatID = lines[3] Answers = [] Answers.append(Message) PreProcessing = Pre_Processing(Answers) Answers = PreProcessing.MainFunction() del Answers[0] Features = FeatureExtraction(Answers) Answers = Features.Test_TFIDF() Data1 = pd.DataFrame(Answers) del Answers Data1 = Data1.fillna(0) SentimentList = Features.Sentiment() Data1[315477] = SentimentList Processing_Object = Processing(Data1) Results = Processing_Object.Testing() Classified = db.collection(u"Chat").document(ChatID).collection(u"chating") Classified.add({ u"message": Message, u"from": From, u"time": Time, u"Class": str(Results[0]) }) print("Message Has Been Classified")
def test_connect_from_connectors_wrongDirecction(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) multi2 = Processing("multi2", engine) try: multi1 < multi2._inports except BadConnectorDirectionOrder, e: self.assertEqual(e.message, "Wrong connectors order: Input < Output")
def test_connect_to_connector_wrongDirection(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) csource = Processing("csource", engine) try: multi1 > csource.OutControl1 except BadConnectorDirectionOrder, e: self.assertEqual(e.message, "Wrong connectors order: Output > Input")
def test_connect_controls_from_slice(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) multi2 = Processing("multi2", engine) self.assertEqual(2, multi1._incontrols < multi2[::2]) self.assertEqual([ ("multi2", "OutControl1", "multi1", "InControl1"), ("multi2", "OutControl3", "multi1", "InControl2"), ], engine.controlConnections()) self.assertEqual([], engine.portConnections())
def test_connect_slice_to_slice(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) multi2 = Processing("multi2", engine) self.assertEqual(2, multi2[::2] > multi1[1:]) self.assertEqual([], engine.controlConnections()) self.assertEqual([ ("multi2", "OutPort1", "multi1", "InPort2"), ("multi2", "OutPort3", "multi1", "InPort3"), ], engine.portConnections())
def test_connect_from_processing_to_port(self): engine = self.empty() engine.addProcessing("ProcessingWithPortsAndControls", "proc1") engine.addProcessing("DummyPortSink", "proc2") proc1 = Processing("proc1", engine) proc2 = Processing("proc2", engine) self.assertEqual(1, proc1 > proc2.InPort1) self.assertEqual([], engine.controlConnections()) self.assertEqual([ ("proc1", "OutPort1", "proc2", "InPort1"), ], engine.portConnections())
def test_connect_connectors_from_slice(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) multi2 = Processing("multi2", engine) self.assertEqual(3, multi1._inports < multi2[1:]) self.assertEqual([], engine.controlConnections()) self.assertEqual([ ("multi2", "OutPort2", "multi1", "InPort1"), ("multi2", "OutPort3", "multi1", "InPort2"), ("multi2", "OutPort4", "multi1", "InPort3"), ], engine.portConnections())
def test_connect_to_connector(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) csink = Processing("csink", engine) self.assertEqual(3, multi1 > csink.InControl1) self.assertEqual([ ("multi1", "OutControl1", "csink", "InControl1"), ("multi1", "OutControl2", "csink", "InControl1"), ("multi1", "OutControl3", "csink", "InControl1"), ], engine.controlConnections()) self.assertEqual([], engine.portConnections())
def test_connect_from_processing(self): engine = self.connectivityFixture() p1 = Processing("multi1", engine) p2 = Processing("multi2", engine) p2 < p1 self.assertEqual([ ("multi1", "OutControl1", "multi2", "InControl1"), ("multi1", "OutControl2", "multi2", "InControl2"), ("multi1", "OutControl3", "multi2", "InControl3"), ], engine.controlConnections()) self.assertEqual([ ("multi1", "OutPort1", "multi2", "InPort1"), ("multi1", "OutPort2", "multi2", "InPort2"), ("multi1", "OutPort3", "multi2", "InPort3"), ], engine.portConnections())
def test_dirFunction(self): p = Processing("proc1", self.fixture1()) self.assertEqual( sorted([ 'name', 'type', 'connect', '_config', '_inports', '_outports', '_incontrols', '_outcontrols', 'ConfigParam1', 'ConfigParam2', 'InPort1', 'InPort2', 'InPort3', 'InPort4', 'OutPort1', 'OutPort2', 'InControl1', 'InControl2', 'OutControl1', 'OutControl2', ]), dir(p))
def test_connect_from_connectors(self): engine = self.connectivityFixture() multi1 = Processing("multi1", engine) multi2 = Processing("multi2", engine) self.assertEqual(3, multi2 < multi1._outcontrols) self.assertEqual(3, multi2 < multi1._outports) self.assertEqual([ ("multi1", "OutControl1", "multi2", "InControl1"), ("multi1", "OutControl2", "multi2", "InControl2"), ("multi1", "OutControl3", "multi2", "InControl3"), ], engine.controlConnections()) self.assertEqual([ ("multi1", "OutPort1", "multi2", "InPort1"), ("multi1", "OutPort2", "multi2", "InPort2"), ("multi1", "OutPort3", "multi2", "InPort3"), ], engine.portConnections())
def test_connect_to_alien(self): engine = self.connectivityFixture() proc1 = Processing("multi1", engine) try: proc1 > 34 except AssertionError, e: self.assertEqual(e.message, "Unexpected connection peer: 34")
def test_connect_from_alien(self): engine = self.connectivityFixture() proc1 = Processing("multi1", engine) try: proc1 < 34 except AttributeError, e: self.assertEqual(e.message, "'int' object has no attribute 'connect'")
def bootstrap(): #dataset = Processing().import_data() for dataset, filename in Processing().import_single_data(): training_data_X, training_data_y, testing_data_X, testing_data_y = Processing( ).separate_data(dataset) rs = RankSVM().fit(training_data_X, training_data_y) rs_pred_y = np.around(rs.predict2(testing_data_X)) rs_fpa = PerformanceMeasure(testing_data_y, rs_pred_y).FPA() rs_aae_result = PerformanceMeasure(testing_data_y, rs_pred_y).AAE() print('rs_fpa:', rs_fpa, 'rs_aae_result', rs_aae_result) lr = linear_model.LinearRegression().fit(training_data_X, training_data_y) lr_pred_y = np.around(lr.predict(testing_data_X)) lr_fpa = PerformanceMeasure(testing_data_y, lr_pred_y).FPA() lr_aae_result = PerformanceMeasure(testing_data_y, lr_pred_y).AAE() print('lr_fpa:', lr_fpa, 'lr_aae_result', lr_aae_result)
def ListenThread(Connection, Address): try: while True: Message = Connection.recv(1024).decode("utf-8") Answers = [] Answers.append(Message) Answers2 = Answers PreProcessing = Pre_Processing(Answers) Answers = PreProcessing.MainFunction() del Answers[0] Answers2 = Answers2[0] Features = FeatureExtraction(Answers) Answers = Features.Test_TFIDF() Data = pd.DataFrame(Answers) Data = Data.fillna(0) SentimentList = Features.Sentiment() Data[44247] = SentimentList Processing_Object = Processing(Data) Results = Processing_Object.Testing() Results = str(Results[0]) SendingConnection = None print(Message) for i in range(len(SendingIPS)): if Address != SendingIPS[i]: SendingConnection = SendingConnections[i] break SendThread(SendingConnection, Message, Results) except: # print("Client has Disconnected or un expected error ocurred please DEBUG!!!") # for i in range(len(SendingIPS)): # if Address == SendingIPS[i]: # Connection.close() # del SendingIPS[i] # print(len(SendingConnections)) # print(len(SendingIPS)) # break python = sys.executable os.execl(python, python, *sys.argv)
def main(): for dataset, filename in Processing().dataload(): print(filename) X, y = convert2numpy(dataset) print('X:', X.shape) ratio = 4 # 小于4%的模块合并 k = 5 r = 0.1 ssmote = Ssmote(X=X, y=y, ratio=ratio, k=k, r=r) synX, synY = ssmote.synthesis() print(synX) print(synY)
def bootstrap(): #dataset = Processing().import_data() for dataset, filename in Processing().import_single_data(): print(filename) training_data_X, training_data_y, testing_data_X, testing_data_y = Processing( ).separate_data(dataset) # print('train shape', training_data_X.shape) training_data_X = training_data_X.tolist() training_data_y = training_data_y.tolist() from PyOptimize.General_Opt import Test_function def LTR(a, **kwargs): return Test_function().LTR(a, **kwargs) ga = pyGaft(objfunc=LTR, var_bounds=[(-2, 2)] * 20, individual_size=50, max_iter=10, max_or_min='max', X=training_data_X, y=training_data_y).run() importlib.reload(best_fit) a, fitness_value = best_fit.best_fit[-1][-2], best_fit.best_fit[-1][-1] print('a = {0}'.format(a)) pred_y = [] for test_x in testing_data_X: pred_y.append(np.dot(test_x, a)) fpa = PerformanceMeasure(testing_data_y, pred_y).FPA() print('fpa = {0}'.format(fpa))
def proportion(): # 先加入header with open('proportion2.csv', 'w') as f: columns = [ 'Release', 'Count 0', 'Count 1', 'Count 2', 'Count 3', 'Count 4', 'Count 5', 'Count 6', 'Count 7', 'Count 8', 'Count gt 8' ] writer = csv.DictWriter(f, fieldnames=columns) writer.writeheader() # 添加数据 for dataset, filename in Processing().dataload(): proportion_(dataset, filename, mode=1) pass
def main(is_print=False): params = {'root_path': root_path} processing_ = Processing(params) stats = Analyze(processing_.dict_, columns=['title', 'paper_id', 'abstract', 'body_text']) if is_print is True: print(stats.df_covid.describe(include='all')) count = 0 for x in processing_.dict_['title']: if x is "": count += 1 print("\n\n\n\nQuantity of title is empty: {}".format(count)) return stats
def fiveElements(): """ 统计五个元素 """ all_filenames = [] all_instances = [] all_defects = [] all_percent_defects = [] all_maxs = [] all_avgs = [] for dataset, filename in Processing().dataload(): data = convert2numpy(dataset) statics = Statistics(data[1]) # #Instance # print(filename) number_instance = statics.numberInstance() number_defect = statics.numberDefect() percent_defect = statics.percentDefect() max_bug = statics.max_() avg_bug = statics.avg_() # print('\n') # print('#Instance :{0}'.format(number_instance)) # print('#Defects :{0}'.format(number_defect)) # print('%Defects :{:.1f}'.format(percent_defect)) # print('max_bug :{0}'.format(max_bug)) # print('avg_bug :{:.2f}'.format(avg_bug)) all_filenames.append(filename) all_instances.append(number_instance) all_defects.append(number_defect) all_percent_defects.append(percent_defect) all_maxs.append(max_bug) all_avgs.append(avg_bug) print(all_filenames) data_ = {'Release': all_filenames, 'number_instance': all_instances, 'number_defect': all_defects, 'percent_defect': all_percent_defects, 'max_bug': all_maxs, 'avg_bug': all_avgs} save_(r'fiveElements.csv', filename, **data_) pass
def bootstrap(dataset): training_data_X, training_data_y, testing_data_X, testing_data_y = Processing( ).separate_data(dataset) brr = BayesianRidge() de = DE(NP=100, F=0.6, CR=0.7, generation=2000, len_x=3, ratioRange=[0.5, 1.0, 2.0, 4.0], kRange=list(range(1, 21)), value_up_range=5.0, value_down_range=0.1, X=training_data_X, y=training_data_y, classifier=brr) de.process()
def on_previewBtn_clicked(self): print('on_previewBtn_clicked') # find out what kind of document to create (crb = checked radio btn) crbName, crbWidget = self.get_checked_radio_button() doctype = self._radio_choices.get(crbName, None) filledFields = self.get_form_brief() # open a popup for progressbar, in which we call a DocBuilder dialog = QtWidgets.QDialog() process_bar_dialog = Processing(dialog) process_bar_dialog.configurate_doc_worker(doctype, filledFields) process_bar_dialog.run_thread() dialog.exec_() # call whatever the system use to open pdf path_to_file = ('file:///' + os.getcwd().replace('\\', '/') + '/result.pdf') QtGui.QDesktopServices.openUrl(QtCore.QUrl(path_to_file))
def run_scraper(self): self.p.linker(self.p.scraper_dict, 'scraper') scraper = Scraper(self.p.scraper) processing = Processing(self.p.scraper) scraper.run() self.ProgressBar.setValue(25) processing.pre_processor(scraper.scraper_df) self.ProgressBar.setValue(50) processing.spacy_processor(scraper.scraper_df) self.ProgressBar.setValue(75) self.connection = Connect(settings=None, mongo_cfg=self.p.scraper) self.connection.update_from_df(scraper.scraper_df) self.mongo_update() self.ProgressBar.setValue(100)
from Time import Time as t from Lists import Lists as l from Processing import Processing as p ### PROCESSING # get all files from a directory (default is current directory) print(p.get_files()) int_list = [i for i in range(20)] str_list = [s for s in "abcdefg"] ### LISTS # create batches batched = [i for i in l.batch(int_list)] print("Batched list: {}".format(batched)) ### TIME # Timeout with printed countdown t.sleep_countdown(1, print_step=1) # current timestamp print("Current timestamp: {}".format(t.timestamp_now())) # add years to date from datetime import datetime d = datetime.datetime(1970, 1, 1) d_plus = t.date_add_year(d, 2) print(d) print(d_plus)
def bootstrap(): #dataset = Processing().import_data() count = 0 for dataset, filename in Processing().import_single_data(): print(filename) count += 1 training_data_X, training_data_y, testing_data_X, testing_data_y = Processing( ).separate_data(dataset) # print('train shape', training_data_X.shape) # 1.降序排列训练集(Processing中已完成) # 2.利用transfrom_pairwise() 得到Pi,ri # P是一个矩阵,每个向量是两个x相减的结果 # r是一个向量 因为排序过,所以结果r = [1,1,1,1,1,1...] rs = RankSVM() P, r = rs.transform_pairwise(training_data_X, training_data_y) #print('p shape ', P.shape, 'r len ', len(r)) P = P.tolist() r = r.tolist() print('type of P ', type(P[0][0]), 'type of r ', type(r[0])) # P = [[1, 1, 2], [1, -1, 3], [3, 2, 1], [1, -5, 1], [2, 1, -2]] # r = [1, 1, 1, 1, 1] # 3.用training_data_y计算u,n u, n = PerformanceMeasure(training_data_y).calc_UN(type='cs') # print(len(u), len(n)) print(type(u[0]), type(n[0])) # 4. 将Pi,ri,u,n导入genetic algorithm 计算w from PyOptimize.General_Opt import Test_function def Loss(x, **kwargs): return Test_function().Loss(x, **kwargs) ga = pyGaft(objfunc=Loss, var_bounds=[(-2, 2)] * 20, individual_size=50, max_iter=200, max_or_min='min', P=P, r=r, u=u, n=n).run() # 5.编写predict3 # w 从best_fit中获得 if count == 1: import best_fit else: importlib.reload(best_fit) w, fitness = best_fit.best_fit[-1][-2], best_fit.best_fit[-1][-1] print('w = ', w) rs_pred_y = RankSVM(w=w).predict3(testing_data_X) rs_pred_y = np.around(rs_pred_y) rs_fpa = PerformanceMeasure(testing_data_y, rs_pred_y).FPA() print('rs_fpa:', rs_fpa) # f1.append(rs_fpa) # RankSVM 效果 from sklearn.utils import shuffle X_shuf, y_shuf = shuffle(training_data_X, training_data_y) rs2 = RankSVM().fit(X_shuf, y_shuf) rs_pred_y2 = np.around(rs2.predict2(testing_data_X)) rs_fpa2 = PerformanceMeasure(testing_data_y, rs_pred_y2).FPA() rs_aae_result = PerformanceMeasure(testing_data_y, rs_pred_y2).AAE() print('rs_fpa2:', rs_fpa2)
def test_processing(): image = cv2.imread("../images/buffer/output.png", cv2.IMREAD_GRAYSCALE) assert not np.array_equal( Processing.process_image('../images/buffer/new.png'), image )
def test_IncontrolsGettingByNameAsDictionary(self): p = Processing("proc1", self.fixture1()) self.assertEqual(p["InControl1"].name, "InControl1") self.assertEqual(p["InControl1"].kind, "Control") self.assertEqual(p["InControl1"].direction, "In")
def test_IncontrolsGettingByNameAsAttribute(self): p = Processing("proc1", self.fixture1()) self.assertEqual(p.InControl2.name, "InControl2") self.assertEqual(p.InControl2.kind, "Control") self.assertEqual(p.InControl2.direction, "In")