def getResult(self, src, save_image=False): """ @param {string} src {bool}save_image output debug image @return {OCRDocument} doucument """ pro = DataProcessor(src, ImageType.RAW, save_image=save_image) if pro.prepare() is None: logger.error('image error:{0}'.format(src)) return None buffer = pro.tobinary(pro.batch()) temp_file_name = self.create_TemporyFile(buffer, True) document = self.ocr.recognize(temp_file_name) os.remove(temp_file_name) output = '#' + datetime.now().strftime('%F %T.%f')[:-3] + '\n' output += '\n'.join(document.names()) + '\n' with Serializer.open_stream('../temp/corpus.txt', mode='a') as file: file.write(output) # ocr corpus data -> NaiveBayes classifier # ranking name swap change = self.naivebayes.predict_all(document.names()) #doucument.changeNames(change) document.dump() return document
def _generate_model_examples(processor: DataProcessor, data_dir: str, config: dict): train_examples = processor.get_train_examples(data_dir) eval_examples = processor.get_eval_examples(data_dir) test_examples = processor.get_test_examples(data_dir) config["train_size"] = len(train_examples) config["eval_size"] = len(eval_examples) config["test_size"] = len(test_examples) return [train_examples, eval_examples, test_examples], config
def __init__(self, media, target=None): self.media = media self.target = target pro = DataProcessor(self.media, ImageType.PLAN) pro.prepare() self.binary = pro.batch() self.color = pro.color self.drawClipSource = True self.imageout_dir = '../temp/trash' os.makedirs(self.imageout_dir, exist_ok=True)
def __init__(self, callback): tk.Toplevel.__init__(self) self.grab_set() self.title('Process file') self.geometry("%dx%d%+d%+d" % (400, 300, 300, 200)) container = tk.Frame(self) container.pack(side="top", fill="both", expand=True) container.grid_rowconfigure(0, weight=1) container.grid_columnconfigure(0, weight=1) self.callback = callback self.processor = DataProcessor() self.defaultvar = 'none' self.radiovar = tk.StringVar(None, self.defaultvar) self.frames = {} for F in (PageTwo, PageThree, PageFour, PageFive): page_name = F.__name__ frame = F(parent=container, controller=self) self.frames[page_name] = frame frame.grid(row=0, column=0, sticky="nsew") self.show_frame("PageTwo")
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._dp = DataProcessor.get_instance(hostname=etcd_ip, port=etcd_port, username=username, password=password, xtc_ip=xtc_ip)
def main(): config = ConfigParser() config.read("config.ini") interval = config.getint("main", "interval") base_endpoint = config.get("main", "base_endpoint") storage_account_conn_str = config.get("azure", "storage_account_conn_str") azure_client = AzureStorageClient(storage_account_conn_str, 'asdsWiki') data = azure_client.get_table() scrapper = WikiScrapper(base_endpoint) dataframe = scrapper.start(data, interval) dataProcessor = DataProcessor() dataProcessor.preprocess_dataframe(dataframe) azure_client.update_by_dataframe(dataframe)
def main(): ui = UserInterface() url_builder = URLBuilder() web_dao = WebDAO() data_processor = DataProcessor() url_builder\ .set_args_dict(ui.get_args_dict())\ .build() web_dao\ .set_download_url(url_builder.get_url())\ .download() data_processor\ .set_raw_content(web_dao.get_raw_content()) data_processor.save_json( f"target/cpsc_recalls_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
class OdometryHandler(object): def __init__(self): self.arduino_data_processor = DataProcessor("arduino_data", 0.025, 0.210) self.pub = Publisher('odom', "odom", "base_link") def main(self): current_time = self.pub.getCurrentTime() x_y_theta_t, vx_vth = self.arduino_data_processor.getPublisherInfo() if self.arduino_data_processor.arduino_data is not None: self.pub.publishMessage( self.pub.createNavMsg(current_time, x_y_theta_t, vx_vth), self.pub.createTF(current_time, x_y_theta_t))
parser.add_argument("--visdom", help="visualize training via visdom_enabled library", default=True) parser.add_argument("--gru", help="use GRU units instead of LSTM units", default=False) parser.add_argument( "--sanitycheck", help= 'list of words for which the nearest word embeddings are found during training, ' 'serves as sanity check, i.e. "dog family king eye"', default="dog family king eye") if __name__ == "__main__": parser = argparse.ArgumentParser() init_parser(parser) args = parser.parse_args() data_proc = DataProcessor(args) model = Rec2Vec(data_proc) bytes_read = 0 epochs = 30 for e in range(epochs): print(f"Starting epoch: {e}") bytes_read = model._train(previously_read=bytes_read, epoch=e) model.save(f"trained/Trefined_Oembeddings{epochs}.vec", model.in_embeddings) model.save(f"trained/Trefined_Vembeddings{epochs}.vec", model.v_embeddings)
def __init__(self): self.arduino_data_processor = DataProcessor("arduino_data", 0.025, 0.210) self.pub = Publisher('odom', "odom", "base_link")
etcd_ip = opt_value print("[*] Etcd IP is {}".format(etcd_ip)) if opt_name in ('-e', '--etcd-port'): etcd_port = int(opt_value) print("[*] Etcd Port is {}".format(etcd_port)) if opt_name in ('-x', '--xtc-ip'): xtc_ip = opt_value print("[*] XTC IP is {}".format(xtc_ip)) if None in [username, password, etcd_ip, etcd_port, xtc_ip]: print( "[*] Help: Please enter username, password, Etcd IP, Etcd Port in parameters. Example: \n python main.py -u cisco -p cisco -e 127.0.0.1 -ep 2379" ) exit() application = tornado.web.Application([ (r"/api/v1/devices", DeviceListHandler), (r"/api/v1/calculate", RouterCalcHandler) ]) application.listen(9888) loop = asyncio.get_event_loop() dp = DataProcessor.get_instance(hostname=etcd_ip, port=etcd_port, username=username, password=password, xtc_ip=xtc_ip) asyncio.ensure_future(dp.start_watch()) loop.run_forever() # tornado.ioloop.IOLoop.current().start()
from datasources import BPMData data_source = BPMData() elif bpm_name_parsed == "all": from datasources_all import BPMDataAll data_source = BPMDataAll() else: from datasources_bpm import BPMData data_source = BPMData(bpm_name=bpm_name_parsed) if data_source is None: print("Data source doesn't exists!!! You can't use this program!!!") exit() data_proc_X = DataProcessor("X") data_proc_Z = DataProcessor("Z") settingsControl = SettingsControl() mw = MainWindow(data_source, data_proc_X, data_proc_Z, settingsControl, bpm_name_parsed) mw.setWindowTitle('BTMS ({})'.format(bpm_name_parsed)) icon_path = os.path.dirname(os.path.abspath(__file__)) mw_icon = QIcon() mw_icon.addFile(os.path.join(icon_path, 'etc/icons/app_icon_color.png'), QSize(32, 32)) mw.setWindowIcon(mw_icon) data_source.data_ready.connect(mw.on_data1_ready) data_source.data_ready.connect(mw.on_data3_ready) data_source.data_ready.connect(data_proc_X.on_data_recv) data_source.data_ready.connect(data_proc_Z.on_data_recv)
from dataprocessor import DataProcessor from preprocess import clean from fileio import FileIO from joblib import load import numpy as np processor = DataProcessor() io = FileIO() path_negation_words = 'negation' def predict(document): # predict document input doc_clean = clean.clean_review(document) doc_clean_process_number = clean.number_process(doc_clean) print(doc_clean_process_number) tfidf = processor.transform(np.array([doc_clean_process_number])) reload = load("model_test") predict = reload.predict(tfidf) if predict == "2": print("POSITIVE :)") elif predict == "1": print("NEUTRAL") else: print("NEGATIVE -_-")
class Application(Frame): def __init__(self): super().__init__() self.is_all_data = IntVar() self.selected_month_1 = StringVar() self.selected_year_1 = StringVar() self.selected_month_2 = StringVar() self.selected_year_2 = StringVar() self.selected_saham = StringVar() self.finished = False self.initUI() def initUI(self): self.master.title("Analisa Harga Saham") self.pack(fill=BOTH, expand=True) self.columnconfigure(1, weight=1) self.columnconfigure(3, pad=7) self.rowconfigure(3, weight=1) self.rowconfigure(5, pad=7) self.lbl = Label(self, text="Filename: ") self.lbl.grid(sticky=W, pady=4, padx=5, columnspan=4) self.right_frame = Frame(self, width=800, height=400, borderwidth = 1) self.right_frame.grid(row=2, column=1, columnspan=5, rowspan=4, padx=5, sticky=E+W+S+N) self.right_frame.config(relief=SOLID) self.area = Text(self.right_frame, height = 30, width = 40) self.area.grid(row=0, column=1, padx=5, sticky=W+S+N+E) self.splash = Toplevel(self.right_frame) self.splash.overrideredirect(True) self.splash.geometry('200x23+100+100') self.splash.overrideredirect(1) self.splash.bind("<B1-Motion>", self.move_window) self.splash.attributes('-topmost', 'true') window_height = 23 window_width = 400 screen_width = self.splash.winfo_screenwidth() screen_height = self.splash.winfo_screenheight() x_cordinate = int((screen_width/2) - (window_width/2)) y_cordinate = int((screen_height/2) - (window_height/2)) self.splash.geometry("{}x{}+{}+{}".format(window_width, window_height, x_cordinate, y_cordinate)) self.splash.withdraw() pb = Progressbar(self.splash, orient=HORIZONTAL, length=400) pb.config(mode='indeterminate') pb.start(10) pb.grid(row=1, column=1, sticky=W+E+S+N) # self.splash.withdraw() self.dp = DataProcessor() # upload # lblUpload = Label(self, text="upload") # lblUpload.grid(row=1, column=0, columnspan=2) abtn = Button(self, text="Upload", command=self.openFile) abtn.grid(row=1, column=0, sticky=W, padx=5) self.left_frame = Frame(self, width=200, height=400, borderwidth = 1) self.left_frame.grid(row=2, column=0) self.left_frame.config(relief=SOLID) # self.chkBox = Checkbutton(self.left_frame, text = "All data", variable=self.is_all_data, command=self.cbCallback) # self.chkBox.grid(row=1, column=0, sticky=W, padx=5) Separator(self.left_frame,orient=HORIZONTAL).grid(row=2, columnspan=1, ipadx=75, padx=5, sticky=W) self.rangeFrame = self.rangeFrame() #Frame(self.left_frame, borderwidth = 1) self.rangeFrame.grid(row=3, column=0, columnspan=2) # Button Filter self.btnFilter = Button(self.left_frame, text="Filter", command=self.callFilter) self.btnFilter.grid(row=4, column=0, sticky=W, padx=5) Separator(self.left_frame,orient=HORIZONTAL).grid(row=5, columnspan=1, ipadx=75, padx=5, sticky=W) self.txSaham = Text(self.left_frame) self.cbSaham = Combobox(self.left_frame, textvariable=self.selected_saham) self.cbSaham['values'] = [] #self.kodeSaham self.cbSaham['state'] = 'readonly' # normal self.cbSaham.set('-- Pilih Saham --') self.cbSaham.grid(row=7, column=0,padx=5, pady=5) # Buton Proses self.btnProses = Button(self.left_frame, text="proses", command=self.callProses) self.btnProses.grid(row=8, column=0, sticky=W, padx=5) Separator(self.left_frame,orient=HORIZONTAL).grid(row=9, columnspan=1, ipadx=75, padx=5, sticky=W) def rangeFrame(self): frame = Frame(self.left_frame, borderwidth = 1) months = ('01-Jan', '02-Feb', '03-Mar', '04-Apr', '05-May', '06-Jun', '07-Jul', '08-Aug', '09-Sep', '10-Oct', '11-Nov', '12-Dec') years = (2018, 2019, 2020, 2021) lblFrom = Label(frame, text='Dari') lblFrom.grid(row=1, column=0, sticky=W, padx=5) # start bulan month_cb_1 = Combobox(frame, textvariable=self.selected_month_1) month_cb_1['values'] = months month_cb_1['state'] = 'readonly' # normal month_cb_1.set('-- Pilih bulan --') month_cb_1.grid(row=2, column=0,padx=5, pady=5) # start tahun year_cb_1 = Combobox(frame, textvariable=self.selected_year_1) year_cb_1['values'] = years year_cb_1['state'] = 'readonly' # normal year_cb_1.set('-- Pilih tahun --') year_cb_1.grid(row=3, column=0,padx=5, pady=5) lblTo = Label(frame, text='Ke') lblTo.grid(row=4, column=0, sticky=W, padx=5) # end bulan month_cb_2 = Combobox(frame, textvariable=self.selected_month_2) month_cb_2['values'] = months month_cb_2['state'] = 'readonly' # normal month_cb_2.set('-- Pilih bulan --') month_cb_2.grid( row=5, column=0,padx=5, pady=5) # end tahun year_cb_2 = Combobox(frame, textvariable=self.selected_year_2) year_cb_2['values'] = years year_cb_2['state'] = 'readonly' # normal year_cb_2.set('-- Pilih tahun --') year_cb_2.grid(row=6, column=0,padx=5, pady=5) return frame # open data file def openFile(self): name = fd.askopenfilename() self.lbl['text'] = self.lbl['text']+' '+name self.splash.deiconify() msg = self.dp.load(name) self.splash.withdraw() messagebox.showinfo("Info", msg) # show and hide frame def cbCallback(self): if(self.is_all_data.get() == 1): self.rangeFrame.grid_forget() else: self.rangeFrame.grid() def disableButton(self, disable=True): if disable: self.btnFilter['state'] = DISABLED self.btnProses['state'] = DISABLED else: self.btnFilter['state'] = NORMAL self.btnProses['state'] = NORMAL ''' filter data berdasarkan bulan dipilih ''' def filter(self): if self.dp.df is None: messagebox.showerror("Error", "Data kosong, silahkan upload data") return if 'messages' not in self.dp.df.columns: messagebox.showerror("Error", "Data tidak sesuai atau bukan dari log Telegram") return if 'pilih' in self.selected_year_1.get().lower() or 'pilih' in self.selected_month_1.get().lower(): messagebox.showerror("Error", "Bulan dan Tahun awal harus dipilih!") return self.disableButton() self.splash.deiconify() if(self.is_all_data.get()==1): self.dp.filter(awal = '0000', akhir='0000') else: if('Pilih' not in ''.join([self.selected_year_1.get(),self.selected_month_1.get()]) and 'Pilih' in ''.join([self.selected_year_2.get(),self.selected_month_2.get()])): self.dp.filter(awal = str(self.selected_year_1.get())+'-'+self.selected_month_1.get()[:2], akhir = '0000') else: awal = str(self.selected_year_1.get())+'-'+self.selected_month_1.get()[:2] akhir = str(self.selected_year_2.get())+'-'+self.selected_month_2.get()[:2] if datetime.datetime.strptime(awal, '%Y-%m') > datetime.datetime.strptime(akhir, '%Y-%m'): self.disableButton(False) self.splash.withdraw() messagebox.showerror("Error", "Waktu awal > Waktu akhir") return self.dp.filter(awal = awal, akhir = akhir) print(self.dp.dffilter) lst = self.dp.getKodeSaham() self.cbSaham['values'] = lst self.splash.withdraw() messagebox.showinfo("Info", "filter selesai") self.disableButton(False) def callFilter(self): threading.Thread(target=self.filter).start() ''' sanding data dari 2 paket data: mentions counter dan price dari yfinance plot line graph ''' def proses(self): if self.dp.dffilter is None: messagebox.showerror("Error", "Data kosong silahkan filter data terlebih dahulu") return stockcode = self.cbSaham.get() if 'pilih' in stockcode.lower(): messagebox.showerror("Error", "Kode saham belum dipilih!") return stockcode = stockcode.split()[0] self.disableButton() self.splash.deiconify() '''if(len(self.txSaham.get())>0): stockcode = self.txSaham.get()''' # menghitung frekuensi mentions saham di chat dfa = self.dp.getCount(stockcode) print('dfa ',dfa.shape) # mendapatkan stock price's series from yahoo finance self.company_name, stockprice, dfstock = self.dp.getPergerakanHargaSaham(stockcode) print('dfstock ', dfstock.shape) # sanding data self.dfnorm, dfbefnorm = self.dp.sandingData(dfa, stockprice) print('sandingData--- ') self.area.delete('1.0', END) self.area.insert(END, 'correlation {0}\n\n {1}'.format(self.dp.corr(self.dfnorm), dfbefnorm)) # # plot # self.plot(self.right_frame, self.dfnorm, self.company_name) self.finished = True self.disableButton(False) self.splash.withdraw() def callProses(self): # kami memanfaatkan link ini untuk mengetahui bahwa thread sudah selesai # https://stackoverflow.com/a/56010976 POLLING_DELAY = 250 # ms lock = threading.Lock() # Lock for shared resources. def check_status(): with lock: if not self.finished: self.after(POLLING_DELAY, check_status) # Keep polling. else: self.plot(self.right_frame, self.dfnorm, self.company_name) with lock: self.finished = False t = threading.Thread(target=self.proses) t.daemon = True self.after(POLLING_DELAY, check_status) # Start polling. t.start() def move_window(self, e): self.splash.geometry(f'+{e.x_root}+{e.y_root}') def plot(self, root, dfa1, company_name): print('plot') figure1 = plt.Figure(figsize=(9,5), dpi=100) ax1 = figure1.add_subplot(111) bar1 = FigureCanvasTkAgg(figure1, root) bar1.get_tk_widget().grid(row=0, column=2) df1 = dfa1[['date','mentions']].groupby('date').sum() df1.plot( kind='line', legend=True, ax=ax1, color='skyblue',marker='o', fontsize=10) df2 = dfa1[['date', 'price']].groupby('date').sum() df2.plot( kind='line', legend=True, ax=ax1, marker='', color='olive', linewidth=2) # plt.legend() ax1.set_title("Pergerakan Harga Saham "+company_name)
pd.set_option('display.max_row', 10000) pd.set_option('display.max_columns', 1000) if __name__ == '__main__': # Reading datset and removing redundant features data_file = "/Users/salil/Downloads/training_car.csv" df_read = pd.read_csv(data_file) df_read = df_read.drop([ 'AUCGUART', 'PRIMEUNIT', 'Nationality', 'VNZIP1', 'VNST', 'BYRNO', 'WheelTypeID', 'PurchDate', 'VehYear' ], axis=1) data = DataProcessor() total_columns = df_read.columns catcols, contcols = data.get_cat_cont_cols(df_read, total_columns) print("Categorical columns: ", catcols) uid = ['RefId'] target = ['IsBadBuy'] contcols = list(set(contcols) - set(uid) - set(target)) features = catcols + contcols print("Numerical columns after target and id removal: ", contcols) df_read.Transmission[df_read.Transmission == 'Manual'] = 'MANUAL' df_read.Color[df_read.Color == 'NOT AVAIL'] = 'NA' df_read.Color[df_read.Color == 'OTHER'] = 'NA' df_read.TopThreeAmericanName[df_read.TopThreeAmericanName == 'OTHER'] = 'NA'
import matplotlib.pyplot as plt import os import numpy as np import folium import pytz as tz # better alternatives -> Apache arrow or pendulum from datetime import datetime from PIL import Image import urllib import urllib.request import wget import ray from scipy.spatial import Voronoi from numpy import array # data process dataprocessor = DataProcessor('/Users/wangyifan/Google Drive/checkin', 'loc-gowalla_totalCheckins.txt') df = dataprocessor.load_date() df = dataprocessor.data_filter(df) df = dataprocessor.data_process(df) # df = df.sample(n=2000, replace=False).reset_index(drop=True) # config: data 30000 cluster_k: 20 df = df[:30000] df_kmeans = df.copy() df_kmeans = df_kmeans[['lat', 'lon']] batch_num = 10 cluster_k = 20 epsilon = 1e-4 precision = 1e-6 iteration = 10 """
def kickoffDataProcessor(self): from dataprocessor import DataProcessor d = DataProcessor(self.results) d.process() print("ALL RESULTS: \n" + str(self.results) + "\n")
def initUI(self): self.master.title("Analisa Harga Saham") self.pack(fill=BOTH, expand=True) self.columnconfigure(1, weight=1) self.columnconfigure(3, pad=7) self.rowconfigure(3, weight=1) self.rowconfigure(5, pad=7) self.lbl = Label(self, text="Filename: ") self.lbl.grid(sticky=W, pady=4, padx=5, columnspan=4) self.right_frame = Frame(self, width=800, height=400, borderwidth = 1) self.right_frame.grid(row=2, column=1, columnspan=5, rowspan=4, padx=5, sticky=E+W+S+N) self.right_frame.config(relief=SOLID) self.area = Text(self.right_frame, height = 30, width = 40) self.area.grid(row=0, column=1, padx=5, sticky=W+S+N+E) self.splash = Toplevel(self.right_frame) self.splash.overrideredirect(True) self.splash.geometry('200x23+100+100') self.splash.overrideredirect(1) self.splash.bind("<B1-Motion>", self.move_window) self.splash.attributes('-topmost', 'true') window_height = 23 window_width = 400 screen_width = self.splash.winfo_screenwidth() screen_height = self.splash.winfo_screenheight() x_cordinate = int((screen_width/2) - (window_width/2)) y_cordinate = int((screen_height/2) - (window_height/2)) self.splash.geometry("{}x{}+{}+{}".format(window_width, window_height, x_cordinate, y_cordinate)) self.splash.withdraw() pb = Progressbar(self.splash, orient=HORIZONTAL, length=400) pb.config(mode='indeterminate') pb.start(10) pb.grid(row=1, column=1, sticky=W+E+S+N) # self.splash.withdraw() self.dp = DataProcessor() # upload # lblUpload = Label(self, text="upload") # lblUpload.grid(row=1, column=0, columnspan=2) abtn = Button(self, text="Upload", command=self.openFile) abtn.grid(row=1, column=0, sticky=W, padx=5) self.left_frame = Frame(self, width=200, height=400, borderwidth = 1) self.left_frame.grid(row=2, column=0) self.left_frame.config(relief=SOLID) # self.chkBox = Checkbutton(self.left_frame, text = "All data", variable=self.is_all_data, command=self.cbCallback) # self.chkBox.grid(row=1, column=0, sticky=W, padx=5) Separator(self.left_frame,orient=HORIZONTAL).grid(row=2, columnspan=1, ipadx=75, padx=5, sticky=W) self.rangeFrame = self.rangeFrame() #Frame(self.left_frame, borderwidth = 1) self.rangeFrame.grid(row=3, column=0, columnspan=2) # Button Filter self.btnFilter = Button(self.left_frame, text="Filter", command=self.callFilter) self.btnFilter.grid(row=4, column=0, sticky=W, padx=5) Separator(self.left_frame,orient=HORIZONTAL).grid(row=5, columnspan=1, ipadx=75, padx=5, sticky=W) self.txSaham = Text(self.left_frame) self.cbSaham = Combobox(self.left_frame, textvariable=self.selected_saham) self.cbSaham['values'] = [] #self.kodeSaham self.cbSaham['state'] = 'readonly' # normal self.cbSaham.set('-- Pilih Saham --') self.cbSaham.grid(row=7, column=0,padx=5, pady=5) # Buton Proses self.btnProses = Button(self.left_frame, text="proses", command=self.callProses) self.btnProses.grid(row=8, column=0, sticky=W, padx=5) Separator(self.left_frame,orient=HORIZONTAL).grid(row=9, columnspan=1, ipadx=75, padx=5, sticky=W)
def __init__(self): super(MainWindow, self).__init__() self.ui = uic.loadUi('MainWindow.ui', self) self.setWindowTitle("Frequency Measurer") self.window_str = "None" self.frq_founded = 0.0 self.buttonExit.clicked.connect(self.on_exit_button) self.buttonExit.clicked.connect(QApplication.instance().quit) self.data_source = BPMData(1024, self) self.data_source.data_ready.connect(self.on_data1_ready) self.data_source.data_ready.connect(self.on_data3_ready) self.data_proc_X = DataProcessor("X") self.data_proc_Z = DataProcessor("Z") self.data_source.data_ready.connect(self.data_proc_X.on_data_recv) self.data_source.data_ready.connect(self.data_proc_Z.on_data_recv) self.data_proc_X.data_processed.connect(self.on_data2_ready) self.data_proc_Z.data_processed.connect(self.on_data4_ready) self.controlWidgetX.window_changed_str.connect( self.data_proc_X.on_wind_changed) self.controlWidgetX.groupBox.setTitle("Data_X") self.controlWidgetX.set_str_id("Data_X") self.controlWidgetX.scale_changed_obj.connect(self.on_scale_changing) self.controlWidgetZ.window_changed_str.connect( self.data_proc_Z.on_wind_changed) self.controlWidgetZ.groupBox.setTitle("Data_Z") self.controlWidgetZ.set_str_id("Data_Z") self.controlWidgetZ.scale_changed_obj.connect(self.on_scale_changing) self.controlWidgetX.method_changed_str.connect( self.data_proc_X.on_method_changed) self.controlWidgetX.boards_changed.connect( self.data_proc_X.on_boards_changed) self.controlWidgetZ.method_changed_str.connect( self.data_proc_Z.on_method_changed) self.controlWidgetZ.boards_changed.connect( self.data_proc_Z.on_boards_changed) self.settingsControl = SettingsControl() self.settingsControl.add_object(self.controlWidgetX) self.settingsControl.add_object(self.controlWidgetZ) self.buttonRead.clicked.connect(self.on_read_button) self.buttonSave.clicked.connect(self.on_save_button) self.settingsControl.read_settings() self.data_proc_X.data_processed.connect(self.on_freq_status_X) self.data_proc_Z.data_processed.connect(self.on_freq_status_Z) self.plots_customization() self.controlWidgetX.boards_changed.connect(self.boards_X_changed) self.controlWidgetZ.boards_changed.connect(self.boards_Z_changed) self.data_curve1 = self.ui.plotX.plot(pen='r', title='Generated signal X_plot') self.data_curve2 = self.ui.plotFX.plot( pen='r', title='Fourier Transform X_plot') self.data_curve3 = self.ui.plotZ.plot(pen='b', title='Generated signal Z_plot') self.data_curve4 = self.ui.plotFZ.plot( pen='b', title='Fourier Transform Z_plot')
def initUI(self): #Menu self.menubar = self.menuBar() self.fileMenu = self.menubar.addMenu('&File') self.openAct = QAction('&Open', self) self.openAct.setShortcut('Ctrl+O') self.openAct.setIcon(QIcon("./image/open.ico")) self.saveAct = QAction('&Save', self) self.saveAct.setShortcut('Ctrl+S') self.saveAct.setIcon(QIcon("./image/save.ico")) self.importAct = QAction('&Import Data', self) self.importAct.setShortcut('Ctrl+I') self.importAct.setIcon(QIcon("./image/import.ico")) self.exportAct = QAction('&Export Data', self) self.exportAct.setShortcut('Ctrl+E') self.exportAct.setIcon(QIcon("./image/export.ico")) self.exportAct.setEnabled(False) self.exitAct = QAction('&Exit', self) self.exitAct.setIcon(QIcon("./image/exit.ico")) self.fileMenu.addAction(self.openAct) self.fileMenu.addAction(self.saveAct) self.fileMenu.addSeparator() self.fileMenu.addAction(self.importAct) self.fileMenu.addAction(self.exportAct) self.fileMenu.addSeparator() self.fileMenu.addAction(self.exitAct) self.importAct.triggered.connect(self.OnImport) self.exportAct.triggered.connect(self.OnExport) self.exitAct.triggered.connect(self.close) #Data Browser self.DataBrowser = DataBrowser(self) #Process Region Expansion button self.PRButton = QPushButton(">") self.PRButton.setFixedSize(20, 80) self.PRButton.setCheckable(True) self.PRButton.toggled.connect(self.showDataProcessor) #Data Processor self.DataProcessor = DataProcessor(self) #Image Viewer self.ImageViewer = ImageViewer(self) #Mayavi Region Expansion button self.MYButton = QPushButton(">") self.MYButton.setFixedSize(20, 80) self.MYButton.setCheckable(True) #self.MYButton.setEnabled(False) self.MYButton.toggled.connect(self.show3D) #Mayavi scene self.MYWidget = MYWidget(self) #Layout self.panel, self.splitter, self.Databox, self.DataWidget = self.WinLayout() QTimer.singleShot(10, lambda: self.splitter.moveSplitter(self.DataBrowser.minimumWidth(), 1)) self.splitter.splitterMoved.connect(self.splitterMovedEvent) #center panel self.centralPanel = QWidget(self) self.centralPanel.setLayout(self.panel) self.setCentralWidget(self.centralPanel) self.setWindowTitle('ARPES Data Viewer -- By Wei Yao -- Ver 1.0') self.show() self.initCompleteFlag = True
class MainWindow(QMainWindow): ''' MainWindow class ''' def __init__(self, App): super(MainWindow, self).__init__() self.App = App self.splitterPos = 0 self.initCompleteFlag = False self.signal = resizeSignal() self.setGeometry(300, 100, 900, 900) self.setWindowIcon(QIcon("./image/mainwin.ico")) self.initUI() self.signal.resize.connect(self.ImageViewer.resizeEvent_wrapper) def initUI(self): #Menu self.menubar = self.menuBar() self.fileMenu = self.menubar.addMenu('&File') self.openAct = QAction('&Open', self) self.openAct.setShortcut('Ctrl+O') self.openAct.setIcon(QIcon("./image/open.ico")) self.saveAct = QAction('&Save', self) self.saveAct.setShortcut('Ctrl+S') self.saveAct.setIcon(QIcon("./image/save.ico")) self.importAct = QAction('&Import Data', self) self.importAct.setShortcut('Ctrl+I') self.importAct.setIcon(QIcon("./image/import.ico")) self.exportAct = QAction('&Export Data', self) self.exportAct.setShortcut('Ctrl+E') self.exportAct.setIcon(QIcon("./image/export.ico")) self.exportAct.setEnabled(False) self.exitAct = QAction('&Exit', self) self.exitAct.setIcon(QIcon("./image/exit.ico")) self.fileMenu.addAction(self.openAct) self.fileMenu.addAction(self.saveAct) self.fileMenu.addSeparator() self.fileMenu.addAction(self.importAct) self.fileMenu.addAction(self.exportAct) self.fileMenu.addSeparator() self.fileMenu.addAction(self.exitAct) self.importAct.triggered.connect(self.OnImport) self.exportAct.triggered.connect(self.OnExport) self.exitAct.triggered.connect(self.close) #Data Browser self.DataBrowser = DataBrowser(self) #Process Region Expansion button self.PRButton = QPushButton(">") self.PRButton.setFixedSize(20, 80) self.PRButton.setCheckable(True) self.PRButton.toggled.connect(self.showDataProcessor) #Data Processor self.DataProcessor = DataProcessor(self) #Image Viewer self.ImageViewer = ImageViewer(self) #Mayavi Region Expansion button self.MYButton = QPushButton(">") self.MYButton.setFixedSize(20, 80) self.MYButton.setCheckable(True) #self.MYButton.setEnabled(False) self.MYButton.toggled.connect(self.show3D) #Mayavi scene self.MYWidget = MYWidget(self) #Layout self.panel, self.splitter, self.Databox, self.DataWidget = self.WinLayout() QTimer.singleShot(10, lambda: self.splitter.moveSplitter(self.DataBrowser.minimumWidth(), 1)) self.splitter.splitterMoved.connect(self.splitterMovedEvent) #center panel self.centralPanel = QWidget(self) self.centralPanel.setLayout(self.panel) self.setCentralWidget(self.centralPanel) self.setWindowTitle('ARPES Data Viewer -- By Wei Yao -- Ver 1.0') self.show() self.initCompleteFlag = True def WinLayout(self): panel = QHBoxLayout() splitter = QSplitter() splitter.addWidget(self.DataBrowser) Databox = QHBoxLayout() Databox.addWidget(self.PRButton) Databox.addWidget(self.ImageViewer) Databox.addWidget(self.MYButton) DataWidget = QWidget() DataWidget.setLayout(Databox) splitter.addWidget(DataWidget) splitter.setStretchFactor(0,0) splitter.setStretchFactor(1,1) panel.addWidget(splitter) return panel, splitter, Databox, DataWidget def showDataProcessor(self, state): if state: self.Databox.insertWidget(0, self.DataProcessor) self.DataProcessor.show() self.resizeDataWidget(self.DataProcessor.width()+7, 'DP') #7 is for additional space due to using layout self.PRButton.setText("<") else: self.Databox.takeAt(0) self.DataProcessor.hide() #wait for some events are processed in the event loop. #https://stackoverflow.com/questions/28660960/resize-qmainwindow-to-minimal-size-after-content-of-layout-changes #This QTimer is used to wait for correct process of layout QTimer.singleShot(10, lambda: self.resizeDataWidget(-self.DataProcessor.width()-7, 'DP')) self.PRButton.setText(">") def show3D(self, state): if self.DataProcessor.isVisible(): pos = 3 else: pos = 2 if state: self.Databox.insertWidget(pos, self.MYWidget) self.MYWidget.show() self.resizeDataWidget(self.MYWidget.width()+7, 'MY') self.MYButton.setText("<") self.MYWidget.setData(self.DataBrowser.DataList.currentItem().Data) else: self.Databox.takeAt(pos) self.MYWidget.hide() QTimer.singleShot(10, lambda: self.resizeDataWidget(-self.MYWidget.width()-7, 'MY')) self.MYButton.setText(">") def resizeDataWidget(self, dwidth, flag): if dwidth > 0: screen_width = QApplication.desktop().screenGeometry().width() if self.width()+dwidth+1 < screen_width: self.resize(self.width()+dwidth+1, self.height()) self.resize(self.width()-1, self.height()) # to redraw the gui else: self.resize(screen_width, self.height()) elif dwidth < 0: #This QTimer is used to wait for correct minimum width of datawidget if flag == 'DP': delay = 10 elif flag == 'MY': delay = 30 QTimer.singleShot(delay, lambda: self.resize(self.width()+dwidth, self.height())) def OnImport(self): filelist = QFileDialog.getOpenFileNames(self, "Import Data", ".", "Igor Packed Files(*.pxt; *.pxp);;ArPy Files(*.arpy);;Ig2Py Files(*.Ig2Py)")[0] if len(filelist) > 0: self.ImportData(filelist) def OnExport(self): name = self.DataBrowser.DataList.currentItem().Data.name savedfile = QFileDialog.getSaveFileName(self, "Export Data", name, "Igor Packed Files(*.pxt);;ArPy Files(*.arpy)")[0] if len(savedfile) > 0: self.ExportData(savedfile) def ImportData(self, filelist): for filepath in filelist: base = os.path.basename(filepath) ext = os.path.splitext(base)[1] if ext.lower() == ".ig2py": spec = Data.Ig2Py2Data(filepath) if spec != None: self.NewData(spec) elif ext.lower() == ".arpy": spec = Data.ArPy2Data(filepath) if spec != None: self.NewData(spec) elif ext.lower() == ".pxt": specList = IgorPackedFile2Data(filepath, True) for spec in specList: self.NewData(spec) elif ext.lower() == ".pxp": specList = IgorPackedFile2Data(filepath, False) for spec in specList: self.NewData(spec) def ExportData(self, savedfile): base = os.path.basename(savedfile) ext = os.path.splitext(base)[1] if ext == ".arpy": data = self.DataBrowser.DataList.currentItem().Data Data.Data2ArPy(data, savedfile) elif ext == ".pxt": #data = self.DataBrowser.DataList.currentItem().Data datalist = [item.Data for item in self.DataBrowser.DataList.selectedItems()] Data2IgorPackedFile(datalist, savedfile) def setData(self, data): self.DataProcessor.updateUI(data) self.ImageViewer.setData(data) if self.MYButton.isChecked(): self.MYWidget.setData(data) def NewData(self, data): self.DataBrowser.DataList.addDataItem(data) def resizeEvent(self, event): if self.DataBrowser.width() < self.splitterPos: self.splitter.moveSplitter(self.splitterPos, 1) elif self.initCompleteFlag: self.signal.resize.emit() def splitterMovedEvent(self, pos, idx): self.splitterPos = pos if pos > self.DataBrowser.minimumWidth(): if self.initCompleteFlag: self.signal.resize.emit() else: self.resize(self.width()+1, self.height()) # to redraw the gui self.resize(self.width()-1, self.height()) def closeEvent(self, event): self.DataBrowser.DataList.noteWin.close()
from encoder.oneplane import OnePlaneEncoder from encoder.sevenplanes import SevenPlaneEncoder # import arch import smallarch as arch from keras.models import Sequential from keras.layers.core import Dense from keras.callbacks import ModelCheckpoint from keras.utils import to_categorical go_board_rows, go_board_cols = 19, 19 num_classes = go_board_rows * go_board_cols num_games = 100 # encoder = OnePlaneEncoder((go_board_rows, go_board_cols)) encoder = SevenPlaneEncoder((go_board_rows, go_board_cols)) processor = DataProcessor(encoder) generator = processor.load_go_data('train', num_games, use_generator=True) X = generator.generate(32, num_classes) print(X) # test_generator =processor.load_go_data('test', num_games,use_generator=True) # from split import Splitter # dir = 'dataset/data' # splitter = Splitter(data_dir=dir) # data = splitter.draw_data('train', num_games) # data_test = splitter.draw_data('test', num_games) # generator = DataGenerator(dir, data) # test_generator = DataGenerator(dir,data_test)
def __init__(self, data_files=[], data_processor=DataProcessor(test_set_proportion=0.20, random_seed=42)): """ Parameters ---------- data_folder : str a directory path where data set files used to build models can be found should be suffixed with '/' """ self.data_files = data_files self.data_processor = data_processor self.model_constructors = { 'decision_tree': DecisionTreeClassifier, 'random_forest': RandomForestClassifier, 'naive_bayes': GaussianNB, 'svc_linear': SVC, 'svc_polynomial': SVC, 'svc_rbf': SVC, 'svc_sigmoid': SVC } self.model_params = { "decision_tree": { "constructor": None, "hp_search": { "max_depth": np.arange(1, 100), "min_samples_leaf": [1, 5, 10, 20, 50, 100, 250] } }, "random_forest": { "constructor": None, "hp_search": { "criterion": ["gini", "entropy"], "n_estimators": np.arange(10, 50), "max_depths": np.arange(1, 100), "min_samples_leaf": [1, 5, 10, 20, 50, 100, 250] } }, "naive_bayes": { "constructor": None, "hp_search": None }, "svc_linear": { "constructor": { "kernel": "linear" }, "hp_search": { 'C': [0.01, 0.1, 1, 10, 100], 'gamma': [1, 0.1, 0.01, 0.001, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9], } }, "svc_polynomial": { "constructor": { "kernel": "poly" }, "hp_search": { 'C': [0.01, 0.1, 1, 10, 100], 'gamma': [1, 0.1, 0.01, 0.001, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9], 'degree': np.arange(3, 10) } }, "svc_rbf": { "constructor": { "kernel": "rbf" }, "hp_search": { 'C': [0.01, 0.1, 1, 10, 100], 'gamma': [1, 0.1, 0.01, 0.001, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9], } }, "svc_sigmoid": { "constructor": { "kernel": "sigmoid" }, "hp_search": { 'C': [0.01, 0.1, 1, 10, 100], 'gamma': [1, 0.1, 0.01, 0.001, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9], } }, } self.processed_data_files = {}